1// SPDX-License-Identifier: GPL-2.0-or-later 2/* -*- linux-c -*- ------------------------------------------------------- * 3 * 4 * Copyright 2002 H. Peter Anvin - All Rights Reserved 5 * 6 * ----------------------------------------------------------------------- */ 7 8/* 9 * raid6/mmx.c 10 * 11 * MMX implementation of RAID-6 syndrome functions 12 */ 13 14#ifdef CONFIG_X86_32 15 16#include <linux/raid/pq.h> 17#include "x86.h" 18 19/* Shared with raid6/sse1.c */ 20const struct raid6_mmx_constants { 21 u64 x1d; 22} raid6_mmx_constants = { 23 0x1d1d1d1d1d1d1d1dULL, 24}; 25 26static int raid6_have_mmx(void) 27{ 28 /* Not really "boot_cpu" but "all_cpus" */ 29 return boot_cpu_has(X86_FEATURE_MMX); 30} 31 32/* 33 * Plain MMX implementation 34 */ 35static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs) 36{ 37 u8 **dptr = (u8 **)ptrs; 38 u8 *p, *q; 39 int d, z, z0; 40 41 z0 = disks - 3; /* Highest data disk */ 42 p = dptr[z0+1]; /* XOR parity */ 43 q = dptr[z0+2]; /* RS syndrome */ 44 45 kernel_fpu_begin(); 46 47 asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d)); 48 asm volatile("pxor %mm5,%mm5"); /* Zero temp */ 49 50 for ( d = 0 ; d < bytes ; d += 8 ) { 51 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */ 52 asm volatile("movq %mm2,%mm4"); /* Q[0] */ 53 for ( z = z0-1 ; z >= 0 ; z-- ) { 54 asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d])); 55 asm volatile("pcmpgtb %mm4,%mm5"); 56 asm volatile("paddb %mm4,%mm4"); 57 asm volatile("pand %mm0,%mm5"); 58 asm volatile("pxor %mm5,%mm4"); 59 asm volatile("pxor %mm5,%mm5"); 60 asm volatile("pxor %mm6,%mm2"); 61 asm volatile("pxor %mm6,%mm4"); 62 } 63 asm volatile("movq %%mm2,%0" : "=m" (p[d])); 64 asm volatile("pxor %mm2,%mm2"); 65 asm volatile("movq %%mm4,%0" : "=m" (q[d])); 66 asm volatile("pxor %mm4,%mm4"); 67 } 68 69 kernel_fpu_end(); 70} 71 72const struct raid6_calls raid6_mmxx1 = { 73 raid6_mmx1_gen_syndrome, 74 NULL, /* XOR not yet implemented */ 75 raid6_have_mmx, 76 "mmxx1", 77 0 78}; 79 80/* 81 * Unrolled-by-2 MMX implementation 82 */ 83static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs) 84{ 85 u8 **dptr = (u8 **)ptrs; 86 u8 *p, *q; 87 int d, z, z0; 88 89 z0 = disks - 3; /* Highest data disk */ 90 p = dptr[z0+1]; /* XOR parity */ 91 q = dptr[z0+2]; /* RS syndrome */ 92 93 kernel_fpu_begin(); 94 95 asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d)); 96 asm volatile("pxor %mm5,%mm5"); /* Zero temp */ 97 asm volatile("pxor %mm7,%mm7"); /* Zero temp */ 98 99 for ( d = 0 ; d < bytes ; d += 16 ) { 100 asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */ 101 asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8])); 102 asm volatile("movq %mm2,%mm4"); /* Q[0] */ 103 asm volatile("movq %mm3,%mm6"); /* Q[1] */ 104 for ( z = z0-1 ; z >= 0 ; z-- ) { 105 asm volatile("pcmpgtb %mm4,%mm5"); 106 asm volatile("pcmpgtb %mm6,%mm7"); 107 asm volatile("paddb %mm4,%mm4"); 108 asm volatile("paddb %mm6,%mm6"); 109 asm volatile("pand %mm0,%mm5"); 110 asm volatile("pand %mm0,%mm7"); 111 asm volatile("pxor %mm5,%mm4"); 112 asm volatile("pxor %mm7,%mm6"); 113 asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d])); 114 asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8])); 115 asm volatile("pxor %mm5,%mm2"); 116 asm volatile("pxor %mm7,%mm3"); 117 asm volatile("pxor %mm5,%mm4"); 118 asm volatile("pxor %mm7,%mm6"); 119 asm volatile("pxor %mm5,%mm5"); 120 asm volatile("pxor %mm7,%mm7"); 121 } 122 asm volatile("movq %%mm2,%0" : "=m" (p[d])); 123 asm volatile("movq %%mm3,%0" : "=m" (p[d+8])); 124 asm volatile("movq %%mm4,%0" : "=m" (q[d])); 125 asm volatile("movq %%mm6,%0" : "=m" (q[d+8])); 126 } 127 128 kernel_fpu_end(); 129} 130 131const struct raid6_calls raid6_mmxx2 = { 132 raid6_mmx2_gen_syndrome, 133 NULL, /* XOR not yet implemented */ 134 raid6_have_mmx, 135 "mmxx2", 136 0 137}; 138 139#endif 140