xref: /kernel/linux/linux-5.10/lib/raid6/avx512.c (revision 8c2ecf20) 
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* -*- linux-c -*- --------------------------------------------------------
3 *
4 *   Copyright (C) 2016 Intel Corporation
5 *
6 *   Author: Gayatri Kammela <gayatri.kammela@intel.com>
7 *   Author: Megha Dey <megha.dey@linux.intel.com>
8 *
9 *   Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
10 *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
11 *
12 * -----------------------------------------------------------------------
13 */
14
15/*
16 * AVX512 implementation of RAID-6 syndrome functions
17 *
18 */
19
20#ifdef CONFIG_AS_AVX512
21
22#include <linux/raid/pq.h>
23#include "x86.h"
24
25static const struct raid6_avx512_constants {
26	u64 x1d[8];
27} raid6_avx512_constants __aligned(512/8) = {
28	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
29	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
30	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
31	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
32};
33
34static int raid6_have_avx512(void)
35{
36	return boot_cpu_has(X86_FEATURE_AVX2) &&
37		boot_cpu_has(X86_FEATURE_AVX) &&
38		boot_cpu_has(X86_FEATURE_AVX512F) &&
39		boot_cpu_has(X86_FEATURE_AVX512BW) &&
40		boot_cpu_has(X86_FEATURE_AVX512VL) &&
41		boot_cpu_has(X86_FEATURE_AVX512DQ);
42}
43
44static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
45{
46	u8 **dptr = (u8 **)ptrs;
47	u8 *p, *q;
48	int d, z, z0;
49
50	z0 = disks - 3;         /* Highest data disk */
51	p = dptr[z0+1];         /* XOR parity */
52	q = dptr[z0+2];         /* RS syndrome */
53
54	kernel_fpu_begin();
55
56	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
57		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
58		     :
59		     : "m" (raid6_avx512_constants.x1d[0]));
60
61	for (d = 0; d < bytes; d += 64) {
62		asm volatile("prefetchnta %0\n\t"
63			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
64			     "prefetchnta %1\n\t"
65			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
66			     "vmovdqa64 %1,%%zmm6"
67			     :
68			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
69		for (z = z0-2; z >= 0; z--) {
70			asm volatile("prefetchnta %0\n\t"
71				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
72				     "vpmovm2b %%k1,%%zmm5\n\t"
73				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
74				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
75				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
76				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
77				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
78				     "vmovdqa64 %0,%%zmm6"
79				     :
80				     : "m" (dptr[z][d]));
81		}
82		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
83			     "vpmovm2b %%k1,%%zmm5\n\t"
84			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
85			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
86			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
87			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
88			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
89			     "vmovntdq %%zmm2,%0\n\t"
90			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
91			     "vmovntdq %%zmm4,%1\n\t"
92			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
93			     :
94			     : "m" (p[d]), "m" (q[d]));
95	}
96
97	asm volatile("sfence" : : : "memory");
98	kernel_fpu_end();
99}
100
101static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
102				       size_t bytes, void **ptrs)
103{
104	u8 **dptr = (u8 **)ptrs;
105	u8 *p, *q;
106	int d, z, z0;
107
108	z0 = stop;		/* P/Q right side optimization */
109	p = dptr[disks-2];	/* XOR parity */
110	q = dptr[disks-1];	/* RS syndrome */
111
112	kernel_fpu_begin();
113
114	asm volatile("vmovdqa64 %0,%%zmm0"
115		     : : "m" (raid6_avx512_constants.x1d[0]));
116
117	for (d = 0 ; d < bytes ; d += 64) {
118		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
119			     "vmovdqa64 %1,%%zmm2\n\t"
120			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
121			     :
122			     : "m" (dptr[z0][d]),  "m" (p[d]));
123		/* P/Q data pages */
124		for (z = z0-1 ; z >= start ; z--) {
125			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
126				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
127				     "vpmovm2b %%k1,%%zmm5\n\t"
128				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
129				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
130				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
131				     "vmovdqa64 %0,%%zmm5\n\t"
132				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
133				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
134				     :
135				     : "m" (dptr[z][d]));
136		}
137		/* P/Q left side optimization */
138		for (z = start-1 ; z >= 0 ; z--) {
139			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
140				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
141				     "vpmovm2b %%k1,%%zmm5\n\t"
142				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
143				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
144				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
145				     :
146				     : );
147		}
148		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
149		/* Don't use movntdq for r/w memory area < cache line */
150			     "vmovdqa64 %%zmm4,%0\n\t"
151			     "vmovdqa64 %%zmm2,%1"
152			     :
153			     : "m" (q[d]), "m" (p[d]));
154	}
155
156	asm volatile("sfence" : : : "memory");
157	kernel_fpu_end();
158}
159
160const struct raid6_calls raid6_avx512x1 = {
161	raid6_avx5121_gen_syndrome,
162	raid6_avx5121_xor_syndrome,
163	raid6_have_avx512,
164	"avx512x1",
165	1                       /* Has cache hints */
166};
167
168/*
169 * Unrolled-by-2 AVX512 implementation
170 */
171static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
172{
173	u8 **dptr = (u8 **)ptrs;
174	u8 *p, *q;
175	int d, z, z0;
176
177	z0 = disks - 3;         /* Highest data disk */
178	p = dptr[z0+1];         /* XOR parity */
179	q = dptr[z0+2];         /* RS syndrome */
180
181	kernel_fpu_begin();
182
183	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
184		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
185		     :
186		     : "m" (raid6_avx512_constants.x1d[0]));
187
188	/* We uniformly assume a single prefetch covers at least 64 bytes */
189	for (d = 0; d < bytes; d += 128) {
190		asm volatile("prefetchnta %0\n\t"
191			     "prefetchnta %1\n\t"
192			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
193			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
194			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
195			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
196			     :
197			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
198		for (z = z0-1; z >= 0; z--) {
199			asm volatile("prefetchnta %0\n\t"
200				     "prefetchnta %1\n\t"
201				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
202				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
203				     "vpmovm2b %%k1,%%zmm5\n\t"
204				     "vpmovm2b %%k2,%%zmm7\n\t"
205				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
206				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
207				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
208				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
209				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
210				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
211				     "vmovdqa64 %0,%%zmm5\n\t"
212				     "vmovdqa64 %1,%%zmm7\n\t"
213				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
214				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
215				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
216				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
217				     :
218				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
219		}
220		asm volatile("vmovntdq %%zmm2,%0\n\t"
221			     "vmovntdq %%zmm3,%1\n\t"
222			     "vmovntdq %%zmm4,%2\n\t"
223			     "vmovntdq %%zmm6,%3"
224			     :
225			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
226			       "m" (q[d+64]));
227	}
228
229	asm volatile("sfence" : : : "memory");
230	kernel_fpu_end();
231}
232
233static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
234				       size_t bytes, void **ptrs)
235{
236	u8 **dptr = (u8 **)ptrs;
237	u8 *p, *q;
238	int d, z, z0;
239
240	z0 = stop;		/* P/Q right side optimization */
241	p = dptr[disks-2];	/* XOR parity */
242	q = dptr[disks-1];	/* RS syndrome */
243
244	kernel_fpu_begin();
245
246	asm volatile("vmovdqa64 %0,%%zmm0"
247		     : : "m" (raid6_avx512_constants.x1d[0]));
248
249	for (d = 0 ; d < bytes ; d += 128) {
250		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
251			     "vmovdqa64 %1,%%zmm6\n\t"
252			     "vmovdqa64 %2,%%zmm2\n\t"
253			     "vmovdqa64 %3,%%zmm3\n\t"
254			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
255			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
256			     :
257			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
258			       "m" (p[d]), "m" (p[d+64]));
259		/* P/Q data pages */
260		for (z = z0-1 ; z >= start ; z--) {
261			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
262				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
263				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
264				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
265				     "vpmovm2b %%k1,%%zmm5\n\t"
266				     "vpmovm2b %%k2,%%zmm7\n\t"
267				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
268				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
269				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
270				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
271				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
272				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
273				     "vmovdqa64 %0,%%zmm5\n\t"
274				     "vmovdqa64 %1,%%zmm7\n\t"
275				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
276				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
277				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
278				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
279				     :
280				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
281		}
282		/* P/Q left side optimization */
283		for (z = start-1 ; z >= 0 ; z--) {
284			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
285				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
286				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
287				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
288				     "vpmovm2b %%k1,%%zmm5\n\t"
289				     "vpmovm2b %%k2,%%zmm7\n\t"
290				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
291				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
292				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
293				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
294				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
295				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
296				     :
297				     : );
298		}
299		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
300			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
301			     /* Don't use movntdq for r/w
302			      * memory area < cache line
303			      */
304			     "vmovdqa64 %%zmm4,%0\n\t"
305			     "vmovdqa64 %%zmm6,%1\n\t"
306			     "vmovdqa64 %%zmm2,%2\n\t"
307			     "vmovdqa64 %%zmm3,%3"
308			     :
309			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
310			       "m" (p[d+64]));
311	}
312
313	asm volatile("sfence" : : : "memory");
314	kernel_fpu_end();
315}
316
317const struct raid6_calls raid6_avx512x2 = {
318	raid6_avx5122_gen_syndrome,
319	raid6_avx5122_xor_syndrome,
320	raid6_have_avx512,
321	"avx512x2",
322	1                       /* Has cache hints */
323};
324
325#ifdef CONFIG_X86_64
326
327/*
328 * Unrolled-by-4 AVX2 implementation
329 */
330static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
331{
332	u8 **dptr = (u8 **)ptrs;
333	u8 *p, *q;
334	int d, z, z0;
335
336	z0 = disks - 3;         /* Highest data disk */
337	p = dptr[z0+1];         /* XOR parity */
338	q = dptr[z0+2];         /* RS syndrome */
339
340	kernel_fpu_begin();
341
342	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
343		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
344		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
345		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
346		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
347		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
348		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
349		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
350		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
351		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
352		     :
353		     : "m" (raid6_avx512_constants.x1d[0]));
354
355	for (d = 0; d < bytes; d += 256) {
356		for (z = z0; z >= 0; z--) {
357		asm volatile("prefetchnta %0\n\t"
358			     "prefetchnta %1\n\t"
359			     "prefetchnta %2\n\t"
360			     "prefetchnta %3\n\t"
361			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
362			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
363			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
364			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
365			     "vpmovm2b %%k1,%%zmm5\n\t"
366			     "vpmovm2b %%k2,%%zmm7\n\t"
367			     "vpmovm2b %%k3,%%zmm13\n\t"
368			     "vpmovm2b %%k4,%%zmm15\n\t"
369			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
370			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
371			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
372			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
373			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
374			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
375			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
376			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
377			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
378			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
379			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
380			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
381			     "vmovdqa64 %0,%%zmm5\n\t"
382			     "vmovdqa64 %1,%%zmm7\n\t"
383			     "vmovdqa64 %2,%%zmm13\n\t"
384			     "vmovdqa64 %3,%%zmm15\n\t"
385			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
386			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
387			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
388			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
389			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
390			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
391			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
392			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
393			     :
394			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
395			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
396		}
397		asm volatile("vmovntdq %%zmm2,%0\n\t"
398			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
399			     "vmovntdq %%zmm3,%1\n\t"
400			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
401			     "vmovntdq %%zmm10,%2\n\t"
402			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
403			     "vmovntdq %%zmm11,%3\n\t"
404			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
405			     "vmovntdq %%zmm4,%4\n\t"
406			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
407			     "vmovntdq %%zmm6,%5\n\t"
408			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
409			     "vmovntdq %%zmm12,%6\n\t"
410			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
411			     "vmovntdq %%zmm14,%7\n\t"
412			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
413			     :
414			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
415			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
416			       "m" (q[d+128]), "m" (q[d+192]));
417	}
418
419	asm volatile("sfence" : : : "memory");
420	kernel_fpu_end();
421}
422
423static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
424				       size_t bytes, void **ptrs)
425{
426	u8 **dptr = (u8 **)ptrs;
427	u8 *p, *q;
428	int d, z, z0;
429
430	z0 = stop;		/* P/Q right side optimization */
431	p = dptr[disks-2];	/* XOR parity */
432	q = dptr[disks-1];	/* RS syndrome */
433
434	kernel_fpu_begin();
435
436	asm volatile("vmovdqa64 %0,%%zmm0"
437		     :: "m" (raid6_avx512_constants.x1d[0]));
438
439	for (d = 0 ; d < bytes ; d += 256) {
440		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
441			     "vmovdqa64 %1,%%zmm6\n\t"
442			     "vmovdqa64 %2,%%zmm12\n\t"
443			     "vmovdqa64 %3,%%zmm14\n\t"
444			     "vmovdqa64 %4,%%zmm2\n\t"
445			     "vmovdqa64 %5,%%zmm3\n\t"
446			     "vmovdqa64 %6,%%zmm10\n\t"
447			     "vmovdqa64 %7,%%zmm11\n\t"
448			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
449			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
450			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
451			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
452			     :
453			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
454			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
455			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
456			       "m" (p[d+192]));
457		/* P/Q data pages */
458		for (z = z0-1 ; z >= start ; z--) {
459			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
460				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
461				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
462				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
463				     "prefetchnta %0\n\t"
464				     "prefetchnta %2\n\t"
465				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
466				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
467				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
468				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
469				     "vpmovm2b %%k1,%%zmm5\n\t"
470				     "vpmovm2b %%k2,%%zmm7\n\t"
471				     "vpmovm2b %%k3,%%zmm13\n\t"
472				     "vpmovm2b %%k4,%%zmm15\n\t"
473				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
474				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
475				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
476				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
477				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
478				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
479				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
480				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
481				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
482				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
483				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
484				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
485				     "vmovdqa64 %0,%%zmm5\n\t"
486				     "vmovdqa64 %1,%%zmm7\n\t"
487				     "vmovdqa64 %2,%%zmm13\n\t"
488				     "vmovdqa64 %3,%%zmm15\n\t"
489				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
490				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
491				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
492				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
493				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
494				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
495				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
496				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
497				     :
498				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
499				       "m" (dptr[z][d+128]),
500				       "m" (dptr[z][d+192]));
501		}
502		asm volatile("prefetchnta %0\n\t"
503			     "prefetchnta %1\n\t"
504			     :
505			     : "m" (q[d]), "m" (q[d+128]));
506		/* P/Q left side optimization */
507		for (z = start-1 ; z >= 0 ; z--) {
508			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
509				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
510				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
511				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
512				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
513				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
514				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
515				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
516				     "vpmovm2b %%k1,%%zmm5\n\t"
517				     "vpmovm2b %%k2,%%zmm7\n\t"
518				     "vpmovm2b %%k3,%%zmm13\n\t"
519				     "vpmovm2b %%k4,%%zmm15\n\t"
520				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
521				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
522				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
523				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
524				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
525				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
526				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
527				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
528				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
529				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
530				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
531				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
532				     :
533				     : );
534		}
535		asm volatile("vmovntdq %%zmm2,%0\n\t"
536			     "vmovntdq %%zmm3,%1\n\t"
537			     "vmovntdq %%zmm10,%2\n\t"
538			     "vmovntdq %%zmm11,%3\n\t"
539			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
540			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
541			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
542			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
543			     "vmovntdq %%zmm4,%4\n\t"
544			     "vmovntdq %%zmm6,%5\n\t"
545			     "vmovntdq %%zmm12,%6\n\t"
546			     "vmovntdq %%zmm14,%7"
547			     :
548			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
549			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
550			       "m" (q[d+128]), "m" (q[d+192]));
551	}
552	asm volatile("sfence" : : : "memory");
553	kernel_fpu_end();
554}
555const struct raid6_calls raid6_avx512x4 = {
556	raid6_avx5124_gen_syndrome,
557	raid6_avx5124_xor_syndrome,
558	raid6_have_avx512,
559	"avx512x4",
560	1                       /* Has cache hints */
561};
562#endif
563
564#endif /* CONFIG_AS_AVX512 */
565