1 // SPDX-License-Identifier: GPL-2.0-only
30 	const u8 x0f = 0x0f;  in raid6_2data_recov_avx512()
32 	p = (u8 *)ptrs[disks-2];  in raid6_2data_recov_avx512()
33 	q = (u8 *)ptrs[disks-1];  in raid6_2data_recov_avx512()
43 	ptrs[disks-2] = dp;  in raid6_2data_recov_avx512()
46 	ptrs[disks-1] = dq;  in raid6_2data_recov_avx512()
53 	ptrs[disks-2] = p;  in raid6_2data_recov_avx512()
54 	ptrs[disks-1] = q;  in raid6_2data_recov_avx512()
57 	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];  in raid6_2data_recov_avx512()
64 	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));  in raid6_2data_recov_avx512()
68 		asm volatile("vmovdqa64 %0, %%zmm1\n\t"  in raid6_2data_recov_avx512()
77 			     : "m" (q[0]), "m" (q[64]), "m" (p[0]),  in raid6_2data_recov_avx512()
78 			       "m" (p[64]), "m" (dq[0]), "m" (dq[64]),  in raid6_2data_recov_avx512()
79 			       "m" (dp[0]), "m" (dp[64]));  in raid6_2data_recov_avx512()
82 		 * 1 = dq[0]  ^ q[0]  in raid6_2data_recov_avx512()
83 		 * 9 = dq[64] ^ q[64]  in raid6_2data_recov_avx512()
84 		 * 0 = dp[0]  ^ p[0]  in raid6_2data_recov_avx512()
85 		 * 8 = dp[64] ^ p[64]  in raid6_2data_recov_avx512()
88 		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"  in raid6_2data_recov_avx512()
91 			     : "m" (qmul[0]), "m" (qmul[16]));  in raid6_2data_recov_avx512()
109 		 * 5 = qx[0]  in raid6_2data_recov_avx512()
110 		 * 15 = qx[64]  in raid6_2data_recov_avx512()
113 		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"  in raid6_2data_recov_avx512()
128 			     : "m" (pbmul[0]), "m" (pbmul[16]));  in raid6_2data_recov_avx512()
131 		 * 1  = pbmul[px[0]]  in raid6_2data_recov_avx512()
132 		 * 13 = pbmul[px[64]]  in raid6_2data_recov_avx512()
141 		 * 13 = db[64] = DQ[64]  in raid6_2data_recov_avx512()
143 		asm volatile("vmovdqa64 %%zmm1, %0\n\t"  in raid6_2data_recov_avx512()
148 			     : "m" (dq[0]), "m" (dq[64]));  in raid6_2data_recov_avx512()
150 		asm volatile("vmovdqa64 %%zmm0, %0\n\t"  in raid6_2data_recov_avx512()
153 			     : "m" (dp[0]), "m" (dp[64]));  in raid6_2data_recov_avx512()
155 		bytes -= 128;  in raid6_2data_recov_avx512()
161 		asm volatile("vmovdqa64 %0, %%zmm1\n\t"  in raid6_2data_recov_avx512()
168 		/* 1 = dq ^ q;  0 = dp ^ p */  in raid6_2data_recov_avx512()
170 		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"  in raid6_2data_recov_avx512()
173 			     : "m" (qmul[0]), "m" (qmul[16]));  in raid6_2data_recov_avx512()
190 		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"  in raid6_2data_recov_avx512()
193 			     : "m" (pbmul[0]), "m" (pbmul[16]));  in raid6_2data_recov_avx512()
207 			     "vmovdqa64 %%zmm1, %0\n\t"  in raid6_2data_recov_avx512()
209 			     : "m" (dq[0]));  in raid6_2data_recov_avx512()
212 			     "vmovdqa64 %%zmm0, %0"  in raid6_2data_recov_avx512()
214 			     : "m" (dp[0]));  in raid6_2data_recov_avx512()
216 		bytes -= 64;  in raid6_2data_recov_avx512()
217 		p += 64;  in raid6_2data_recov_avx512()
218 		q += 64;  in raid6_2data_recov_avx512()
219 		dp += 64;  in raid6_2data_recov_avx512()
220 		dq += 64;  in raid6_2data_recov_avx512()
232 	const u8 x0f = 0x0f;  in raid6_datap_recov_avx512()
234 	p = (u8 *)ptrs[disks-2];  in raid6_datap_recov_avx512()
235 	q = (u8 *)ptrs[disks-1];  in raid6_datap_recov_avx512()
244 	ptrs[disks-1] = dq;  in raid6_datap_recov_avx512()
250 	ptrs[disks-1] = q;  in raid6_datap_recov_avx512()
257 	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));  in raid6_datap_recov_avx512()
261 		asm volatile("vmovdqa64 %0, %%zmm3\n\t"  in raid6_datap_recov_avx512()
266 			     : "m" (dq[0]), "m" (dq[64]), "m" (q[0]),  in raid6_datap_recov_avx512()
267 			       "m" (q[64]));  in raid6_datap_recov_avx512()
270 		 * 3 = q[0] ^ dq[0]  in raid6_datap_recov_avx512()
271 		 * 8 = q[64] ^ dq[64]  in raid6_datap_recov_avx512()
273 		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"  in raid6_datap_recov_avx512()
278 			     : "m" (qmul[0]), "m" (qmul[16]));  in raid6_datap_recov_avx512()
296 		 * 1  = qmul[q[0]  ^ dq[0]]  in raid6_datap_recov_avx512()
297 		 * 14 = qmul[q[64] ^ dq[64]]  in raid6_datap_recov_avx512()
299 		asm volatile("vmovdqa64 %0, %%zmm2\n\t"  in raid6_datap_recov_avx512()
304 			     : "m" (p[0]), "m" (p[64]));  in raid6_datap_recov_avx512()
307 		 * 2  = p[0]  ^ qmul[q[0]  ^ dq[0]]  in raid6_datap_recov_avx512()
308 		 * 12 = p[64] ^ qmul[q[64] ^ dq[64]]  in raid6_datap_recov_avx512()
311 		asm volatile("vmovdqa64 %%zmm1, %0\n\t"  in raid6_datap_recov_avx512()
316 			     : "m" (dq[0]), "m" (dq[64]), "m" (p[0]),  in raid6_datap_recov_avx512()
317 			       "m" (p[64]));  in raid6_datap_recov_avx512()
319 		bytes -= 128;  in raid6_datap_recov_avx512()
324 		asm volatile("vmovdqa64 %0, %%zmm3\n\t"  in raid6_datap_recov_avx512()
327 			     : "m" (dq[0]), "m" (q[0]));  in raid6_datap_recov_avx512()
331 		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"  in raid6_datap_recov_avx512()
334 			     : "m" (qmul[0]), "m" (qmul[16]));  in raid6_datap_recov_avx512()
347 		asm volatile("vmovdqa64 %0, %%zmm2\n\t"  in raid6_datap_recov_avx512()
350 			     : "m" (p[0]));  in raid6_datap_recov_avx512()
354 		asm volatile("vmovdqa64 %%zmm1, %0\n\t"  in raid6_datap_recov_avx512()
357 			     : "m" (dq[0]), "m" (p[0]));  in raid6_datap_recov_avx512()
359 		bytes -= 64;  in raid6_datap_recov_avx512()
360 		p += 64;  in raid6_datap_recov_avx512()
361 		q += 64;  in raid6_datap_recov_avx512()
362 		dq += 64;  in raid6_datap_recov_avx512()