Lines Matching +full:- +full:8 +full:g
2 // Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
5 // Copyright (C) 2019-2024 Google LLC
17 // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
65 // /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
72 .arch armv8-a+crypto
96 * Pairwise long polynomial multiplication of two 16-bit values
100 * by two 64-bit values
107 * This can be implemented using 8x8 long polynomial multiplication, by
108 * reorganizing the input so that each pairwise 8x8 multiplication
114 * 1 (w0*x1 ^ w1*x0) << 8 ^ | (y0*z1 ^ y1*z0) << 8 ^
121 * 8 w1*x7 << 64 | y1*z7 << 64
128 * and after performing 8x8->16 bit long polynomial multiplication of
130 * we obtain the following four vectors of 16-bit elements:
141 * 80-bit results.
158 ext t6.16b, t5.16b, t5.16b, #8
160 pmull t3.8h, t7.8b, t5.8b
161 pmull t4.8h, t7.8b, t6.8b
162 pmull2 t5.8h, t7.16b, t5.16b
163 pmull2 t6.8h, t7.16b, t6.16b
165 ext t8.16b, t3.16b, t3.16b, #8
167 ext t7.16b, t5.16b, t5.16b, #8
168 ext t6.16b, t4.16b, t4.16b, #8
169 eor t8.8b, t8.8b, t3.8b
170 eor t5.8b, t5.8b, t7.8b
171 eor t4.8b, t4.8b, t6.8b
189 CPU_LE( ext v11.16b, v11.16b, v11.16b, #8 )
190 CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 )
231 CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
232 CPU_LE( ext v1.16b, v1.16b, v1.16b, #8 )
233 CPU_LE( ext v2.16b, v2.16b, v2.16b, #8 )
234 CPU_LE( ext v3.16b, v3.16b, v3.16b, #8 )
235 CPU_LE( ext v4.16b, v4.16b, v4.16b, #8 )
236 CPU_LE( ext v5.16b, v5.16b, v5.16b, #8 )
237 CPU_LE( ext v6.16b, v6.16b, v6.16b, #8 )
238 CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
252 // While >= 128 data bytes remain (not counting v0-v7), fold the 128
253 // bytes v0-v7 into them, storing the result back into v0-v7.
263 // Now fold the 112 bytes in v0-v6 into the 16 bytes in v7.
282 adds len, len, #(128-16)
292 CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
312 ldr q0, [buf, #-16]
314 CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 )
316 // v1 = high order part of second chunk: v7 left-shifted by 'len' bytes.
322 // v3 = first chunk: v7 right-shifted by '16-len' bytes.
327 // Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes.
330 // v2 = second chunk: 'len' bytes from v0 (low-order bytes),
331 // then '16-len' bytes from v1 (high-order bytes).
348 CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
355 // Load the fold-across-16-bytes constants.
376 // Compose { 0,0,0,0, 8,8,8,8, 1,1,1,1, 9,9,9,9 }
377 movi perm.4h, #8, lsl #8
386 CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 )
402 // Reduce the 128-bit value M(x), stored in v7, to the final 16-bit CRC.
406 // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'.
410 // x^64. This produces a 128-bit value congruent to x^64 * M(x) and
412 ext v0.16b, v2.16b, v7.16b, #8
413 pmull2 v7.1q, v7.2d, fold_consts.2d // high bits * x^48 * (x^80 mod G(x))
416 // Fold the high 32 bits into the low 96 bits. This produces a 96-bit
420 pmull v1.1q, v1.1d, fold_consts.1d // high 32 bits * x^48 * (x^48 mod G(x))
423 // Load G(x) and floor(x^48 / G(x)).
427 pmull2 v1.1q, v0.2d, fold_consts.2d // high 32 bits * floor(x^48 / G(x))
429 pmull v1.1q, v1.1d, fold_consts.1d // *= G(x)
432 // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0.
442 // G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0
444 .quad 0x0000000000006123 // x^(8*128) mod G(x)
445 .quad 0x0000000000002295 // x^(8*128+64) mod G(x)
447 .quad 0x0000000000001069 // x^(4*128) mod G(x)
448 .quad 0x000000000000dd31 // x^(4*128+64) mod G(x)
450 .quad 0x000000000000857d // x^(2*128) mod G(x)
451 .quad 0x0000000000007acc // x^(2*128+64) mod G(x)
453 .quad 0x000000000000a010 // x^(1*128) mod G(x)
454 .quad 0x0000000000001faa // x^(1*128+64) mod G(x)
456 .quad 0x1368000000000000 // x^48 * (x^48 mod G(x))
457 .quad 0x2d56000000000000 // x^48 * (x^80 mod G(x))
459 .quad 0x0000000000018bb7 // G(x)
460 .quad 0x00000001f65a57f8 // floor(x^48 / G(x))
462 // For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 -
464 // ..., 0x80} XOR the index vector to shift right by '16 - len' bytes.