1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/crc32.h>
3 #include <crypto/internal/simd.h>
4 #include <linux/init.h>
5 #include <linux/module.h>
6 #include <linux/kernel.h>
7 #include <linux/cpufeature.h>
8 #include <asm/simd.h>
9 #include <asm/switch_to.h>
10
11 #define VMX_ALIGN 16
12 #define VMX_ALIGN_MASK (VMX_ALIGN-1)
13
14 #define VECTOR_BREAKPOINT 512
15
16 static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
17
18 u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
19
crc32_le_arch(u32 crc,const u8 * p,size_t len)20 u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21 {
22 return crc32_le_base(crc, p, len);
23 }
24 EXPORT_SYMBOL(crc32_le_arch);
25
crc32c_le_arch(u32 crc,const u8 * p,size_t len)26 u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
27 {
28 unsigned int prealign;
29 unsigned int tail;
30
31 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33 return crc32c_le_base(crc, p, len);
34
35 if ((unsigned long)p & VMX_ALIGN_MASK) {
36 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37 crc = crc32c_le_base(crc, p, prealign);
38 len -= prealign;
39 p += prealign;
40 }
41
42 if (len & ~VMX_ALIGN_MASK) {
43 preempt_disable();
44 pagefault_disable();
45 enable_kernel_altivec();
46 crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
47 disable_kernel_altivec();
48 pagefault_enable();
49 preempt_enable();
50 }
51
52 tail = len & VMX_ALIGN_MASK;
53 if (tail) {
54 p += len & ~VMX_ALIGN_MASK;
55 crc = crc32c_le_base(crc, p, tail);
56 }
57
58 return crc;
59 }
60 EXPORT_SYMBOL(crc32c_le_arch);
61
crc32_be_arch(u32 crc,const u8 * p,size_t len)62 u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
63 {
64 return crc32_be_base(crc, p, len);
65 }
66 EXPORT_SYMBOL(crc32_be_arch);
67
crc32_powerpc_init(void)68 static int __init crc32_powerpc_init(void)
69 {
70 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
71 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
72 static_branch_enable(&have_vec_crypto);
73 return 0;
74 }
75 arch_initcall(crc32_powerpc_init);
76
crc32_powerpc_exit(void)77 static void __exit crc32_powerpc_exit(void)
78 {
79 }
80 module_exit(crc32_powerpc_exit);
81
crc32_optimizations(void)82 u32 crc32_optimizations(void)
83 {
84 if (static_key_enabled(&have_vec_crypto))
85 return CRC32C_OPTIMIZATION;
86 return 0;
87 }
88 EXPORT_SYMBOL(crc32_optimizations);
89
90 MODULE_AUTHOR("Anton Blanchard <[email protected]>");
91 MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
92 MODULE_LICENSE("GPL");
93