1 //===-- Memcmp implementation for x86_64 ------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMCMP_H
10 #define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMCMP_H
11
12 #include "src/__support/macros/attributes.h" // LIBC_INLINE
13 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
14 #include "src/string/memory_utils/op_generic.h"
15 #include "src/string/memory_utils/op_x86.h"
16 #include "src/string/memory_utils/utils.h" // MemcmpReturnType
17
18 namespace LIBC_NAMESPACE_DECL {
19
20 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_generic_gt16(CPtr p1,CPtr p2,size_t count)21 inline_memcmp_generic_gt16(CPtr p1, CPtr p2, size_t count) {
22 return generic::Memcmp<uint64_t>::loop_and_tail_align_above(384, p1, p2,
23 count);
24 }
25
26 #if defined(__SSE4_1__)
27 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_x86_sse41_gt16(CPtr p1,CPtr p2,size_t count)28 inline_memcmp_x86_sse41_gt16(CPtr p1, CPtr p2, size_t count) {
29 return generic::Memcmp<__m128i>::loop_and_tail_align_above(384, p1, p2,
30 count);
31 }
32 #endif // __SSE4_1__
33
34 #if defined(__AVX2__)
35 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_x86_avx2_gt16(CPtr p1,CPtr p2,size_t count)36 inline_memcmp_x86_avx2_gt16(CPtr p1, CPtr p2, size_t count) {
37 if (count <= 32)
38 return generic::Memcmp<__m128i>::head_tail(p1, p2, count);
39 if (count <= 64)
40 return generic::Memcmp<__m256i>::head_tail(p1, p2, count);
41 return generic::Memcmp<__m256i>::loop_and_tail_align_above(384, p1, p2,
42 count);
43 }
44 #endif // __AVX2__
45
46 #if defined(__AVX512BW__)
47 [[maybe_unused]] LIBC_INLINE MemcmpReturnType
inline_memcmp_x86_avx512bw_gt16(CPtr p1,CPtr p2,size_t count)48 inline_memcmp_x86_avx512bw_gt16(CPtr p1, CPtr p2, size_t count) {
49 if (count <= 32)
50 return generic::Memcmp<__m128i>::head_tail(p1, p2, count);
51 if (count <= 64)
52 return generic::Memcmp<__m256i>::head_tail(p1, p2, count);
53 if (count <= 128)
54 return generic::Memcmp<__m512i>::head_tail(p1, p2, count);
55 return generic::Memcmp<__m512i>::loop_and_tail_align_above(384, p1, p2,
56 count);
57 }
58 #endif // __AVX512BW__
59
inline_memcmp_x86(CPtr p1,CPtr p2,size_t count)60 LIBC_INLINE MemcmpReturnType inline_memcmp_x86(CPtr p1, CPtr p2, size_t count) {
61 if (count == 0)
62 return MemcmpReturnType::zero();
63 if (count == 1)
64 return generic::Memcmp<uint8_t>::block(p1, p2);
65 if (count == 2)
66 return generic::Memcmp<uint16_t>::block(p1, p2);
67 if (count == 3)
68 return generic::MemcmpSequence<uint16_t, uint8_t>::block(p1, p2);
69 if (count == 4)
70 return generic::Memcmp<uint32_t>::block(p1, p2);
71 if (count == 5)
72 return generic::MemcmpSequence<uint32_t, uint8_t>::block(p1, p2);
73 if (count == 6)
74 return generic::MemcmpSequence<uint32_t, uint16_t>::block(p1, p2);
75 if (count == 7)
76 return generic::Memcmp<uint32_t>::head_tail(p1, p2, 7);
77 if (count == 8)
78 return generic::Memcmp<uint64_t>::block(p1, p2);
79 if (count <= 16)
80 return generic::Memcmp<uint64_t>::head_tail(p1, p2, count);
81 #if defined(__AVX512BW__)
82 return inline_memcmp_x86_avx512bw_gt16(p1, p2, count);
83 #elif defined(__AVX2__)
84 return inline_memcmp_x86_avx2_gt16(p1, p2, count);
85 #elif defined(__SSE4_1__)
86 return inline_memcmp_x86_sse41_gt16(p1, p2, count);
87 #else
88 return inline_memcmp_generic_gt16(p1, p2, count);
89 #endif
90 }
91
92 } // namespace LIBC_NAMESPACE_DECL
93
94 #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMCMP_H
95