1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-vlrelu/ssse3.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <tmmintrin.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/vlrelu.h>
16 #include <xnnpack/unaligned.h>
17
18
xnn_qs8_vlrelu_ukernel__ssse3_x16(size_t n,const int8_t * x,int8_t * y,const union xnn_qs8_lrelu_params params[restrict XNN_MIN_ELEMENTS (1)])19 void xnn_qs8_vlrelu_ukernel__ssse3_x16(
20 size_t n,
21 const int8_t* x,
22 int8_t* y,
23 const union xnn_qs8_lrelu_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
24 {
25 assert(n != 0);
26 assert(n % sizeof(int8_t) == 0);
27 assert(x != NULL);
28 assert(y != NULL);
29
30 const __m128i vinput_zero_point = _mm_load_si128((const __m128i*) params->sse2.input_zero_point);
31 const __m128i vmultiplier_diff = _mm_load_si128((const __m128i*) params->sse2.multiplier_diff);
32 const __m128i vmultiplier_base = _mm_load_si128((const __m128i*) params->sse2.multiplier_base);
33 const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->sse2.output_zero_point);
34 for (; n >= 16 * sizeof(int8_t); n -= 16 * sizeof(int8_t)) {
35 const __m128i vx = _mm_loadu_si128((const __m128i*) x);
36 x += 16;
37
38 const __m128i vm = _mm_cmpgt_epi8(_mm_setzero_si128(), vx);
39 __m128i vacc_lo = _mm_unpacklo_epi8(vx, vm);
40 __m128i vacc_hi = _mm_unpackhi_epi8(vx, vm);
41 __m128i vmultiplier_lo = _mm_cmpgt_epi16(vacc_lo, vinput_zero_point);
42 __m128i vmultiplier_hi = _mm_cmpgt_epi16(vacc_hi, vinput_zero_point);
43 vacc_lo = _mm_sub_epi16(vinput_zero_point, vacc_lo);
44 vacc_hi = _mm_sub_epi16(vinput_zero_point, vacc_hi);
45 vmultiplier_lo = _mm_and_si128(vmultiplier_lo, vmultiplier_diff);
46 vmultiplier_hi = _mm_and_si128(vmultiplier_hi, vmultiplier_diff);
47 vacc_lo = _mm_slli_epi16(vacc_lo, 7);
48 vacc_hi = _mm_slli_epi16(vacc_hi, 7);
49 vmultiplier_lo = _mm_xor_si128(vmultiplier_lo, vmultiplier_base);
50 vmultiplier_hi = _mm_xor_si128(vmultiplier_hi, vmultiplier_base);
51 vacc_lo = _mm_mulhrs_epi16(vacc_lo, vmultiplier_lo);
52 vacc_hi = _mm_mulhrs_epi16(vacc_hi, vmultiplier_hi);
53 vacc_lo = _mm_adds_epi16(vacc_lo, voutput_zero_point);
54 vacc_hi = _mm_adds_epi16(vacc_hi, voutput_zero_point);
55
56 const __m128i vy = _mm_packs_epi16(vacc_lo, vacc_hi);
57 _mm_storeu_si128((__m128i*) y, vy);
58 y += 16;
59 }
60 if XNN_UNLIKELY(n != 0) {
61 assert(n >= 1 * sizeof(int8_t));
62 assert(n <= 15 * sizeof(int8_t));
63
64 const __m128i vx = _mm_loadu_si128((const __m128i*) x);
65
66 const __m128i vm = _mm_cmpgt_epi8(_mm_setzero_si128(), vx);
67 __m128i vacc_lo = _mm_unpacklo_epi8(vx, vm);
68 __m128i vacc_hi = _mm_unpackhi_epi8(vx, vm);
69 __m128i vmultiplier_lo = _mm_cmpgt_epi16(vacc_lo, vinput_zero_point);
70 __m128i vmultiplier_hi = _mm_cmpgt_epi16(vacc_hi, vinput_zero_point);
71 vacc_lo = _mm_sub_epi16(vinput_zero_point, vacc_lo);
72 vacc_hi = _mm_sub_epi16(vinput_zero_point, vacc_hi);
73 vmultiplier_lo = _mm_and_si128(vmultiplier_lo, vmultiplier_diff);
74 vmultiplier_hi = _mm_and_si128(vmultiplier_hi, vmultiplier_diff);
75 vacc_lo = _mm_slli_epi16(vacc_lo, 7);
76 vacc_hi = _mm_slli_epi16(vacc_hi, 7);
77 vmultiplier_lo = _mm_xor_si128(vmultiplier_lo, vmultiplier_base);
78 vmultiplier_hi = _mm_xor_si128(vmultiplier_hi, vmultiplier_base);
79 vacc_lo = _mm_mulhrs_epi16(vacc_lo, vmultiplier_lo);
80 vacc_hi = _mm_mulhrs_epi16(vacc_hi, vmultiplier_hi);
81 vacc_lo = _mm_adds_epi16(vacc_lo, voutput_zero_point);
82 vacc_hi = _mm_adds_epi16(vacc_hi, voutput_zero_point);
83
84 __m128i vy = _mm_packs_epi16(vacc_lo, vacc_hi);
85 if (n & (8 * sizeof(int8_t))) {
86 _mm_storel_epi64((__m128i*) y, vy);
87 vy = _mm_unpackhi_epi64(vy, vy);
88 y += 8;
89 }
90 if (n & (4 * sizeof(int8_t))) {
91 unaligned_store_u32(y, (uint32_t) _mm_cvtsi128_si32(vy));
92 vy = _mm_srli_epi64(vy, 32);
93 y += 4;
94 }
95 uint32_t vy_lo = (uint32_t) _mm_cvtsi128_si32(vy);
96 if (n & (2 * sizeof(int8_t))) {
97 unaligned_store_u16(y, (uint16_t) vy_lo);
98 vy_lo >>= 16;
99 y += 2;
100 }
101 if (n & (1 * sizeof(int8_t))) {
102 *y = (int8_t) vy_lo;
103 }
104 }
105 }
106