1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f32-velu/avx-rr2-p6.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2020 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <immintrin.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/common.h>
15*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/vunary.h>
16*4bdc9457SAndroid Build Coastguard Worker
17*4bdc9457SAndroid Build Coastguard Worker
xnn_f32_velu_ukernel__avx_rr2_p6_x24(size_t n,const float * x,float * y,const union xnn_f32_elu_params params[restrict XNN_MIN_ELEMENTS (1)])18*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_velu_ukernel__avx_rr2_p6_x24(
19*4bdc9457SAndroid Build Coastguard Worker size_t n,
20*4bdc9457SAndroid Build Coastguard Worker const float* x,
21*4bdc9457SAndroid Build Coastguard Worker float* y,
22*4bdc9457SAndroid Build Coastguard Worker const union xnn_f32_elu_params params[restrict XNN_MIN_ELEMENTS(1)])
23*4bdc9457SAndroid Build Coastguard Worker {
24*4bdc9457SAndroid Build Coastguard Worker assert(n % sizeof(float) == 0);
25*4bdc9457SAndroid Build Coastguard Worker
26*4bdc9457SAndroid Build Coastguard Worker const __m256 vprescale = _mm256_load_ps(params->avx_rr2_p6.prescale);
27*4bdc9457SAndroid Build Coastguard Worker const __m256 valpha = _mm256_load_ps(params->avx_rr2_p6.alpha);
28*4bdc9457SAndroid Build Coastguard Worker const __m256 vbeta = _mm256_load_ps(params->avx_rr2_p6.beta);
29*4bdc9457SAndroid Build Coastguard Worker const __m256 vsat_cutoff = _mm256_load_ps(params->avx_rr2_p6.sat_cutoff);
30*4bdc9457SAndroid Build Coastguard Worker const __m256 vmagic_bias = _mm256_load_ps(params->avx_rr2_p6.magic_bias);
31*4bdc9457SAndroid Build Coastguard Worker const __m256 vlog2e = _mm256_load_ps(params->avx_rr2_p6.log2e);
32*4bdc9457SAndroid Build Coastguard Worker const __m256 vminus_ln2_hi = _mm256_load_ps(params->avx_rr2_p6.minus_ln2_hi);
33*4bdc9457SAndroid Build Coastguard Worker const __m256 vminus_ln2_lo = _mm256_load_ps(params->avx_rr2_p6.minus_ln2_lo);
34*4bdc9457SAndroid Build Coastguard Worker const __m256 vc6 = _mm256_load_ps(params->avx_rr2_p6.c6);
35*4bdc9457SAndroid Build Coastguard Worker const __m256 vc5 = _mm256_load_ps(params->avx_rr2_p6.c5);
36*4bdc9457SAndroid Build Coastguard Worker const __m256 vc4 = _mm256_load_ps(params->avx_rr2_p6.c4);
37*4bdc9457SAndroid Build Coastguard Worker const __m256 vc3 = _mm256_load_ps(params->avx_rr2_p6.c3);
38*4bdc9457SAndroid Build Coastguard Worker const __m256 vc2 = _mm256_load_ps(params->avx_rr2_p6.c2);
39*4bdc9457SAndroid Build Coastguard Worker const __m256 vone = _mm256_load_ps(params->avx_rr2_p6.one);
40*4bdc9457SAndroid Build Coastguard Worker
41*4bdc9457SAndroid Build Coastguard Worker for (; n >= 24 * sizeof(float); n -= 24 * sizeof(float)) {
42*4bdc9457SAndroid Build Coastguard Worker __m256 vx0 = _mm256_loadu_ps(x);
43*4bdc9457SAndroid Build Coastguard Worker __m256 vx1 = _mm256_loadu_ps(x + 8);
44*4bdc9457SAndroid Build Coastguard Worker __m256 vx2 = _mm256_loadu_ps(x + 16);
45*4bdc9457SAndroid Build Coastguard Worker x += 24;
46*4bdc9457SAndroid Build Coastguard Worker
47*4bdc9457SAndroid Build Coastguard Worker const __m256 vz0 = _mm256_max_ps(vsat_cutoff, _mm256_mul_ps(vx0, vprescale));
48*4bdc9457SAndroid Build Coastguard Worker const __m256 vz1 = _mm256_max_ps(vsat_cutoff, _mm256_mul_ps(vx1, vprescale));
49*4bdc9457SAndroid Build Coastguard Worker const __m256 vz2 = _mm256_max_ps(vsat_cutoff, _mm256_mul_ps(vx2, vprescale));
50*4bdc9457SAndroid Build Coastguard Worker
51*4bdc9457SAndroid Build Coastguard Worker __m256 vn0 = _mm256_add_ps(_mm256_mul_ps(vz0, vlog2e), vmagic_bias);
52*4bdc9457SAndroid Build Coastguard Worker __m256 vn1 = _mm256_add_ps(_mm256_mul_ps(vz1, vlog2e), vmagic_bias);
53*4bdc9457SAndroid Build Coastguard Worker __m256 vn2 = _mm256_add_ps(_mm256_mul_ps(vz2, vlog2e), vmagic_bias);
54*4bdc9457SAndroid Build Coastguard Worker
55*4bdc9457SAndroid Build Coastguard Worker const __m128 vs0_lo = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_castps256_ps128(vn0)), 23));
56*4bdc9457SAndroid Build Coastguard Worker const __m128 vs0_hi = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_extractf128_ps(vn0, 1)), 23));
57*4bdc9457SAndroid Build Coastguard Worker vn0 = _mm256_sub_ps(vn0, vmagic_bias);
58*4bdc9457SAndroid Build Coastguard Worker const __m128 vs1_lo = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_castps256_ps128(vn1)), 23));
59*4bdc9457SAndroid Build Coastguard Worker const __m128 vs1_hi = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_extractf128_ps(vn1, 1)), 23));
60*4bdc9457SAndroid Build Coastguard Worker vn1 = _mm256_sub_ps(vn1, vmagic_bias);
61*4bdc9457SAndroid Build Coastguard Worker const __m128 vs2_lo = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_castps256_ps128(vn2)), 23));
62*4bdc9457SAndroid Build Coastguard Worker const __m128 vs2_hi = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_extractf128_ps(vn2, 1)), 23));
63*4bdc9457SAndroid Build Coastguard Worker vn2 = _mm256_sub_ps(vn2, vmagic_bias);
64*4bdc9457SAndroid Build Coastguard Worker
65*4bdc9457SAndroid Build Coastguard Worker __m256 vt0 = _mm256_add_ps(_mm256_mul_ps(vn0, vminus_ln2_hi), vz0);
66*4bdc9457SAndroid Build Coastguard Worker __m256 vs0 = _mm256_insertf128_ps(_mm256_castps128_ps256(vs0_lo), vs0_hi, 1);
67*4bdc9457SAndroid Build Coastguard Worker __m256 vt1 = _mm256_add_ps(_mm256_mul_ps(vn1, vminus_ln2_hi), vz1);
68*4bdc9457SAndroid Build Coastguard Worker __m256 vs1 = _mm256_insertf128_ps(_mm256_castps128_ps256(vs1_lo), vs1_hi, 1);
69*4bdc9457SAndroid Build Coastguard Worker __m256 vt2 = _mm256_add_ps(_mm256_mul_ps(vn2, vminus_ln2_hi), vz2);
70*4bdc9457SAndroid Build Coastguard Worker __m256 vs2 = _mm256_insertf128_ps(_mm256_castps128_ps256(vs2_lo), vs2_hi, 1);
71*4bdc9457SAndroid Build Coastguard Worker
72*4bdc9457SAndroid Build Coastguard Worker vt0 = _mm256_add_ps(_mm256_mul_ps(vn0, vminus_ln2_lo), vt0);
73*4bdc9457SAndroid Build Coastguard Worker vt1 = _mm256_add_ps(_mm256_mul_ps(vn1, vminus_ln2_lo), vt1);
74*4bdc9457SAndroid Build Coastguard Worker vt2 = _mm256_add_ps(_mm256_mul_ps(vn2, vminus_ln2_lo), vt2);
75*4bdc9457SAndroid Build Coastguard Worker
76*4bdc9457SAndroid Build Coastguard Worker __m256 vp0 = _mm256_add_ps(_mm256_mul_ps(vc6, vt0), vc5);
77*4bdc9457SAndroid Build Coastguard Worker __m256 vp1 = _mm256_add_ps(_mm256_mul_ps(vc6, vt1), vc5);
78*4bdc9457SAndroid Build Coastguard Worker __m256 vp2 = _mm256_add_ps(_mm256_mul_ps(vc6, vt2), vc5);
79*4bdc9457SAndroid Build Coastguard Worker
80*4bdc9457SAndroid Build Coastguard Worker vp0 = _mm256_add_ps(_mm256_mul_ps(vp0, vt0), vc4);
81*4bdc9457SAndroid Build Coastguard Worker vp1 = _mm256_add_ps(_mm256_mul_ps(vp1, vt1), vc4);
82*4bdc9457SAndroid Build Coastguard Worker vp2 = _mm256_add_ps(_mm256_mul_ps(vp2, vt2), vc4);
83*4bdc9457SAndroid Build Coastguard Worker
84*4bdc9457SAndroid Build Coastguard Worker vp0 = _mm256_add_ps(_mm256_mul_ps(vp0, vt0), vc3);
85*4bdc9457SAndroid Build Coastguard Worker vp1 = _mm256_add_ps(_mm256_mul_ps(vp1, vt1), vc3);
86*4bdc9457SAndroid Build Coastguard Worker vp2 = _mm256_add_ps(_mm256_mul_ps(vp2, vt2), vc3);
87*4bdc9457SAndroid Build Coastguard Worker
88*4bdc9457SAndroid Build Coastguard Worker vp0 = _mm256_add_ps(_mm256_mul_ps(vp0, vt0), vc2);
89*4bdc9457SAndroid Build Coastguard Worker vp1 = _mm256_add_ps(_mm256_mul_ps(vp1, vt1), vc2);
90*4bdc9457SAndroid Build Coastguard Worker vp2 = _mm256_add_ps(_mm256_mul_ps(vp2, vt2), vc2);
91*4bdc9457SAndroid Build Coastguard Worker
92*4bdc9457SAndroid Build Coastguard Worker vp0 = _mm256_mul_ps(vp0, vt0);
93*4bdc9457SAndroid Build Coastguard Worker vp1 = _mm256_mul_ps(vp1, vt1);
94*4bdc9457SAndroid Build Coastguard Worker vp2 = _mm256_mul_ps(vp2, vt2);
95*4bdc9457SAndroid Build Coastguard Worker
96*4bdc9457SAndroid Build Coastguard Worker vt0 = _mm256_mul_ps(vt0, vs0);
97*4bdc9457SAndroid Build Coastguard Worker vs0 = _mm256_sub_ps(vs0, vone);
98*4bdc9457SAndroid Build Coastguard Worker vt1 = _mm256_mul_ps(vt1, vs1);
99*4bdc9457SAndroid Build Coastguard Worker vs1 = _mm256_sub_ps(vs1, vone);
100*4bdc9457SAndroid Build Coastguard Worker vt2 = _mm256_mul_ps(vt2, vs2);
101*4bdc9457SAndroid Build Coastguard Worker vs2 = _mm256_sub_ps(vs2, vone);
102*4bdc9457SAndroid Build Coastguard Worker
103*4bdc9457SAndroid Build Coastguard Worker vp0 = _mm256_add_ps(_mm256_mul_ps(vp0, vt0), vt0);
104*4bdc9457SAndroid Build Coastguard Worker vp1 = _mm256_add_ps(_mm256_mul_ps(vp1, vt1), vt1);
105*4bdc9457SAndroid Build Coastguard Worker vp2 = _mm256_add_ps(_mm256_mul_ps(vp2, vt2), vt2);
106*4bdc9457SAndroid Build Coastguard Worker
107*4bdc9457SAndroid Build Coastguard Worker const __m256 ve0 = _mm256_mul_ps(_mm256_add_ps(vp0, vs0), valpha);
108*4bdc9457SAndroid Build Coastguard Worker vx0 = _mm256_mul_ps(vx0, vbeta);
109*4bdc9457SAndroid Build Coastguard Worker const __m256 ve1 = _mm256_mul_ps(_mm256_add_ps(vp1, vs1), valpha);
110*4bdc9457SAndroid Build Coastguard Worker vx1 = _mm256_mul_ps(vx1, vbeta);
111*4bdc9457SAndroid Build Coastguard Worker const __m256 ve2 = _mm256_mul_ps(_mm256_add_ps(vp2, vs2), valpha);
112*4bdc9457SAndroid Build Coastguard Worker vx2 = _mm256_mul_ps(vx2, vbeta);
113*4bdc9457SAndroid Build Coastguard Worker
114*4bdc9457SAndroid Build Coastguard Worker const __m256 vy0 = _mm256_blendv_ps(vx0, ve0, vx0);
115*4bdc9457SAndroid Build Coastguard Worker const __m256 vy1 = _mm256_blendv_ps(vx1, ve1, vx1);
116*4bdc9457SAndroid Build Coastguard Worker const __m256 vy2 = _mm256_blendv_ps(vx2, ve2, vx2);
117*4bdc9457SAndroid Build Coastguard Worker
118*4bdc9457SAndroid Build Coastguard Worker _mm256_storeu_ps(y, vy0);
119*4bdc9457SAndroid Build Coastguard Worker _mm256_storeu_ps(y + 8, vy1);
120*4bdc9457SAndroid Build Coastguard Worker _mm256_storeu_ps(y + 16, vy2);
121*4bdc9457SAndroid Build Coastguard Worker y += 24;
122*4bdc9457SAndroid Build Coastguard Worker }
123*4bdc9457SAndroid Build Coastguard Worker for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) {
124*4bdc9457SAndroid Build Coastguard Worker __m256 vx = _mm256_loadu_ps(x);
125*4bdc9457SAndroid Build Coastguard Worker x += 8;
126*4bdc9457SAndroid Build Coastguard Worker
127*4bdc9457SAndroid Build Coastguard Worker const __m256 vz = _mm256_max_ps(vsat_cutoff, _mm256_mul_ps(vx, vprescale));
128*4bdc9457SAndroid Build Coastguard Worker
129*4bdc9457SAndroid Build Coastguard Worker __m256 vn = _mm256_add_ps(_mm256_mul_ps(vz, vlog2e), vmagic_bias);
130*4bdc9457SAndroid Build Coastguard Worker const __m128 vs_lo = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_castps256_ps128(vn)), 23));
131*4bdc9457SAndroid Build Coastguard Worker const __m128 vs_hi = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_extractf128_ps(vn, 1)), 23));
132*4bdc9457SAndroid Build Coastguard Worker vn = _mm256_sub_ps(vn, vmagic_bias);
133*4bdc9457SAndroid Build Coastguard Worker
134*4bdc9457SAndroid Build Coastguard Worker __m256 vt = _mm256_add_ps(_mm256_mul_ps(vn, vminus_ln2_hi), vz);
135*4bdc9457SAndroid Build Coastguard Worker __m256 vs = _mm256_insertf128_ps(_mm256_castps128_ps256(vs_lo), vs_hi, 1);
136*4bdc9457SAndroid Build Coastguard Worker vt = _mm256_add_ps(_mm256_mul_ps(vn, vminus_ln2_lo), vt);
137*4bdc9457SAndroid Build Coastguard Worker
138*4bdc9457SAndroid Build Coastguard Worker __m256 vp = _mm256_add_ps(_mm256_mul_ps(vc6, vt), vc5);
139*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc4);
140*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc3);
141*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc2);
142*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_mul_ps(vp, vt);
143*4bdc9457SAndroid Build Coastguard Worker
144*4bdc9457SAndroid Build Coastguard Worker vt = _mm256_mul_ps(vt, vs);
145*4bdc9457SAndroid Build Coastguard Worker vs = _mm256_sub_ps(vs, vone);
146*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vt);
147*4bdc9457SAndroid Build Coastguard Worker
148*4bdc9457SAndroid Build Coastguard Worker const __m256 ve = _mm256_mul_ps(_mm256_add_ps(vp, vs), valpha);
149*4bdc9457SAndroid Build Coastguard Worker vx = _mm256_mul_ps(vx, vbeta);
150*4bdc9457SAndroid Build Coastguard Worker const __m256 vy = _mm256_blendv_ps(vx, ve, vx);
151*4bdc9457SAndroid Build Coastguard Worker
152*4bdc9457SAndroid Build Coastguard Worker _mm256_storeu_ps(y, vy);
153*4bdc9457SAndroid Build Coastguard Worker y += 8;
154*4bdc9457SAndroid Build Coastguard Worker }
155*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
156*4bdc9457SAndroid Build Coastguard Worker assert(n >= 1 * sizeof(float));
157*4bdc9457SAndroid Build Coastguard Worker assert(n <= 7 * sizeof(float));
158*4bdc9457SAndroid Build Coastguard Worker const __m256i vmask = _mm256_loadu_si256((const __m256i*) ((uintptr_t) ¶ms->avx_rr2_p6.mask_table[7] - n));
159*4bdc9457SAndroid Build Coastguard Worker
160*4bdc9457SAndroid Build Coastguard Worker __m256 vx = _mm256_maskload_ps(x, vmask);
161*4bdc9457SAndroid Build Coastguard Worker
162*4bdc9457SAndroid Build Coastguard Worker const __m256 vz = _mm256_max_ps(vsat_cutoff, _mm256_mul_ps(vx, vprescale));
163*4bdc9457SAndroid Build Coastguard Worker
164*4bdc9457SAndroid Build Coastguard Worker __m256 vn = _mm256_add_ps(_mm256_mul_ps(vz, vlog2e), vmagic_bias);
165*4bdc9457SAndroid Build Coastguard Worker const __m128 vs_lo = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_castps256_ps128(vn)), 23));
166*4bdc9457SAndroid Build Coastguard Worker const __m128 vs_hi = _mm_castsi128_ps(_mm_slli_epi32(_mm_castps_si128(_mm256_extractf128_ps(vn, 1)), 23));
167*4bdc9457SAndroid Build Coastguard Worker vn = _mm256_sub_ps(vn, vmagic_bias);
168*4bdc9457SAndroid Build Coastguard Worker
169*4bdc9457SAndroid Build Coastguard Worker __m256 vt = _mm256_add_ps(_mm256_mul_ps(vn, vminus_ln2_hi), vz);
170*4bdc9457SAndroid Build Coastguard Worker __m256 vs = _mm256_insertf128_ps(_mm256_castps128_ps256(vs_lo), vs_hi, 1);
171*4bdc9457SAndroid Build Coastguard Worker vt = _mm256_add_ps(_mm256_mul_ps(vn, vminus_ln2_lo), vt);
172*4bdc9457SAndroid Build Coastguard Worker
173*4bdc9457SAndroid Build Coastguard Worker __m256 vp = _mm256_add_ps(_mm256_mul_ps(vc6, vt), vc5);
174*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc4);
175*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc3);
176*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vc2);
177*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_mul_ps(vp, vt);
178*4bdc9457SAndroid Build Coastguard Worker
179*4bdc9457SAndroid Build Coastguard Worker vt = _mm256_mul_ps(vt, vs);
180*4bdc9457SAndroid Build Coastguard Worker vs = _mm256_sub_ps(vs, vone);
181*4bdc9457SAndroid Build Coastguard Worker vp = _mm256_add_ps(_mm256_mul_ps(vp, vt), vt);
182*4bdc9457SAndroid Build Coastguard Worker
183*4bdc9457SAndroid Build Coastguard Worker const __m256 ve = _mm256_mul_ps(_mm256_add_ps(vp, vs), valpha);
184*4bdc9457SAndroid Build Coastguard Worker vx = _mm256_mul_ps(vx, vbeta);
185*4bdc9457SAndroid Build Coastguard Worker const __m256 vy = _mm256_blendv_ps(vx, ve, vx);
186*4bdc9457SAndroid Build Coastguard Worker
187*4bdc9457SAndroid Build Coastguard Worker __m128 vy_lo = _mm256_castps256_ps128(vy);
188*4bdc9457SAndroid Build Coastguard Worker if (n & (4 * sizeof(float))) {
189*4bdc9457SAndroid Build Coastguard Worker _mm_storeu_ps(y, vy_lo);
190*4bdc9457SAndroid Build Coastguard Worker vy_lo = _mm256_extractf128_ps(vy, 1);
191*4bdc9457SAndroid Build Coastguard Worker y += 4;
192*4bdc9457SAndroid Build Coastguard Worker }
193*4bdc9457SAndroid Build Coastguard Worker if (n & (2 * sizeof(float))) {
194*4bdc9457SAndroid Build Coastguard Worker _mm_storel_pi((__m64*) y, vy_lo);
195*4bdc9457SAndroid Build Coastguard Worker vy_lo = _mm_movehl_ps(vy_lo, vy_lo);
196*4bdc9457SAndroid Build Coastguard Worker y += 2;
197*4bdc9457SAndroid Build Coastguard Worker }
198*4bdc9457SAndroid Build Coastguard Worker if (n & (1 * sizeof(float))) {
199*4bdc9457SAndroid Build Coastguard Worker _mm_store_ss(y, vy_lo);
200*4bdc9457SAndroid Build Coastguard Worker }
201*4bdc9457SAndroid Build Coastguard Worker }
202*4bdc9457SAndroid Build Coastguard Worker }
203