1 // Auto-generated file. Do not edit!
2 // Template: src/f32-vsigmoid/wasmsimd-rr2-lut64-p2-div.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include <xnnpack/common.h>
15 #include <xnnpack/vunary.h>
16
17
18 extern XNN_INTERNAL const float xnn_table_exp2minus_k_over_64[64];
19
xnn_f32_vsigmoid_ukernel__wasmsimd_rr2_lut64_p2_div_x16(size_t n,const float * x,float * y,const union xnn_f32_sigmoid_params params[restrict XNN_MIN_ELEMENTS (1)])20 void xnn_f32_vsigmoid_ukernel__wasmsimd_rr2_lut64_p2_div_x16(
21 size_t n,
22 const float* x,
23 float* y,
24 const union xnn_f32_sigmoid_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
25 {
26 assert(n % sizeof(float) == 0);
27
28 const v128_t vmagic_bias = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.magic_bias);
29 const v128_t vminus_log2e = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.minus_log2e);
30 const v128_t vindex_mask = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.index_mask);
31 const v128_t vln2_hi = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.ln2_hi);
32 const v128_t vln2_lo = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.ln2_lo);
33 const v128_t vc2 = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.c2);
34 const v128_t vone = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.one);
35 const v128_t vdenorm_cutoff = wasm_v128_load64_splat(params->wasmsimd_rr2_lut64_p2.denorm_cutoff);
36
37 for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
38 const v128_t vx0123 = wasm_v128_load(x);
39 const v128_t vx4567 = wasm_v128_load(x + 4);
40 const v128_t vx89AB = wasm_v128_load(x + 8);
41 const v128_t vxCDEF = wasm_v128_load(x + 12);
42 x += 16;
43
44 const v128_t vz0123 = wasm_f32x4_abs(vx0123);
45 const v128_t vz4567 = wasm_f32x4_abs(vx4567);
46 const v128_t vz89AB = wasm_f32x4_abs(vx89AB);
47 const v128_t vzCDEF = wasm_f32x4_abs(vxCDEF);
48
49 v128_t vn0123 = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vz0123, vminus_log2e));
50 v128_t vn4567 = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vz4567, vminus_log2e));
51 v128_t vn89AB = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vz89AB, vminus_log2e));
52 v128_t vnCDEF = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vzCDEF, vminus_log2e));
53
54 const v128_t ve0123 = wasm_i32x4_shl(vn0123, 17);
55 const v128_t ve4567 = wasm_i32x4_shl(vn4567, 17);
56 const v128_t ve89AB = wasm_i32x4_shl(vn89AB, 17);
57 const v128_t veCDEF = wasm_i32x4_shl(vnCDEF, 17);
58
59 const v128_t vidx0123 = wasm_i32x4_shl(wasm_v128_and(vn0123, vindex_mask), 2);
60 const v128_t vidx4567 = wasm_i32x4_shl(wasm_v128_and(vn4567, vindex_mask), 2);
61 const v128_t vidx89AB = wasm_i32x4_shl(wasm_v128_and(vn89AB, vindex_mask), 2);
62 const v128_t vidxCDEF = wasm_i32x4_shl(wasm_v128_and(vnCDEF, vindex_mask), 2);
63
64 const uint64_t vidx01 = wasm_i64x2_extract_lane(vidx0123, 0);
65 const uint64_t vidx23 = wasm_i64x2_extract_lane(vidx0123, 1);
66 const float vl0 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx01));
67 const float vl1 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx01 >> 32)));
68 const float vl2 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx23));
69 const float vl3 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx23 >> 32)));
70 const v128_t vl0123 = wasm_f32x4_make(vl0, vl1, vl2, vl3);
71 const uint64_t vidx45 = wasm_i64x2_extract_lane(vidx4567, 0);
72 const uint64_t vidx67 = wasm_i64x2_extract_lane(vidx4567, 1);
73 const float vl4 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx45));
74 const float vl5 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx45 >> 32)));
75 const float vl6 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx67));
76 const float vl7 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx67 >> 32)));
77 const v128_t vl4567 = wasm_f32x4_make(vl4, vl5, vl6, vl7);
78 const uint64_t vidx89 = wasm_i64x2_extract_lane(vidx89AB, 0);
79 const uint64_t vidxAB = wasm_i64x2_extract_lane(vidx89AB, 1);
80 const float vl8 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx89));
81 const float vl9 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx89 >> 32)));
82 const float vlA = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidxAB));
83 const float vlB = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidxAB >> 32)));
84 const v128_t vl89AB = wasm_f32x4_make(vl8, vl9, vlA, vlB);
85 const uint64_t vidxCD = wasm_i64x2_extract_lane(vidxCDEF, 0);
86 const uint64_t vidxEF = wasm_i64x2_extract_lane(vidxCDEF, 1);
87 const float vlC = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidxCD));
88 const float vlD = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidxCD >> 32)));
89 const float vlE = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidxEF));
90 const float vlF = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidxEF >> 32)));
91 const v128_t vlCDEF = wasm_f32x4_make(vlC, vlD, vlE, vlF);
92
93 const v128_t vs0123 = wasm_i32x4_add(vl0123, ve0123);
94 const v128_t vs4567 = wasm_i32x4_add(vl4567, ve4567);
95 const v128_t vs89AB = wasm_i32x4_add(vl89AB, ve89AB);
96 const v128_t vsCDEF = wasm_i32x4_add(vlCDEF, veCDEF);
97
98 vn0123 = wasm_f32x4_sub(vn0123, vmagic_bias);
99 vn4567 = wasm_f32x4_sub(vn4567, vmagic_bias);
100 vn89AB = wasm_f32x4_sub(vn89AB, vmagic_bias);
101 vnCDEF = wasm_f32x4_sub(vnCDEF, vmagic_bias);
102
103 v128_t vt0123 = wasm_f32x4_add(vz0123, wasm_f32x4_mul(vn0123, vln2_hi));
104 v128_t vt4567 = wasm_f32x4_add(vz4567, wasm_f32x4_mul(vn4567, vln2_hi));
105 v128_t vt89AB = wasm_f32x4_add(vz89AB, wasm_f32x4_mul(vn89AB, vln2_hi));
106 v128_t vtCDEF = wasm_f32x4_add(vzCDEF, wasm_f32x4_mul(vnCDEF, vln2_hi));
107
108 vt0123 = wasm_f32x4_add(vt0123, wasm_f32x4_mul(vn0123, vln2_lo));
109 vt4567 = wasm_f32x4_add(vt4567, wasm_f32x4_mul(vn4567, vln2_lo));
110 vt89AB = wasm_f32x4_add(vt89AB, wasm_f32x4_mul(vn89AB, vln2_lo));
111 vtCDEF = wasm_f32x4_add(vtCDEF, wasm_f32x4_mul(vnCDEF, vln2_lo));
112
113 v128_t vp0123 = wasm_f32x4_mul(vt0123, vc2);
114 v128_t vp4567 = wasm_f32x4_mul(vt4567, vc2);
115 v128_t vp89AB = wasm_f32x4_mul(vt89AB, vc2);
116 v128_t vpCDEF = wasm_f32x4_mul(vtCDEF, vc2);
117
118 vp0123 = wasm_f32x4_sub(vt0123, wasm_f32x4_mul(vp0123, vt0123));
119 vp4567 = wasm_f32x4_sub(vt4567, wasm_f32x4_mul(vp4567, vt4567));
120 vp89AB = wasm_f32x4_sub(vt89AB, wasm_f32x4_mul(vp89AB, vt89AB));
121 vpCDEF = wasm_f32x4_sub(vtCDEF, wasm_f32x4_mul(vpCDEF, vtCDEF));
122
123 const v128_t vy0123 = wasm_f32x4_sub(vs0123, wasm_f32x4_mul(vs0123, vp0123));
124 const v128_t vy4567 = wasm_f32x4_sub(vs4567, wasm_f32x4_mul(vs4567, vp4567));
125 const v128_t vy89AB = wasm_f32x4_sub(vs89AB, wasm_f32x4_mul(vs89AB, vp89AB));
126 const v128_t vyCDEF = wasm_f32x4_sub(vsCDEF, wasm_f32x4_mul(vsCDEF, vpCDEF));
127
128 const v128_t vd0123 = wasm_f32x4_add(vy0123, vone);
129 const v128_t vd4567 = wasm_f32x4_add(vy4567, vone);
130 const v128_t vd89AB = wasm_f32x4_add(vy89AB, vone);
131 const v128_t vdCDEF = wasm_f32x4_add(vyCDEF, vone);
132
133 v128_t vf0123 = wasm_f32x4_div(vy0123, vd0123);
134 v128_t vf4567 = wasm_f32x4_div(vy4567, vd4567);
135 v128_t vf89AB = wasm_f32x4_div(vy89AB, vd89AB);
136 v128_t vfCDEF = wasm_f32x4_div(vyCDEF, vdCDEF);
137
138 vf0123 = wasm_v128_andnot(vf0123, wasm_f32x4_gt(vz0123, vdenorm_cutoff));
139 vf4567 = wasm_v128_andnot(vf4567, wasm_f32x4_gt(vz4567, vdenorm_cutoff));
140 vf89AB = wasm_v128_andnot(vf89AB, wasm_f32x4_gt(vz89AB, vdenorm_cutoff));
141 vfCDEF = wasm_v128_andnot(vfCDEF, wasm_f32x4_gt(vzCDEF, vdenorm_cutoff));
142
143 vf0123 = wasm_v128_bitselect(vf0123, wasm_f32x4_sub(vone, vf0123), wasm_i32x4_shr(vx0123, 31));
144 vf4567 = wasm_v128_bitselect(vf4567, wasm_f32x4_sub(vone, vf4567), wasm_i32x4_shr(vx4567, 31));
145 vf89AB = wasm_v128_bitselect(vf89AB, wasm_f32x4_sub(vone, vf89AB), wasm_i32x4_shr(vx89AB, 31));
146 vfCDEF = wasm_v128_bitselect(vfCDEF, wasm_f32x4_sub(vone, vfCDEF), wasm_i32x4_shr(vxCDEF, 31));
147
148 wasm_v128_store(y, vf0123);
149 wasm_v128_store(y + 4, vf4567);
150 wasm_v128_store(y + 8, vf89AB);
151 wasm_v128_store(y + 12, vfCDEF);
152 y += 16;
153 }
154 for (; n >= 4 * sizeof(float); n -= 4 * sizeof(float)) {
155 const v128_t vx = wasm_v128_load(x);
156 x += 4;
157
158 const v128_t vz = wasm_f32x4_abs(vx);
159
160 v128_t vn = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vz, vminus_log2e));
161 const v128_t ve = wasm_i32x4_shl(vn, 17);
162
163 const v128_t vidx = wasm_i32x4_shl(wasm_v128_and(vn, vindex_mask), 2);
164 const uint64_t vidx_lo = wasm_i64x2_extract_lane(vidx, 0);
165 const uint64_t vidx_hi = wasm_i64x2_extract_lane(vidx, 1);
166 const float vl0 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx_lo));
167 const float vl1 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx_lo >> 32)));
168 const float vl2 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx_hi));
169 const float vl3 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx_hi >> 32)));
170 const v128_t vl = wasm_f32x4_make(vl0, vl1, vl2, vl3);
171
172 const v128_t vs = wasm_i32x4_add(vl, ve);
173 vn = wasm_f32x4_sub(vn, vmagic_bias);
174
175 v128_t vt = wasm_f32x4_add(vz, wasm_f32x4_mul(vn, vln2_hi));
176 vt = wasm_f32x4_add(vt, wasm_f32x4_mul(vn, vln2_lo));
177
178 v128_t vp = wasm_f32x4_mul(vt, vc2);
179 vp = wasm_f32x4_sub(vt, wasm_f32x4_mul(vp, vt));
180
181 const v128_t vy = wasm_f32x4_sub(vs, wasm_f32x4_mul(vs, vp));
182 const v128_t vd = wasm_f32x4_add(vy, vone);
183
184 v128_t vf = wasm_f32x4_div(vy, vd);
185 vf = wasm_v128_andnot(vf, wasm_f32x4_gt(vz, vdenorm_cutoff));
186 vf = wasm_v128_bitselect(vf, wasm_f32x4_sub(vone, vf), wasm_i32x4_shr(vx, 31));
187
188 wasm_v128_store(y, vf);
189 y += 4;
190 }
191 if XNN_UNLIKELY(n != 0) {
192 const v128_t vx = wasm_v128_load(x);
193
194 const v128_t vz = wasm_f32x4_abs(vx);
195
196 v128_t vn = wasm_f32x4_add(vmagic_bias, wasm_f32x4_mul(vz, vminus_log2e));
197 const v128_t ve = wasm_i32x4_shl(vn, 17);
198
199 const v128_t vidx = wasm_i32x4_shl(wasm_v128_and(vn, vindex_mask), 2);
200 const uint64_t vidx_lo = wasm_i64x2_extract_lane(vidx, 0);
201 const uint64_t vidx_hi = wasm_i64x2_extract_lane(vidx, 1);
202 const float vl0 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx_lo));
203 const float vl1 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx_lo >> 32)));
204 const float vl2 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) vidx_hi));
205 const float vl3 = *((const float*) ((uintptr_t) xnn_table_exp2minus_k_over_64 + (uint32_t) (vidx_hi >> 32)));
206 const v128_t vl = wasm_f32x4_make(vl0, vl1, vl2, vl3);
207
208 const v128_t vs = wasm_i32x4_add(vl, ve);
209 vn = wasm_f32x4_sub(vn, vmagic_bias);
210
211 v128_t vt = wasm_f32x4_add(vz, wasm_f32x4_mul(vn, vln2_hi));
212 vt = wasm_f32x4_add(vt, wasm_f32x4_mul(vn, vln2_lo));
213
214 v128_t vp = wasm_f32x4_mul(vt, vc2);
215 vp = wasm_f32x4_sub(vt, wasm_f32x4_mul(vp, vt));
216
217 const v128_t vy = wasm_f32x4_sub(vs, wasm_f32x4_mul(vs, vp));
218 const v128_t vd = wasm_f32x4_add(vy, vone);
219
220 v128_t vf = wasm_f32x4_div(vy, vd);
221 vf = wasm_v128_andnot(vf, wasm_f32x4_gt(vz, vdenorm_cutoff));
222 vf = wasm_v128_bitselect(vf, wasm_f32x4_sub(vone, vf), wasm_i32x4_shr(vx, 31));
223
224 if (n & (2 * sizeof(float))) {
225 *((double*) y) = wasm_f64x2_extract_lane(vf, 0);
226 vf = wasm_v32x4_shuffle(vf, vf, 2, 3, 2, 3);
227 y += 2;
228 }
229 if (n & (1 * sizeof(float))) {
230 *y = wasm_f32x4_extract_lane(vf, 0);
231 }
232 }
233 }
234