1 // Auto-generated file. Do not edit!
2 // Template: src/f32-igemm/wasmsimd-s4.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include <xnnpack/igemm.h>
15
16
xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_x86(size_t mr,size_t nc,size_t kc,size_t ks,const float ** restrict a,const float * restrict w,float * restrict c,size_t cm_stride,size_t cn_stride,size_t a_offset,const float * zero,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_x86(
18 size_t mr,
19 size_t nc,
20 size_t kc,
21 size_t ks,
22 const float**restrict a,
23 const float*restrict w,
24 float*restrict c,
25 size_t cm_stride,
26 size_t cn_stride,
27 size_t a_offset,
28 const float* zero,
29 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
30 {
31 assert(mr != 0);
32 assert(mr <= 6);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(kc % sizeof(float) == 0);
36 assert(ks != 0);
37 assert(ks % (6 * sizeof(void*)) == 0);
38 assert(a_offset % sizeof(float) == 0);
39 assert(a != NULL);
40 assert(w != NULL);
41 assert(c != NULL);
42
43 float* c0 = c;
44 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
45 if XNN_UNPREDICTABLE(mr < 2) {
46 c1 = c0;
47 }
48 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
49 if XNN_UNPREDICTABLE(mr <= 2) {
50 c2 = c1;
51 }
52 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
53 if XNN_UNPREDICTABLE(mr < 4) {
54 c3 = c2;
55 }
56 float* c4 = (float*) ((uintptr_t) c3 + cm_stride);
57 if XNN_UNPREDICTABLE(mr <= 4) {
58 c4 = c3;
59 }
60 float* c5 = (float*) ((uintptr_t) c4 + cm_stride);
61 if XNN_UNPREDICTABLE(mr != 6) {
62 c5 = c4;
63 }
64
65 const v128_t vmin = wasm_v128_load64_splat(params->wasmsimd.min);
66 const v128_t vmax = wasm_v128_load64_splat(params->wasmsimd.max);
67 do {
68 v128_t vacc0x0123 = wasm_v128_load(w);
69 v128_t vacc0x4567 = wasm_v128_load(w + 4);
70 v128_t vacc1x0123 = vacc0x0123;
71 v128_t vacc1x4567 = vacc0x4567;
72 v128_t vacc2x0123 = vacc0x0123;
73 v128_t vacc2x4567 = vacc0x4567;
74 v128_t vacc3x0123 = vacc0x0123;
75 v128_t vacc3x4567 = vacc0x4567;
76 v128_t vacc4x0123 = vacc0x0123;
77 v128_t vacc4x4567 = vacc0x4567;
78 v128_t vacc5x0123 = vacc0x0123;
79 v128_t vacc5x4567 = vacc0x4567;
80 w += 8;
81
82 size_t p = ks;
83 do {
84 const float* restrict a0 = a[0];
85 assert(a0 != NULL);
86 if XNN_UNPREDICTABLE(a0 != zero) {
87 a0 = (const float*) ((uintptr_t) a0 + a_offset);
88 }
89 const float* restrict a1 = a[1];
90 assert(a1 != NULL);
91 if XNN_UNPREDICTABLE(a1 != zero) {
92 a1 = (const float*) ((uintptr_t) a1 + a_offset);
93 }
94 const float* restrict a2 = a[2];
95 assert(a2 != NULL);
96 if XNN_UNPREDICTABLE(a2 != zero) {
97 a2 = (const float*) ((uintptr_t) a2 + a_offset);
98 }
99 const float* restrict a3 = a[3];
100 assert(a3 != NULL);
101 if XNN_UNPREDICTABLE(a3 != zero) {
102 a3 = (const float*) ((uintptr_t) a3 + a_offset);
103 }
104 const float* restrict a4 = a[4];
105 assert(a4 != NULL);
106 if XNN_UNPREDICTABLE(a4 != zero) {
107 a4 = (const float*) ((uintptr_t) a4 + a_offset);
108 }
109 const float* restrict a5 = a[5];
110 assert(a5 != NULL);
111 if XNN_UNPREDICTABLE(a5 != zero) {
112 a5 = (const float*) ((uintptr_t) a5 + a_offset);
113 }
114 a += 6;
115
116 size_t k = kc;
117 while (k >= 4 * sizeof(float)) {
118 v128_t va0 = wasm_v128_load(a0);
119 a0 += 4;
120 v128_t va1 = wasm_v128_load(a1);
121 a1 += 4;
122 v128_t va2 = wasm_v128_load(a2);
123 a2 += 4;
124 v128_t va3 = wasm_v128_load(a3);
125 a3 += 4;
126 v128_t va4 = wasm_v128_load(a4);
127 a4 += 4;
128 v128_t va5 = wasm_v128_load(a5);
129 a5 += 4;
130
131
132 const v128_t vb0123c0 = wasm_v128_load(w + 0);
133 const v128_t vb4567c0 = wasm_v128_load(w + 4);
134
135 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123c0));
136 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123c0));
137 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123c0));
138 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3, vb0123c0));
139 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4, vb0123c0));
140 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5, vb0123c0));
141 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567c0));
142 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567c0));
143 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567c0));
144 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3, vb4567c0));
145 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4, vb4567c0));
146 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5, vb4567c0));
147
148 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
149 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
150 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
151 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
152 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
153 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
154
155 const v128_t vb0123c1 = wasm_v128_load(w + 8);
156 const v128_t vb4567c1 = wasm_v128_load(w + 12);
157
158 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123c1));
159 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123c1));
160 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123c1));
161 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3, vb0123c1));
162 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4, vb0123c1));
163 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5, vb0123c1));
164 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567c1));
165 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567c1));
166 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567c1));
167 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3, vb4567c1));
168 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4, vb4567c1));
169 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5, vb4567c1));
170
171 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
172 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
173 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
174 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
175 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
176 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
177
178 const v128_t vb0123c2 = wasm_v128_load(w + 16);
179 const v128_t vb4567c2 = wasm_v128_load(w + 20);
180
181 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123c2));
182 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123c2));
183 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123c2));
184 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3, vb0123c2));
185 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4, vb0123c2));
186 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5, vb0123c2));
187 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567c2));
188 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567c2));
189 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567c2));
190 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3, vb4567c2));
191 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4, vb4567c2));
192 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5, vb4567c2));
193
194 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
195 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
196 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
197 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
198 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
199 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
200
201 const v128_t vb0123c3 = wasm_v128_load(w + 24);
202 const v128_t vb4567c3 = wasm_v128_load(w + 28);
203
204 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123c3));
205 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123c3));
206 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123c3));
207 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3, vb0123c3));
208 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4, vb0123c3));
209 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5, vb0123c3));
210 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567c3));
211 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567c3));
212 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567c3));
213 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3, vb4567c3));
214 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4, vb4567c3));
215 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5, vb4567c3));
216
217
218 w += 32;
219 k -= 4 * sizeof(float);
220 }
221 if XNN_UNLIKELY(k != 0) {
222 v128_t va0 = wasm_v128_load(a0);
223 a0 = (const float*) ((uintptr_t) a0 + k);
224 v128_t va1 = wasm_v128_load(a1);
225 a1 = (const float*) ((uintptr_t) a1 + k);
226 v128_t va2 = wasm_v128_load(a2);
227 a2 = (const float*) ((uintptr_t) a2 + k);
228 v128_t va3 = wasm_v128_load(a3);
229 a3 = (const float*) ((uintptr_t) a3 + k);
230 v128_t va4 = wasm_v128_load(a4);
231 a4 = (const float*) ((uintptr_t) a4 + k);
232 v128_t va5 = wasm_v128_load(a5);
233 a5 = (const float*) ((uintptr_t) a5 + k);
234
235 const v128_t vzero = wasm_f32x4_const_splat(0.0f);
236
237 const v128_t vb0123c0 = wasm_v128_load(w + 0);
238 const v128_t vb4567c0 = wasm_v128_load(w + 4);
239
240 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
241 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
242 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
243 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
244 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
245 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb0123c0, vzero)), vb0123c0));
246 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
247 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
248 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
249 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
250 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
251 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb4567c0, vzero)), vb4567c0));
252
253 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
254 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
255 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
256 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
257 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
258 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
259
260 const v128_t vb0123c1 = wasm_v128_load(w + 8);
261 const v128_t vb4567c1 = wasm_v128_load(w + 12);
262
263 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
264 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
265 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
266 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
267 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
268 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb0123c1, vzero)), vb0123c1));
269 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
270 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
271 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
272 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
273 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
274 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb4567c1, vzero)), vb4567c1));
275
276 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
277 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
278 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
279 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
280 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
281 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
282
283 const v128_t vb0123c2 = wasm_v128_load(w + 16);
284 const v128_t vb4567c2 = wasm_v128_load(w + 20);
285
286 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
287 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
288 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
289 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
290 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
291 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb0123c2, vzero)), vb0123c2));
292 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
293 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
294 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
295 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
296 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
297 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb4567c2, vzero)), vb4567c2));
298
299 va0 = wasm_v32x4_shuffle(va0, va0, 1, 2, 3, 0);
300 va1 = wasm_v32x4_shuffle(va1, va1, 1, 2, 3, 0);
301 va2 = wasm_v32x4_shuffle(va2, va2, 1, 2, 3, 0);
302 va3 = wasm_v32x4_shuffle(va3, va3, 1, 2, 3, 0);
303 va4 = wasm_v32x4_shuffle(va4, va4, 1, 2, 3, 0);
304 va5 = wasm_v32x4_shuffle(va5, va5, 1, 2, 3, 0);
305
306 const v128_t vb0123c3 = wasm_v128_load(w + 24);
307 const v128_t vb4567c3 = wasm_v128_load(w + 28);
308
309 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
310 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
311 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
312 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
313 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
314 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb0123c3, vzero)), vb0123c3));
315 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(wasm_v128_andnot(va0, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
316 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(wasm_v128_andnot(va1, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
317 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(wasm_v128_andnot(va2, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
318 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(wasm_v128_andnot(va3, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
319 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(wasm_v128_andnot(va4, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
320 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(wasm_v128_andnot(va5, wasm_f32x4_eq(vb4567c3, vzero)), vb4567c3));
321
322
323 w += 32;
324 }
325 p -= 6 * sizeof(void*);
326 } while (p != 0);
327
328 vacc0x0123 = wasm_f32x4_pmax(vmin, vacc0x0123);
329 vacc1x0123 = wasm_f32x4_pmax(vmin, vacc1x0123);
330 vacc2x0123 = wasm_f32x4_pmax(vmin, vacc2x0123);
331 vacc3x0123 = wasm_f32x4_pmax(vmin, vacc3x0123);
332 vacc4x0123 = wasm_f32x4_pmax(vmin, vacc4x0123);
333 vacc5x0123 = wasm_f32x4_pmax(vmin, vacc5x0123);
334 vacc0x4567 = wasm_f32x4_pmax(vmin, vacc0x4567);
335 vacc1x4567 = wasm_f32x4_pmax(vmin, vacc1x4567);
336 vacc2x4567 = wasm_f32x4_pmax(vmin, vacc2x4567);
337 vacc3x4567 = wasm_f32x4_pmax(vmin, vacc3x4567);
338 vacc4x4567 = wasm_f32x4_pmax(vmin, vacc4x4567);
339 vacc5x4567 = wasm_f32x4_pmax(vmin, vacc5x4567);
340
341 vacc0x0123 = wasm_f32x4_pmin(vmax, vacc0x0123);
342 vacc1x0123 = wasm_f32x4_pmin(vmax, vacc1x0123);
343 vacc2x0123 = wasm_f32x4_pmin(vmax, vacc2x0123);
344 vacc3x0123 = wasm_f32x4_pmin(vmax, vacc3x0123);
345 vacc4x0123 = wasm_f32x4_pmin(vmax, vacc4x0123);
346 vacc5x0123 = wasm_f32x4_pmin(vmax, vacc5x0123);
347 vacc0x4567 = wasm_f32x4_pmin(vmax, vacc0x4567);
348 vacc1x4567 = wasm_f32x4_pmin(vmax, vacc1x4567);
349 vacc2x4567 = wasm_f32x4_pmin(vmax, vacc2x4567);
350 vacc3x4567 = wasm_f32x4_pmin(vmax, vacc3x4567);
351 vacc4x4567 = wasm_f32x4_pmin(vmax, vacc4x4567);
352 vacc5x4567 = wasm_f32x4_pmin(vmax, vacc5x4567);
353
354 if XNN_LIKELY(nc >= 8) {
355 wasm_v128_store(c5, vacc5x0123);
356 wasm_v128_store(c5 + 4, vacc5x4567);
357 c5 = (float*) ((uintptr_t) c5 + cn_stride);
358 wasm_v128_store(c4, vacc4x0123);
359 wasm_v128_store(c4 + 4, vacc4x4567);
360 c4 = (float*) ((uintptr_t) c4 + cn_stride);
361 wasm_v128_store(c3, vacc3x0123);
362 wasm_v128_store(c3 + 4, vacc3x4567);
363 c3 = (float*) ((uintptr_t) c3 + cn_stride);
364 wasm_v128_store(c2, vacc2x0123);
365 wasm_v128_store(c2 + 4, vacc2x4567);
366 c2 = (float*) ((uintptr_t) c2 + cn_stride);
367 wasm_v128_store(c1, vacc1x0123);
368 wasm_v128_store(c1 + 4, vacc1x4567);
369 c1 = (float*) ((uintptr_t) c1 + cn_stride);
370 wasm_v128_store(c0, vacc0x0123);
371 wasm_v128_store(c0 + 4, vacc0x4567);
372 c0 = (float*) ((uintptr_t) c0 + cn_stride);
373
374 a = (const float**restrict) ((uintptr_t) a - ks);
375 nc -= 8;
376 } else {
377 if (nc & 4) {
378 wasm_v128_store(c5, vacc5x0123);
379 wasm_v128_store(c4, vacc4x0123);
380 wasm_v128_store(c3, vacc3x0123);
381 wasm_v128_store(c2, vacc2x0123);
382 wasm_v128_store(c1, vacc1x0123);
383 wasm_v128_store(c0, vacc0x0123);
384
385 vacc5x0123 = vacc5x4567;
386 vacc4x0123 = vacc4x4567;
387 vacc3x0123 = vacc3x4567;
388 vacc2x0123 = vacc2x4567;
389 vacc1x0123 = vacc1x4567;
390 vacc0x0123 = vacc0x4567;
391
392 c5 += 4;
393 c4 += 4;
394 c3 += 4;
395 c2 += 4;
396 c1 += 4;
397 c0 += 4;
398 }
399 if (nc & 2) {
400 *((double*) c5) = wasm_f64x2_extract_lane(vacc5x0123, 0);
401 *((double*) c4) = wasm_f64x2_extract_lane(vacc4x0123, 0);
402 *((double*) c3) = wasm_f64x2_extract_lane(vacc3x0123, 0);
403 *((double*) c2) = wasm_f64x2_extract_lane(vacc2x0123, 0);
404 *((double*) c1) = wasm_f64x2_extract_lane(vacc1x0123, 0);
405 *((double*) c0) = wasm_f64x2_extract_lane(vacc0x0123, 0);
406
407 vacc5x0123 = wasm_v32x4_shuffle(vacc5x0123, vacc5x0123, 2, 3, 2, 3);
408 vacc4x0123 = wasm_v32x4_shuffle(vacc4x0123, vacc4x0123, 2, 3, 2, 3);
409 vacc3x0123 = wasm_v32x4_shuffle(vacc3x0123, vacc3x0123, 2, 3, 2, 3);
410 vacc2x0123 = wasm_v32x4_shuffle(vacc2x0123, vacc2x0123, 2, 3, 2, 3);
411 vacc1x0123 = wasm_v32x4_shuffle(vacc1x0123, vacc1x0123, 2, 3, 2, 3);
412 vacc0x0123 = wasm_v32x4_shuffle(vacc0x0123, vacc0x0123, 2, 3, 2, 3);
413
414 c5 += 2;
415 c4 += 2;
416 c3 += 2;
417 c2 += 2;
418 c1 += 2;
419 c0 += 2;
420 }
421 if (nc & 1) {
422 *c5 = wasm_f32x4_extract_lane(vacc5x0123, 0);
423 *c4 = wasm_f32x4_extract_lane(vacc4x0123, 0);
424 *c3 = wasm_f32x4_extract_lane(vacc3x0123, 0);
425 *c2 = wasm_f32x4_extract_lane(vacc2x0123, 0);
426 *c1 = wasm_f32x4_extract_lane(vacc1x0123, 0);
427 *c0 = wasm_f32x4_extract_lane(vacc0x0123, 0);
428 }
429
430 nc = 0;
431 }
432 } while (nc != 0);
433 }
434