1 // Auto-generated file. Do not edit!
2 // Template: src/f32-igemm/wasmsimd-splat.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <wasm_simd128.h>
13
14 #include <xnnpack/igemm.h>
15
16
xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_x86_splat(size_t mr,size_t nc,size_t kc,size_t ks,const float ** restrict a,const float * restrict w,float * restrict c,size_t cm_stride,size_t cn_stride,size_t a_offset,const float * zero,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_x86_splat(
18 size_t mr,
19 size_t nc,
20 size_t kc,
21 size_t ks,
22 const float**restrict a,
23 const float*restrict w,
24 float*restrict c,
25 size_t cm_stride,
26 size_t cn_stride,
27 size_t a_offset,
28 const float* zero,
29 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
30 {
31 assert(mr != 0);
32 assert(mr <= 6);
33 assert(nc != 0);
34 assert(kc != 0);
35 assert(kc % sizeof(float) == 0);
36 assert(ks != 0);
37 assert(ks % (6 * sizeof(void*)) == 0);
38 assert(a_offset % sizeof(float) == 0);
39 assert(a != NULL);
40 assert(w != NULL);
41 assert(c != NULL);
42
43 float* c0 = c;
44 float* c1 = (float*) ((uintptr_t) c0 + cm_stride);
45 if XNN_UNPREDICTABLE(mr < 2) {
46 c1 = c0;
47 }
48 float* c2 = (float*) ((uintptr_t) c1 + cm_stride);
49 if XNN_UNPREDICTABLE(mr <= 2) {
50 c2 = c1;
51 }
52 float* c3 = (float*) ((uintptr_t) c2 + cm_stride);
53 if XNN_UNPREDICTABLE(mr < 4) {
54 c3 = c2;
55 }
56 float* c4 = (float*) ((uintptr_t) c3 + cm_stride);
57 if XNN_UNPREDICTABLE(mr <= 4) {
58 c4 = c3;
59 }
60 float* c5 = (float*) ((uintptr_t) c4 + cm_stride);
61 if XNN_UNPREDICTABLE(mr != 6) {
62 c5 = c4;
63 }
64
65 const v128_t vmin = wasm_v128_load64_splat(params->wasmsimd.min);
66 const v128_t vmax = wasm_v128_load64_splat(params->wasmsimd.max);
67 do {
68 v128_t vacc0x0123 = wasm_v128_load(w);
69 v128_t vacc0x4567 = wasm_v128_load(w + 4);
70 v128_t vacc1x0123 = vacc0x0123;
71 v128_t vacc1x4567 = vacc0x4567;
72 v128_t vacc2x0123 = vacc0x0123;
73 v128_t vacc2x4567 = vacc0x4567;
74 v128_t vacc3x0123 = vacc0x0123;
75 v128_t vacc3x4567 = vacc0x4567;
76 v128_t vacc4x0123 = vacc0x0123;
77 v128_t vacc4x4567 = vacc0x4567;
78 v128_t vacc5x0123 = vacc0x0123;
79 v128_t vacc5x4567 = vacc0x4567;
80 w += 8;
81
82 size_t p = ks;
83 do {
84 const float* restrict a0 = a[0];
85 assert(a0 != NULL);
86 if XNN_UNPREDICTABLE(a0 != zero) {
87 a0 = (const float*) ((uintptr_t) a0 + a_offset);
88 }
89 const float* restrict a1 = a[1];
90 assert(a1 != NULL);
91 if XNN_UNPREDICTABLE(a1 != zero) {
92 a1 = (const float*) ((uintptr_t) a1 + a_offset);
93 }
94 const float* restrict a2 = a[2];
95 assert(a2 != NULL);
96 if XNN_UNPREDICTABLE(a2 != zero) {
97 a2 = (const float*) ((uintptr_t) a2 + a_offset);
98 }
99 const float* restrict a3 = a[3];
100 assert(a3 != NULL);
101 if XNN_UNPREDICTABLE(a3 != zero) {
102 a3 = (const float*) ((uintptr_t) a3 + a_offset);
103 }
104 const float* restrict a4 = a[4];
105 assert(a4 != NULL);
106 if XNN_UNPREDICTABLE(a4 != zero) {
107 a4 = (const float*) ((uintptr_t) a4 + a_offset);
108 }
109 const float* restrict a5 = a[5];
110 assert(a5 != NULL);
111 if XNN_UNPREDICTABLE(a5 != zero) {
112 a5 = (const float*) ((uintptr_t) a5 + a_offset);
113 }
114 a += 6;
115
116 size_t k = kc;
117 while (k >= 4 * sizeof(float)) {
118 const v128_t va0 = wasm_v128_load(a0);
119 a0 += 4;
120 const v128_t va1 = wasm_v128_load(a1);
121 a1 += 4;
122 const v128_t va2 = wasm_v128_load(a2);
123 a2 += 4;
124 const v128_t va3 = wasm_v128_load(a3);
125 a3 += 4;
126 const v128_t va4 = wasm_v128_load(a4);
127 a4 += 4;
128 const v128_t va5 = wasm_v128_load(a5);
129 a5 += 4;
130
131 const v128_t va0c0 = wasm_v32x4_shuffle(va0, va0, 0, 0, 0, 0);
132 const v128_t va1c0 = wasm_v32x4_shuffle(va1, va1, 0, 0, 0, 0);
133 const v128_t va2c0 = wasm_v32x4_shuffle(va2, va2, 0, 0, 0, 0);
134 const v128_t va3c0 = wasm_v32x4_shuffle(va3, va3, 0, 0, 0, 0);
135 const v128_t va4c0 = wasm_v32x4_shuffle(va4, va4, 0, 0, 0, 0);
136 const v128_t va5c0 = wasm_v32x4_shuffle(va5, va5, 0, 0, 0, 0);
137
138 const v128_t vb0123c0 = wasm_v128_load(w + 0);
139 const v128_t vb4567c0 = wasm_v128_load(w + 4);
140
141 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0c0, vb0123c0));
142 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1c0, vb0123c0));
143 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2c0, vb0123c0));
144 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3c0, vb0123c0));
145 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4c0, vb0123c0));
146 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5c0, vb0123c0));
147 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0c0, vb4567c0));
148 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1c0, vb4567c0));
149 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2c0, vb4567c0));
150 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3c0, vb4567c0));
151 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4c0, vb4567c0));
152 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5c0, vb4567c0));
153 const v128_t va0c1 = wasm_v32x4_shuffle(va0, va0, 1, 1, 1, 1);
154 const v128_t va1c1 = wasm_v32x4_shuffle(va1, va1, 1, 1, 1, 1);
155 const v128_t va2c1 = wasm_v32x4_shuffle(va2, va2, 1, 1, 1, 1);
156 const v128_t va3c1 = wasm_v32x4_shuffle(va3, va3, 1, 1, 1, 1);
157 const v128_t va4c1 = wasm_v32x4_shuffle(va4, va4, 1, 1, 1, 1);
158 const v128_t va5c1 = wasm_v32x4_shuffle(va5, va5, 1, 1, 1, 1);
159
160 const v128_t vb0123c1 = wasm_v128_load(w + 8);
161 const v128_t vb4567c1 = wasm_v128_load(w + 12);
162
163 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0c1, vb0123c1));
164 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1c1, vb0123c1));
165 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2c1, vb0123c1));
166 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3c1, vb0123c1));
167 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4c1, vb0123c1));
168 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5c1, vb0123c1));
169 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0c1, vb4567c1));
170 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1c1, vb4567c1));
171 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2c1, vb4567c1));
172 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3c1, vb4567c1));
173 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4c1, vb4567c1));
174 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5c1, vb4567c1));
175 const v128_t va0c2 = wasm_v32x4_shuffle(va0, va0, 2, 2, 2, 2);
176 const v128_t va1c2 = wasm_v32x4_shuffle(va1, va1, 2, 2, 2, 2);
177 const v128_t va2c2 = wasm_v32x4_shuffle(va2, va2, 2, 2, 2, 2);
178 const v128_t va3c2 = wasm_v32x4_shuffle(va3, va3, 2, 2, 2, 2);
179 const v128_t va4c2 = wasm_v32x4_shuffle(va4, va4, 2, 2, 2, 2);
180 const v128_t va5c2 = wasm_v32x4_shuffle(va5, va5, 2, 2, 2, 2);
181
182 const v128_t vb0123c2 = wasm_v128_load(w + 16);
183 const v128_t vb4567c2 = wasm_v128_load(w + 20);
184
185 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0c2, vb0123c2));
186 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1c2, vb0123c2));
187 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2c2, vb0123c2));
188 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3c2, vb0123c2));
189 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4c2, vb0123c2));
190 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5c2, vb0123c2));
191 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0c2, vb4567c2));
192 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1c2, vb4567c2));
193 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2c2, vb4567c2));
194 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3c2, vb4567c2));
195 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4c2, vb4567c2));
196 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5c2, vb4567c2));
197 const v128_t va0c3 = wasm_v32x4_shuffle(va0, va0, 3, 3, 3, 3);
198 const v128_t va1c3 = wasm_v32x4_shuffle(va1, va1, 3, 3, 3, 3);
199 const v128_t va2c3 = wasm_v32x4_shuffle(va2, va2, 3, 3, 3, 3);
200 const v128_t va3c3 = wasm_v32x4_shuffle(va3, va3, 3, 3, 3, 3);
201 const v128_t va4c3 = wasm_v32x4_shuffle(va4, va4, 3, 3, 3, 3);
202 const v128_t va5c3 = wasm_v32x4_shuffle(va5, va5, 3, 3, 3, 3);
203
204 const v128_t vb0123c3 = wasm_v128_load(w + 24);
205 const v128_t vb4567c3 = wasm_v128_load(w + 28);
206
207 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0c3, vb0123c3));
208 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1c3, vb0123c3));
209 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2c3, vb0123c3));
210 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3c3, vb0123c3));
211 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4c3, vb0123c3));
212 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5c3, vb0123c3));
213 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0c3, vb4567c3));
214 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1c3, vb4567c3));
215 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2c3, vb4567c3));
216 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3c3, vb4567c3));
217 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4c3, vb4567c3));
218 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5c3, vb4567c3));
219
220 w += 32;
221 k -= 4 * sizeof(float);
222 }
223 if XNN_UNLIKELY(k != 0) {
224 do {
225 const v128_t vb0123 = wasm_v128_load(w);
226 const v128_t vb4567 = wasm_v128_load(w + 4);
227 w += 8;
228
229 const v128_t va0 = wasm_v128_load32_splat(a0);
230 a0 += 1;
231 const v128_t va1 = wasm_v128_load32_splat(a1);
232 a1 += 1;
233 const v128_t va2 = wasm_v128_load32_splat(a2);
234 a2 += 1;
235 const v128_t va3 = wasm_v128_load32_splat(a3);
236 a3 += 1;
237 const v128_t va4 = wasm_v128_load32_splat(a4);
238 a4 += 1;
239 const v128_t va5 = wasm_v128_load32_splat(a5);
240 a5 += 1;
241
242 vacc0x0123 = wasm_f32x4_add(vacc0x0123, wasm_f32x4_mul(va0, vb0123));
243 vacc0x4567 = wasm_f32x4_add(vacc0x4567, wasm_f32x4_mul(va0, vb4567));
244 vacc1x0123 = wasm_f32x4_add(vacc1x0123, wasm_f32x4_mul(va1, vb0123));
245 vacc1x4567 = wasm_f32x4_add(vacc1x4567, wasm_f32x4_mul(va1, vb4567));
246 vacc2x0123 = wasm_f32x4_add(vacc2x0123, wasm_f32x4_mul(va2, vb0123));
247 vacc2x4567 = wasm_f32x4_add(vacc2x4567, wasm_f32x4_mul(va2, vb4567));
248 vacc3x0123 = wasm_f32x4_add(vacc3x0123, wasm_f32x4_mul(va3, vb0123));
249 vacc3x4567 = wasm_f32x4_add(vacc3x4567, wasm_f32x4_mul(va3, vb4567));
250 vacc4x0123 = wasm_f32x4_add(vacc4x0123, wasm_f32x4_mul(va4, vb0123));
251 vacc4x4567 = wasm_f32x4_add(vacc4x4567, wasm_f32x4_mul(va4, vb4567));
252 vacc5x0123 = wasm_f32x4_add(vacc5x0123, wasm_f32x4_mul(va5, vb0123));
253 vacc5x4567 = wasm_f32x4_add(vacc5x4567, wasm_f32x4_mul(va5, vb4567));
254 k -= sizeof(float);
255 } while (k != 0);
256 }
257 p -= 6 * sizeof(void*);
258 } while (p != 0);
259
260 vacc0x0123 = wasm_f32x4_pmax(vmin, vacc0x0123);
261 vacc1x0123 = wasm_f32x4_pmax(vmin, vacc1x0123);
262 vacc2x0123 = wasm_f32x4_pmax(vmin, vacc2x0123);
263 vacc3x0123 = wasm_f32x4_pmax(vmin, vacc3x0123);
264 vacc4x0123 = wasm_f32x4_pmax(vmin, vacc4x0123);
265 vacc5x0123 = wasm_f32x4_pmax(vmin, vacc5x0123);
266 vacc0x4567 = wasm_f32x4_pmax(vmin, vacc0x4567);
267 vacc1x4567 = wasm_f32x4_pmax(vmin, vacc1x4567);
268 vacc2x4567 = wasm_f32x4_pmax(vmin, vacc2x4567);
269 vacc3x4567 = wasm_f32x4_pmax(vmin, vacc3x4567);
270 vacc4x4567 = wasm_f32x4_pmax(vmin, vacc4x4567);
271 vacc5x4567 = wasm_f32x4_pmax(vmin, vacc5x4567);
272
273 vacc0x0123 = wasm_f32x4_pmin(vmax, vacc0x0123);
274 vacc1x0123 = wasm_f32x4_pmin(vmax, vacc1x0123);
275 vacc2x0123 = wasm_f32x4_pmin(vmax, vacc2x0123);
276 vacc3x0123 = wasm_f32x4_pmin(vmax, vacc3x0123);
277 vacc4x0123 = wasm_f32x4_pmin(vmax, vacc4x0123);
278 vacc5x0123 = wasm_f32x4_pmin(vmax, vacc5x0123);
279 vacc0x4567 = wasm_f32x4_pmin(vmax, vacc0x4567);
280 vacc1x4567 = wasm_f32x4_pmin(vmax, vacc1x4567);
281 vacc2x4567 = wasm_f32x4_pmin(vmax, vacc2x4567);
282 vacc3x4567 = wasm_f32x4_pmin(vmax, vacc3x4567);
283 vacc4x4567 = wasm_f32x4_pmin(vmax, vacc4x4567);
284 vacc5x4567 = wasm_f32x4_pmin(vmax, vacc5x4567);
285
286 if XNN_LIKELY(nc >= 8) {
287 wasm_v128_store(c5, vacc5x0123);
288 wasm_v128_store(c5 + 4, vacc5x4567);
289 c5 = (float*) ((uintptr_t) c5 + cn_stride);
290 wasm_v128_store(c4, vacc4x0123);
291 wasm_v128_store(c4 + 4, vacc4x4567);
292 c4 = (float*) ((uintptr_t) c4 + cn_stride);
293 wasm_v128_store(c3, vacc3x0123);
294 wasm_v128_store(c3 + 4, vacc3x4567);
295 c3 = (float*) ((uintptr_t) c3 + cn_stride);
296 wasm_v128_store(c2, vacc2x0123);
297 wasm_v128_store(c2 + 4, vacc2x4567);
298 c2 = (float*) ((uintptr_t) c2 + cn_stride);
299 wasm_v128_store(c1, vacc1x0123);
300 wasm_v128_store(c1 + 4, vacc1x4567);
301 c1 = (float*) ((uintptr_t) c1 + cn_stride);
302 wasm_v128_store(c0, vacc0x0123);
303 wasm_v128_store(c0 + 4, vacc0x4567);
304 c0 = (float*) ((uintptr_t) c0 + cn_stride);
305
306 a = (const float**restrict) ((uintptr_t) a - ks);
307 nc -= 8;
308 } else {
309 if (nc & 4) {
310 wasm_v128_store(c5, vacc5x0123);
311 wasm_v128_store(c4, vacc4x0123);
312 wasm_v128_store(c3, vacc3x0123);
313 wasm_v128_store(c2, vacc2x0123);
314 wasm_v128_store(c1, vacc1x0123);
315 wasm_v128_store(c0, vacc0x0123);
316
317 vacc5x0123 = vacc5x4567;
318 vacc4x0123 = vacc4x4567;
319 vacc3x0123 = vacc3x4567;
320 vacc2x0123 = vacc2x4567;
321 vacc1x0123 = vacc1x4567;
322 vacc0x0123 = vacc0x4567;
323
324 c5 += 4;
325 c4 += 4;
326 c3 += 4;
327 c2 += 4;
328 c1 += 4;
329 c0 += 4;
330 }
331 if (nc & 2) {
332 *((double*) c5) = wasm_f64x2_extract_lane(vacc5x0123, 0);
333 *((double*) c4) = wasm_f64x2_extract_lane(vacc4x0123, 0);
334 *((double*) c3) = wasm_f64x2_extract_lane(vacc3x0123, 0);
335 *((double*) c2) = wasm_f64x2_extract_lane(vacc2x0123, 0);
336 *((double*) c1) = wasm_f64x2_extract_lane(vacc1x0123, 0);
337 *((double*) c0) = wasm_f64x2_extract_lane(vacc0x0123, 0);
338
339 vacc5x0123 = wasm_v32x4_shuffle(vacc5x0123, vacc5x0123, 2, 3, 2, 3);
340 vacc4x0123 = wasm_v32x4_shuffle(vacc4x0123, vacc4x0123, 2, 3, 2, 3);
341 vacc3x0123 = wasm_v32x4_shuffle(vacc3x0123, vacc3x0123, 2, 3, 2, 3);
342 vacc2x0123 = wasm_v32x4_shuffle(vacc2x0123, vacc2x0123, 2, 3, 2, 3);
343 vacc1x0123 = wasm_v32x4_shuffle(vacc1x0123, vacc1x0123, 2, 3, 2, 3);
344 vacc0x0123 = wasm_v32x4_shuffle(vacc0x0123, vacc0x0123, 2, 3, 2, 3);
345
346 c5 += 2;
347 c4 += 2;
348 c3 += 2;
349 c2 += 2;
350 c1 += 2;
351 c0 += 2;
352 }
353 if (nc & 1) {
354 *c5 = wasm_f32x4_extract_lane(vacc5x0123, 0);
355 *c4 = wasm_f32x4_extract_lane(vacc4x0123, 0);
356 *c3 = wasm_f32x4_extract_lane(vacc3x0123, 0);
357 *c2 = wasm_f32x4_extract_lane(vacc2x0123, 0);
358 *c1 = wasm_f32x4_extract_lane(vacc1x0123, 0);
359 *c0 = wasm_f32x4_extract_lane(vacc0x0123, 0);
360 }
361
362 nc = 0;
363 }
364 } while (nc != 0);
365 }
366