1 // Auto-generated file. Do not edit!
2 //   Template: src/f32-dwconv2d-chw/3x3s2p1-wasmsimd-loadsplat.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <wasm_simd128.h>
13 
14 #include <xnnpack/dwconv.h>
15 #include <xnnpack/math.h>
16 
17 
18 
xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__wasmsimd_x86_loadsplat_4x4(size_t input_height,size_t input_width,const float * input,const float * weights,const float * zero,float * output,uint32_t padding_top,const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS (1)])19 void xnn_f32_dwconv2d_chw_ukernel_3x3s2p1__wasmsimd_x86_loadsplat_4x4(
20     size_t input_height,
21     size_t input_width,
22     const float* input,
23     const float* weights,
24     const float* zero,
25     float* output,
26     uint32_t padding_top,
27     const union xnn_f32_chw_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(input_height != 0);
30   assert(input_width != 0);
31   assert(input_width % sizeof(float) == 0);
32   assert(padding_top >= 0);
33   assert(padding_top <= 1);
34 
35   const v128_t vmask_even = wasm_v128_load(params->scalar.mask_even);
36   const v128_t vmask_odd  = wasm_v128_load(params->scalar.mask_odd);
37   const v128_t vmax = wasm_v128_load32_splat(&params->scalar.max);
38   const v128_t vmin = wasm_v128_load32_splat(&params->scalar.min);
39 
40   const v128_t vw0123 = wasm_v128_load(weights);
41   const v128_t vw4567 = wasm_v128_load(weights + 4);
42   const v128_t vw89 = wasm_v128_load64_splat(weights + 8);
43   const v128_t vbias = wasm_v32x4_shuffle(vw0123, vw0123, 0, 0, 0, 0);
44   const v128_t vk00 = wasm_v32x4_shuffle(vw0123, vw0123, 1, 1, 1, 1);
45   const v128_t vk01 = wasm_v32x4_shuffle(vw0123, vw0123, 2, 2, 2, 2);
46   const v128_t vk02 = wasm_v32x4_shuffle(vw0123, vw0123, 3, 3, 3, 3);
47   const v128_t vk10 = wasm_v32x4_shuffle(vw4567, vw4567, 0, 0, 0, 0);
48   const v128_t vk11 = wasm_v32x4_shuffle(vw4567, vw4567, 1, 1, 1, 1);
49   const v128_t vk12 = wasm_v32x4_shuffle(vw4567, vw4567, 2, 2, 2, 2);
50   const v128_t vk20 = wasm_v32x4_shuffle(vw4567, vw4567, 3, 3, 3, 3);
51   const v128_t vk21 = wasm_v32x4_shuffle(vw89, vw89, 0, 0, 0, 0);
52   const v128_t vk22 = wasm_v32x4_shuffle(vw89, vw89, 1, 1, 1, 1);
53 
54   const size_t input_decrement = round_down_po2(input_width, 4 /* SIMD output width */ * 2 /* subsampling */ * sizeof(float));
55   const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(float)) / 2, sizeof(float));
56 
57   const float* i0 = (const float*) ((uintptr_t) input - ((-padding_top) & input_width));
58   const float* i1 = (const float*) ((uintptr_t) i0 + input_width);
59   if XNN_UNPREDICTABLE(padding_top != 0) {
60     i0 = zero;
61   }
62   const float* i2 = (const float*) ((uintptr_t) i1 + input_width);
63   const float* i3 = (const float*) ((uintptr_t) i2 + input_width);
64   const float* i4 = (const float*) ((uintptr_t) i3 + input_width);
65   const float* i5 = (const float*) ((uintptr_t) i4 + input_width);
66   const float* i6 = (const float*) ((uintptr_t) i5 + input_width);
67   const float* i7 = (const float*) ((uintptr_t) i6 + input_width);
68   const float* i8 = (const float*) ((uintptr_t) i7 + input_width);
69 
70   float* o0 = output;
71   float* o1 = (float*) ((uintptr_t) o0 + output_width);
72   float* o2 = (float*) ((uintptr_t) o1 + output_width);
73   float* o3 = (float*) ((uintptr_t) o2 + output_width);
74 
75   size_t padded_input_height = input_height + padding_top + 1 /* padding bottom */;
76   size_t output_height = (padded_input_height - 3 /* kernel size */ + 2 /* subsampling */) / 2;
77   do {
78     if XNN_UNPREDICTABLE(padded_input_height < 4) {
79       i2 = zero;
80     }
81     if XNN_UNPREDICTABLE(padded_input_height < 5) {
82       i3 = zero;
83       o1 = o0;
84     }
85     if XNN_UNPREDICTABLE(padded_input_height < 6) {
86       i4 = zero;
87     }
88     if XNN_UNPREDICTABLE(padded_input_height < 7) {
89       i5 = zero;
90       o2 = o1;
91     }
92     if XNN_UNPREDICTABLE(padded_input_height < 8) {
93       i6 = zero;
94     }
95     if XNN_UNPREDICTABLE(padded_input_height < 9) {
96       i7 = zero;
97       o3 = o2;
98     }
99     if XNN_UNPREDICTABLE(padded_input_height < 10) {
100       i8 = zero;
101     }
102 
103     v128_t vi0x1357 = wasm_f32x4_const_splat(0.0f);
104     v128_t vi1x1357 = wasm_f32x4_const_splat(0.0f);
105     v128_t vi2x1357 = wasm_f32x4_const_splat(0.0f);
106     v128_t vi3x1357 = wasm_f32x4_const_splat(0.0f);
107     v128_t vi4x1357 = wasm_f32x4_const_splat(0.0f);
108     v128_t vi5x1357 = wasm_f32x4_const_splat(0.0f);
109     v128_t vi6x1357 = wasm_f32x4_const_splat(0.0f);
110     v128_t vi7x1357 = wasm_f32x4_const_splat(0.0f);
111     v128_t vi8x1357 = wasm_f32x4_const_splat(0.0f);
112 
113     size_t w = input_width;
114     for (; w >= 8 * sizeof(float); w -= 8 * sizeof(float)) {
115       v128_t vo0p0 = vbias;
116       v128_t vo1p0 = vbias;
117       v128_t vo2p0 = vbias;
118       v128_t vo3p0 = vbias;
119 
120       const v128_t vi0x89AB = wasm_v128_load(i0);
121       const v128_t vi0xCDEF = wasm_v128_load(i0 + 4);
122       i0 += 8;
123       const v128_t vi1x89AB = wasm_v128_load(i1);
124       const v128_t vi1xCDEF = wasm_v128_load(i1 + 4);
125       i1 += 8;
126       const v128_t vi2x89AB = wasm_v128_load(i2);
127       const v128_t vi2xCDEF = wasm_v128_load(i2 + 4);
128       i2 += 8;
129       const v128_t vi3x89AB = wasm_v128_load(i3);
130       const v128_t vi3xCDEF = wasm_v128_load(i3 + 4);
131       i3 += 8;
132       const v128_t vi4x89AB = wasm_v128_load(i4);
133       const v128_t vi4xCDEF = wasm_v128_load(i4 + 4);
134       i4 += 8;
135       const v128_t vi5x89AB = wasm_v128_load(i5);
136       const v128_t vi5xCDEF = wasm_v128_load(i5 + 4);
137       i5 += 8;
138       const v128_t vi6x89AB = wasm_v128_load(i6);
139       const v128_t vi6xCDEF = wasm_v128_load(i6 + 4);
140       i6 += 8;
141       const v128_t vi7x89AB = wasm_v128_load(i7);
142       const v128_t vi7xCDEF = wasm_v128_load(i7 + 4);
143       i7 += 8;
144       const v128_t vi8x89AB = wasm_v128_load(i8);
145       const v128_t vi8xCDEF = wasm_v128_load(i8 + 4);
146       i8 += 8;
147 
148       const v128_t vi0x8ACE = wasm_v32x4_shuffle(vi0x89AB, vi0xCDEF, 0, 2, 4, 6);
149       const v128_t vi0x9BDF = wasm_v32x4_shuffle(vi0x89AB, vi0xCDEF, 1, 3, 5, 7);
150       const v128_t vi1x8ACE = wasm_v32x4_shuffle(vi1x89AB, vi1xCDEF, 0, 2, 4, 6);
151       const v128_t vi1x9BDF = wasm_v32x4_shuffle(vi1x89AB, vi1xCDEF, 1, 3, 5, 7);
152       const v128_t vi2x8ACE = wasm_v32x4_shuffle(vi2x89AB, vi2xCDEF, 0, 2, 4, 6);
153       const v128_t vi2x9BDF = wasm_v32x4_shuffle(vi2x89AB, vi2xCDEF, 1, 3, 5, 7);
154       const v128_t vi3x8ACE = wasm_v32x4_shuffle(vi3x89AB, vi3xCDEF, 0, 2, 4, 6);
155       const v128_t vi3x9BDF = wasm_v32x4_shuffle(vi3x89AB, vi3xCDEF, 1, 3, 5, 7);
156       const v128_t vi4x8ACE = wasm_v32x4_shuffle(vi4x89AB, vi4xCDEF, 0, 2, 4, 6);
157       const v128_t vi4x9BDF = wasm_v32x4_shuffle(vi4x89AB, vi4xCDEF, 1, 3, 5, 7);
158       const v128_t vi5x8ACE = wasm_v32x4_shuffle(vi5x89AB, vi5xCDEF, 0, 2, 4, 6);
159       const v128_t vi5x9BDF = wasm_v32x4_shuffle(vi5x89AB, vi5xCDEF, 1, 3, 5, 7);
160       const v128_t vi6x8ACE = wasm_v32x4_shuffle(vi6x89AB, vi6xCDEF, 0, 2, 4, 6);
161       const v128_t vi6x9BDF = wasm_v32x4_shuffle(vi6x89AB, vi6xCDEF, 1, 3, 5, 7);
162       const v128_t vi7x8ACE = wasm_v32x4_shuffle(vi7x89AB, vi7xCDEF, 0, 2, 4, 6);
163       const v128_t vi7x9BDF = wasm_v32x4_shuffle(vi7x89AB, vi7xCDEF, 1, 3, 5, 7);
164       const v128_t vi8x8ACE = wasm_v32x4_shuffle(vi8x89AB, vi8xCDEF, 0, 2, 4, 6);
165       const v128_t vi8x9BDF = wasm_v32x4_shuffle(vi8x89AB, vi8xCDEF, 1, 3, 5, 7);
166 
167       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x8ACE, vk01));
168       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x8ACE, vk01));
169       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x8ACE, vk01));
170       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x8ACE, vk01));
171 
172       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x8ACE, vk11));
173       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x8ACE, vk11));
174       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x8ACE, vk11));
175       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x8ACE, vk11));
176 
177       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x8ACE, vk21));
178       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x8ACE, vk21));
179       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x8ACE, vk21));
180       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x8ACE, vk21));
181 
182       const v128_t vi0x7BDF = wasm_v32x4_shuffle(vi0x1357, vi0x9BDF, 3, 4, 5, 6);
183       vi0x1357 = vi0x9BDF;
184       const v128_t vi1x7BDF = wasm_v32x4_shuffle(vi1x1357, vi1x9BDF, 3, 4, 5, 6);
185       vi1x1357 = vi1x9BDF;
186       const v128_t vi2x7BDF = wasm_v32x4_shuffle(vi2x1357, vi2x9BDF, 3, 4, 5, 6);
187       vi2x1357 = vi2x9BDF;
188       const v128_t vi3x7BDF = wasm_v32x4_shuffle(vi3x1357, vi3x9BDF, 3, 4, 5, 6);
189       vi3x1357 = vi3x9BDF;
190       const v128_t vi4x7BDF = wasm_v32x4_shuffle(vi4x1357, vi4x9BDF, 3, 4, 5, 6);
191       vi4x1357 = vi4x9BDF;
192       const v128_t vi5x7BDF = wasm_v32x4_shuffle(vi5x1357, vi5x9BDF, 3, 4, 5, 6);
193       vi5x1357 = vi5x9BDF;
194       const v128_t vi6x7BDF = wasm_v32x4_shuffle(vi6x1357, vi6x9BDF, 3, 4, 5, 6);
195       vi6x1357 = vi6x9BDF;
196       const v128_t vi7x7BDF = wasm_v32x4_shuffle(vi7x1357, vi7x9BDF, 3, 4, 5, 6);
197       vi7x1357 = vi7x9BDF;
198       const v128_t vi8x7BDF = wasm_v32x4_shuffle(vi8x1357, vi8x9BDF, 3, 4, 5, 6);
199       vi8x1357 = vi8x9BDF;
200 
201       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x7BDF, vk00));
202       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x7BDF, vk00));
203       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x7BDF, vk00));
204       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x7BDF, vk00));
205 
206       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x7BDF, vk10));
207       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x7BDF, vk10));
208       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x7BDF, vk10));
209       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x7BDF, vk10));
210 
211       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x7BDF, vk20));
212       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x7BDF, vk20));
213       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x7BDF, vk20));
214       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x7BDF, vk20));
215 
216       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x9BDF, vk02));
217       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x9BDF, vk02));
218       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x9BDF, vk02));
219       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x9BDF, vk02));
220 
221       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x9BDF, vk12));
222       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x9BDF, vk12));
223       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x9BDF, vk12));
224       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x9BDF, vk12));
225 
226       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x9BDF, vk22));
227       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x9BDF, vk22));
228       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x9BDF, vk22));
229       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x9BDF, vk22));
230 
231 
232       v128_t vo0 = wasm_f32x4_pmax(vmin, vo0p0);
233       v128_t vo1 = wasm_f32x4_pmax(vmin, vo1p0);
234       v128_t vo2 = wasm_f32x4_pmax(vmin, vo2p0);
235       v128_t vo3 = wasm_f32x4_pmax(vmin, vo3p0);
236       vo0 = wasm_f32x4_pmin(vmax, vo0);
237       vo1 = wasm_f32x4_pmin(vmax, vo1);
238       vo2 = wasm_f32x4_pmin(vmax, vo2);
239       vo3 = wasm_f32x4_pmin(vmax, vo3);
240 
241       wasm_v128_store(o3, vo3); o3 += 4;
242       wasm_v128_store(o2, vo2); o2 += 4;
243       wasm_v128_store(o1, vo1); o1 += 4;
244       wasm_v128_store(o0, vo0); o0 += 4;
245     }
246     // Last block has 0-7 pixels to process.
247     assert(w < 8 * sizeof(float));
248     if XNN_LIKELY(w != 0) {
249       v128_t vo0p0 = vbias;
250       v128_t vo1p0 = vbias;
251       v128_t vo2p0 = vbias;
252       v128_t vo3p0 = vbias;
253 
254       const v128_t vi0x89AB = wasm_v128_load(i0);
255       const v128_t vi0xCDEF = wasm_v128_load(i0 + 4);
256       const v128_t vi1x89AB = wasm_v128_load(i1);
257       const v128_t vi1xCDEF = wasm_v128_load(i1 + 4);
258       const v128_t vi2x89AB = wasm_v128_load(i2);
259       const v128_t vi2xCDEF = wasm_v128_load(i2 + 4);
260       const v128_t vi3x89AB = wasm_v128_load(i3);
261       const v128_t vi3xCDEF = wasm_v128_load(i3 + 4);
262       const v128_t vi4x89AB = wasm_v128_load(i4);
263       const v128_t vi4xCDEF = wasm_v128_load(i4 + 4);
264       const v128_t vi5x89AB = wasm_v128_load(i5);
265       const v128_t vi5xCDEF = wasm_v128_load(i5 + 4);
266       const v128_t vi6x89AB = wasm_v128_load(i6);
267       const v128_t vi6xCDEF = wasm_v128_load(i6 + 4);
268       const v128_t vi7x89AB = wasm_v128_load(i7);
269       const v128_t vi7xCDEF = wasm_v128_load(i7 + 4);
270       const v128_t vi8x89AB = wasm_v128_load(i8);
271       const v128_t vi8xCDEF = wasm_v128_load(i8 + 4);
272 
273       const v128_t vi0x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi0x89AB, vi0xCDEF, 0, 2, 4, 6));
274       const v128_t vi0x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi0x89AB, vi0xCDEF, 1, 3, 5, 7));
275       const v128_t vi1x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi1x89AB, vi1xCDEF, 0, 2, 4, 6));
276       const v128_t vi1x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi1x89AB, vi1xCDEF, 1, 3, 5, 7));
277       const v128_t vi2x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi2x89AB, vi2xCDEF, 0, 2, 4, 6));
278       const v128_t vi2x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi2x89AB, vi2xCDEF, 1, 3, 5, 7));
279       const v128_t vi3x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi3x89AB, vi3xCDEF, 0, 2, 4, 6));
280       const v128_t vi3x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi3x89AB, vi3xCDEF, 1, 3, 5, 7));
281       const v128_t vi4x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi4x89AB, vi4xCDEF, 0, 2, 4, 6));
282       const v128_t vi4x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi4x89AB, vi4xCDEF, 1, 3, 5, 7));
283       const v128_t vi5x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi5x89AB, vi5xCDEF, 0, 2, 4, 6));
284       const v128_t vi5x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi5x89AB, vi5xCDEF, 1, 3, 5, 7));
285       const v128_t vi6x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi6x89AB, vi6xCDEF, 0, 2, 4, 6));
286       const v128_t vi6x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi6x89AB, vi6xCDEF, 1, 3, 5, 7));
287       const v128_t vi7x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi7x89AB, vi7xCDEF, 0, 2, 4, 6));
288       const v128_t vi7x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi7x89AB, vi7xCDEF, 1, 3, 5, 7));
289       const v128_t vi8x8ACE = wasm_v128_and(vmask_even, wasm_v32x4_shuffle(vi8x89AB, vi8xCDEF, 0, 2, 4, 6));
290       const v128_t vi8x9BDF = wasm_v128_and(vmask_odd,  wasm_v32x4_shuffle(vi8x89AB, vi8xCDEF, 1, 3, 5, 7));
291 
292       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x8ACE, vk01));
293       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x8ACE, vk01));
294       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x8ACE, vk01));
295       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x8ACE, vk01));
296 
297       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x8ACE, vk11));
298       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x8ACE, vk11));
299       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x8ACE, vk11));
300       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x8ACE, vk11));
301 
302       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x8ACE, vk21));
303       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x8ACE, vk21));
304       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x8ACE, vk21));
305       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x8ACE, vk21));
306 
307       const v128_t vi0x7BDF = wasm_v32x4_shuffle(vi0x1357, vi0x9BDF, 3, 4, 5, 6);
308       const v128_t vi1x7BDF = wasm_v32x4_shuffle(vi1x1357, vi1x9BDF, 3, 4, 5, 6);
309       const v128_t vi2x7BDF = wasm_v32x4_shuffle(vi2x1357, vi2x9BDF, 3, 4, 5, 6);
310       const v128_t vi3x7BDF = wasm_v32x4_shuffle(vi3x1357, vi3x9BDF, 3, 4, 5, 6);
311       const v128_t vi4x7BDF = wasm_v32x4_shuffle(vi4x1357, vi4x9BDF, 3, 4, 5, 6);
312       const v128_t vi5x7BDF = wasm_v32x4_shuffle(vi5x1357, vi5x9BDF, 3, 4, 5, 6);
313       const v128_t vi6x7BDF = wasm_v32x4_shuffle(vi6x1357, vi6x9BDF, 3, 4, 5, 6);
314       const v128_t vi7x7BDF = wasm_v32x4_shuffle(vi7x1357, vi7x9BDF, 3, 4, 5, 6);
315       const v128_t vi8x7BDF = wasm_v32x4_shuffle(vi8x1357, vi8x9BDF, 3, 4, 5, 6);
316 
317       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x7BDF, vk00));
318       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x7BDF, vk00));
319       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x7BDF, vk00));
320       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x7BDF, vk00));
321 
322       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x7BDF, vk10));
323       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x7BDF, vk10));
324       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x7BDF, vk10));
325       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x7BDF, vk10));
326 
327       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x7BDF, vk20));
328       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x7BDF, vk20));
329       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x7BDF, vk20));
330       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x7BDF, vk20));
331 
332       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi0x9BDF, vk02));
333       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi2x9BDF, vk02));
334       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi4x9BDF, vk02));
335       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi6x9BDF, vk02));
336 
337       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi1x9BDF, vk12));
338       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi3x9BDF, vk12));
339       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi5x9BDF, vk12));
340       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi7x9BDF, vk12));
341 
342       vo0p0 = wasm_f32x4_add(vo0p0, wasm_f32x4_mul(vi2x9BDF, vk22));
343       vo1p0 = wasm_f32x4_add(vo1p0, wasm_f32x4_mul(vi4x9BDF, vk22));
344       vo2p0 = wasm_f32x4_add(vo2p0, wasm_f32x4_mul(vi6x9BDF, vk22));
345       vo3p0 = wasm_f32x4_add(vo3p0, wasm_f32x4_mul(vi8x9BDF, vk22));
346 
347 
348       v128_t vo0 = wasm_f32x4_pmax(vmin, vo0p0);
349       v128_t vo1 = wasm_f32x4_pmax(vmin, vo1p0);
350       v128_t vo2 = wasm_f32x4_pmax(vmin, vo2p0);
351       v128_t vo3 = wasm_f32x4_pmax(vmin, vo3p0);
352       vo0 = wasm_f32x4_pmin(vmax, vo0);
353       vo1 = wasm_f32x4_pmin(vmax, vo1);
354       vo2 = wasm_f32x4_pmin(vmax, vo2);
355       vo3 = wasm_f32x4_pmin(vmax, vo3);
356 
357       w += 1 * sizeof(float);
358       if (w & (8 * sizeof(float))) {
359         wasm_v128_store(o3, vo3); o3 += 4;
360         wasm_v128_store(o2, vo2); o2 += 4;
361         wasm_v128_store(o1, vo1); o1 += 4;
362         wasm_v128_store(o0, vo0); o0 += 4;
363       } else {
364         if (w & (4 * sizeof(float))) {
365           *((double*) o3) = wasm_f64x2_extract_lane(vo3, 0); o3 += 2;
366           *((double*) o2) = wasm_f64x2_extract_lane(vo2, 0); o2 += 2;
367           *((double*) o1) = wasm_f64x2_extract_lane(vo1, 0); o1 += 2;
368           *((double*) o0) = wasm_f64x2_extract_lane(vo0, 0); o0 += 2;
369 
370           vo0 = wasm_v32x4_shuffle(vo0, vo0, 2, 3, 0, 1);
371           vo1 = wasm_v32x4_shuffle(vo1, vo1, 2, 3, 0, 1);
372           vo2 = wasm_v32x4_shuffle(vo2, vo2, 2, 3, 0, 1);
373           vo3 = wasm_v32x4_shuffle(vo3, vo3, 2, 3, 0, 1);
374         }
375         if (w & (2 * sizeof(float))) {
376           *o3 = wasm_f32x4_extract_lane(vo3, 0); o3 += 1;
377           *o2 = wasm_f32x4_extract_lane(vo2, 0); o2 += 1;
378           *o1 = wasm_f32x4_extract_lane(vo1, 0); o1 += 1;
379           *o0 = wasm_f32x4_extract_lane(vo0, 0); o0 += 1;
380         }
381       }
382     }
383 
384     i0 = (const float*) ((uintptr_t) i8 - input_decrement);
385     i1 = (const float*) ((uintptr_t) i0 + input_width);
386     i2 = (const float*) ((uintptr_t) i1 + input_width);
387     i3 = (const float*) ((uintptr_t) i2 + input_width);
388     i4 = (const float*) ((uintptr_t) i3 + input_width);
389     i5 = (const float*) ((uintptr_t) i4 + input_width);
390     i6 = (const float*) ((uintptr_t) i5 + input_width);
391     i7 = (const float*) ((uintptr_t) i6 + input_width);
392     i8 = (const float*) ((uintptr_t) i7 + input_width);
393 
394     o0 = o3;
395     o1 = (float*) ((uintptr_t) o0 + output_width);
396     o2 = (float*) ((uintptr_t) o1 + output_width);
397     o3 = (float*) ((uintptr_t) o2 + output_width);
398 
399     output_height = doz(output_height, 4);
400     padded_input_height = doz(padded_input_height, 8);
401   } while (output_height != 0);
402 }
403