xref: /aosp_15_r20/external/XNNPACK/src/qu8-gavgpool/gen/7p7x-minmax-fp32-wasmsimd-c24.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-gavgpool/multipass-wasmsimd.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <wasm_simd128.h>
13 
14 #include <xnnpack/gavgpool.h>
15 #include <xnnpack/math.h>
16 
17 
xnn_qu8_gavgpool_minmax_fp32_ukernel_7p7x__wasmsimd_c24(size_t rows,size_t channels,const uint8_t * input,size_t input_stride,const uint8_t * zero,int32_t * buffer,uint8_t * output,const union xnn_qu8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_qu8_gavgpool_minmax_fp32_ukernel_7p7x__wasmsimd_c24(
19     size_t rows,
20     size_t channels,
21     const uint8_t* input,
22     size_t input_stride,
23     const uint8_t* zero,
24     int32_t* buffer,
25     uint8_t* output,
26     const union xnn_qu8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
27 {
28   assert(rows > 7);
29   assert(channels != 0);
30 
31   const uint8_t* i0 = input;
32   const uint8_t* i1 = (const uint8_t*) ((uintptr_t) i0 + input_stride);
33   const uint8_t* i2 = (const uint8_t*) ((uintptr_t) i1 + input_stride);
34   const uint8_t* i3 = (const uint8_t*) ((uintptr_t) i2 + input_stride);
35   const uint8_t* i4 = (const uint8_t*) ((uintptr_t) i3 + input_stride);
36   const uint8_t* i5 = (const uint8_t*) ((uintptr_t) i4 + input_stride);
37   const uint8_t* i6 = (const uint8_t*) ((uintptr_t) i5 + input_stride);
38   const size_t input_increment = 7 * input_stride - round_up_po2(channels, 8) * sizeof(uint8_t);
39 
40   const v128_t vinit_bias = wasm_v128_load64_splat(params->fp32_wasmsimd.init_bias);
41   int32_t* b = buffer;
42   size_t c = channels;
43   for (; c >= 24; c -= 24) {
44     const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
45     const v128_t vxi0x89ABCDEF = wasm_u16x8_load8x8(i0 + 8);
46     const v128_t vxi0xGHIJKLMN = wasm_u16x8_load8x8(i0 + 16);
47     i0 += 24;
48     const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
49     const v128_t vxi1x89ABCDEF = wasm_u16x8_load8x8(i1 + 8);
50     const v128_t vxi1xGHIJKLMN = wasm_u16x8_load8x8(i1 + 16);
51     i1 += 24;
52 
53     v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
54     const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
55     v128_t vacc89ABCDEF = wasm_i16x8_add(vxi0x89ABCDEF, vxi1x89ABCDEF);
56     const v128_t vxi2x89ABCDEF = wasm_u16x8_load8x8(i2 + 8);
57     v128_t vaccGHIJKLMN = wasm_i16x8_add(vxi0xGHIJKLMN, vxi1xGHIJKLMN);
58     const v128_t vxi2xGHIJKLMN = wasm_u16x8_load8x8(i2 + 16);
59     i2 += 24;
60 
61     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
62     const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
63     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi2x89ABCDEF);
64     const v128_t vxi3x89ABCDEF = wasm_u16x8_load8x8(i3 + 8);
65     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi2xGHIJKLMN);
66     const v128_t vxi3xGHIJKLMN = wasm_u16x8_load8x8(i3 + 16);
67     i3 += 24;
68     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
69     const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
70     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi3x89ABCDEF);
71     const v128_t vxi4x89ABCDEF = wasm_u16x8_load8x8(i4 + 8);
72     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi3xGHIJKLMN);
73     const v128_t vxi4xGHIJKLMN = wasm_u16x8_load8x8(i4 + 16);
74     i4 += 24;
75     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
76     const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
77     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi4x89ABCDEF);
78     const v128_t vxi5x89ABCDEF = wasm_u16x8_load8x8(i5 + 8);
79     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi4xGHIJKLMN);
80     const v128_t vxi5xGHIJKLMN = wasm_u16x8_load8x8(i5 + 16);
81     i5 += 24;
82     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
83     const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
84     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi5x89ABCDEF);
85     const v128_t vxi6x89ABCDEF = wasm_u16x8_load8x8(i6 + 8);
86     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi5xGHIJKLMN);
87     const v128_t vxi6xGHIJKLMN = wasm_u16x8_load8x8(i6 + 16);
88     i6 += 24;
89 
90     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
91     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi6x89ABCDEF);
92     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi6xGHIJKLMN);
93 
94     const v128_t vacc0123 = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_low_u16x8(vacc01234567));
95     const v128_t vacc4567 = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_high_u16x8(vacc01234567));
96     const v128_t vacc89AB = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_low_u16x8(vacc89ABCDEF));
97     const v128_t vaccCDEF = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_high_u16x8(vacc89ABCDEF));
98     const v128_t vaccGHIJ = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_low_u16x8(vaccGHIJKLMN));
99     const v128_t vaccKLMN = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_high_u16x8(vaccGHIJKLMN));
100 
101     wasm_v128_store(b, vacc0123);
102     wasm_v128_store(b + 4, vacc4567);
103     wasm_v128_store(b + 8, vacc89AB);
104     wasm_v128_store(b + 12, vaccCDEF);
105     wasm_v128_store(b + 16, vaccGHIJ);
106     wasm_v128_store(b + 20, vaccKLMN);
107     b += 24;
108   }
109   if XNN_UNLIKELY(c != 0) {
110     do {
111       const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
112       i0 += 8;
113       const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
114       i1 += 8;
115 
116       v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
117       const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
118       i2 += 8;
119 
120       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
121       const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
122       i3 += 8;
123       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
124       const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
125       i4 += 8;
126       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
127       const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
128       i5 += 8;
129       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
130       const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
131       i6 += 8;
132 
133       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
134 
135       const v128_t vacc0123 = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_low_u16x8(vacc01234567));
136       const v128_t vacc4567 = wasm_i32x4_add(vinit_bias, wasm_u32x4_extend_high_u16x8(vacc01234567));
137 
138       wasm_v128_store(b, vacc0123);
139       wasm_v128_store(b + 4, vacc4567);
140       b += 8;
141 
142       c = doz(c, 8);
143     } while (c != 0);
144   }
145 
146   for (rows -= 7; rows > 7; rows -= 7) {
147     i0 = (const uint8_t*) ((uintptr_t) i0 + input_increment);
148     i1 = (const uint8_t*) ((uintptr_t) i1 + input_increment);
149     i2 = (const uint8_t*) ((uintptr_t) i2 + input_increment);
150     i3 = (const uint8_t*) ((uintptr_t) i3 + input_increment);
151     i4 = (const uint8_t*) ((uintptr_t) i4 + input_increment);
152     i5 = (const uint8_t*) ((uintptr_t) i5 + input_increment);
153     i6 = (const uint8_t*) ((uintptr_t) i6 + input_increment);
154 
155     int32_t* b = buffer;
156     size_t c = channels;
157     for (; c >= 24; c -= 24) {
158       const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
159       const v128_t vxi0x89ABCDEF = wasm_u16x8_load8x8(i0 + 8);
160       const v128_t vxi0xGHIJKLMN = wasm_u16x8_load8x8(i0 + 16);
161       i0 += 24;
162       const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
163       const v128_t vxi1x89ABCDEF = wasm_u16x8_load8x8(i1 + 8);
164       const v128_t vxi1xGHIJKLMN = wasm_u16x8_load8x8(i1 + 16);
165       i1 += 24;
166 
167       v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
168       const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
169       v128_t vacc89ABCDEF = wasm_i16x8_add(vxi0x89ABCDEF, vxi1x89ABCDEF);
170       const v128_t vxi2x89ABCDEF = wasm_u16x8_load8x8(i2 + 8);
171       v128_t vaccGHIJKLMN = wasm_i16x8_add(vxi0xGHIJKLMN, vxi1xGHIJKLMN);
172       const v128_t vxi2xGHIJKLMN = wasm_u16x8_load8x8(i2 + 16);
173       i2 += 24;
174 
175       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
176       const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
177       vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi2x89ABCDEF);
178       const v128_t vxi3x89ABCDEF = wasm_u16x8_load8x8(i3 + 8);
179       vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi2xGHIJKLMN);
180       const v128_t vxi3xGHIJKLMN = wasm_u16x8_load8x8(i3 + 16);
181       i3 += 24;
182       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
183       const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
184       vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi3x89ABCDEF);
185       const v128_t vxi4x89ABCDEF = wasm_u16x8_load8x8(i4 + 8);
186       vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi3xGHIJKLMN);
187       const v128_t vxi4xGHIJKLMN = wasm_u16x8_load8x8(i4 + 16);
188       i4 += 24;
189       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
190       const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
191       vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi4x89ABCDEF);
192       const v128_t vxi5x89ABCDEF = wasm_u16x8_load8x8(i5 + 8);
193       vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi4xGHIJKLMN);
194       const v128_t vxi5xGHIJKLMN = wasm_u16x8_load8x8(i5 + 16);
195       i5 += 24;
196       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
197       const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
198       vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi5x89ABCDEF);
199       const v128_t vxi6x89ABCDEF = wasm_u16x8_load8x8(i6 + 8);
200       vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi5xGHIJKLMN);
201       const v128_t vxi6xGHIJKLMN = wasm_u16x8_load8x8(i6 + 16);
202       i6 += 24;
203 
204       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
205       vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi6x89ABCDEF);
206       vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi6xGHIJKLMN);
207 
208       v128_t vacc0123 = wasm_v128_load(b);
209       v128_t vacc4567 = wasm_v128_load(b + 4);
210       v128_t vacc89AB = wasm_v128_load(b + 8);
211       v128_t vaccCDEF = wasm_v128_load(b + 12);
212       v128_t vaccGHIJ = wasm_v128_load(b + 16);
213       v128_t vaccKLMN = wasm_v128_load(b + 20);
214 
215       vacc0123 = wasm_i32x4_add(vacc0123, wasm_u32x4_extend_low_u16x8(vacc01234567));
216       vacc4567 = wasm_i32x4_add(vacc4567, wasm_u32x4_extend_high_u16x8(vacc01234567));
217       vacc89AB = wasm_i32x4_add(vacc89AB, wasm_u32x4_extend_low_u16x8(vacc89ABCDEF));
218       vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_u32x4_extend_high_u16x8(vacc89ABCDEF));
219       vaccGHIJ = wasm_i32x4_add(vaccGHIJ, wasm_u32x4_extend_low_u16x8(vaccGHIJKLMN));
220       vaccKLMN = wasm_i32x4_add(vaccKLMN, wasm_u32x4_extend_high_u16x8(vaccGHIJKLMN));
221 
222       wasm_v128_store(b, vacc0123);
223       wasm_v128_store(b + 4, vacc4567);
224       wasm_v128_store(b + 8, vacc89AB);
225       wasm_v128_store(b + 12, vaccCDEF);
226       wasm_v128_store(b + 16, vaccGHIJ);
227       wasm_v128_store(b + 20, vaccKLMN);
228       b += 24;
229     }
230     if XNN_UNLIKELY(c != 0) {
231       do {
232         const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
233         i0 += 8;
234         const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
235         i1 += 8;
236 
237         v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
238         const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
239         i2 += 8;
240 
241         vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
242         const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
243         i3 += 8;
244         vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
245         const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
246         i4 += 8;
247         vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
248         const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
249         i5 += 8;
250         vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
251         const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
252         i6 += 8;
253 
254         vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
255 
256         v128_t vacc0123 = wasm_v128_load(b);
257         v128_t vacc4567 = wasm_v128_load(b + 4);
258 
259         vacc0123 = wasm_i32x4_add(vacc0123, wasm_u32x4_extend_low_u16x8(vacc01234567));
260         vacc4567 = wasm_i32x4_add(vacc4567, wasm_u32x4_extend_high_u16x8(vacc01234567));
261 
262         wasm_v128_store(b, vacc0123);
263         wasm_v128_store(b + 4, vacc4567);
264         b += 8;
265 
266         c = doz(c, 8);
267       } while (c != 0);
268     }
269   }
270 
271   i0 = (const uint8_t*) ((uintptr_t) i0 + input_increment);
272   i1 = (const uint8_t*) ((uintptr_t) i1 + input_increment);
273   if XNN_UNPREDICTABLE(rows < 2) {
274     i1 = zero;
275   }
276   i2 = (const uint8_t*) ((uintptr_t) i2 + input_increment);
277   if XNN_UNPREDICTABLE(rows <= 2) {
278     i2 = zero;
279   }
280   i3 = (const uint8_t*) ((uintptr_t) i3 + input_increment);
281   if XNN_UNPREDICTABLE(rows < 4) {
282     i3 = zero;
283   }
284   i4 = (const uint8_t*) ((uintptr_t) i4 + input_increment);
285   if XNN_UNPREDICTABLE(rows <= 4) {
286     i4 = zero;
287   }
288   i5 = (const uint8_t*) ((uintptr_t) i5 + input_increment);
289   if XNN_UNPREDICTABLE(rows < 6) {
290     i5 = zero;
291   }
292   i6 = (const uint8_t*) ((uintptr_t) i6 + input_increment);
293   if XNN_UNPREDICTABLE(rows <= 6) {
294     i6 = zero;
295   }
296 
297   const v128_t vscale = wasm_v128_load64_splat(params->fp32_wasmsimd.scale);
298   const v128_t vmagic_bias = wasm_v128_load64_splat(params->fp32_wasmsimd.magic_bias);
299   const v128_t vmagic_min = wasm_v128_load64_splat(params->fp32_wasmsimd.magic_min);
300   const v128_t vmagic_bias_less_output_zero_point = wasm_v128_load64_splat(params->fp32_wasmsimd.magic_bias_less_output_zero_point);
301   const v128_t voutput_max = wasm_v128_load64_splat(params->fp32_wasmsimd.output_max);
302   for (; channels >= 24; channels -= 24) {
303     const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
304     const v128_t vxi0x89ABCDEF = wasm_u16x8_load8x8(i0 + 8);
305     const v128_t vxi0xGHIJKLMN = wasm_u16x8_load8x8(i0 + 16);
306     i0 += 24;
307     const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
308     const v128_t vxi1x89ABCDEF = wasm_u16x8_load8x8(i1 + 8);
309     const v128_t vxi1xGHIJKLMN = wasm_u16x8_load8x8(i1 + 16);
310     i1 += 24;
311 
312     v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
313     const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
314     v128_t vacc89ABCDEF = wasm_i16x8_add(vxi0x89ABCDEF, vxi1x89ABCDEF);
315     const v128_t vxi2x89ABCDEF = wasm_u16x8_load8x8(i2 + 8);
316     v128_t vaccGHIJKLMN = wasm_i16x8_add(vxi0xGHIJKLMN, vxi1xGHIJKLMN);
317     const v128_t vxi2xGHIJKLMN = wasm_u16x8_load8x8(i2 + 16);
318     i2 += 24;
319 
320     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
321     const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
322     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi2x89ABCDEF);
323     const v128_t vxi3x89ABCDEF = wasm_u16x8_load8x8(i3 + 8);
324     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi2xGHIJKLMN);
325     const v128_t vxi3xGHIJKLMN = wasm_u16x8_load8x8(i3 + 16);
326     i3 += 24;
327     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
328     const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
329     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi3x89ABCDEF);
330     const v128_t vxi4x89ABCDEF = wasm_u16x8_load8x8(i4 + 8);
331     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi3xGHIJKLMN);
332     const v128_t vxi4xGHIJKLMN = wasm_u16x8_load8x8(i4 + 16);
333     i4 += 24;
334     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
335     const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
336     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi4x89ABCDEF);
337     const v128_t vxi5x89ABCDEF = wasm_u16x8_load8x8(i5 + 8);
338     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi4xGHIJKLMN);
339     const v128_t vxi5xGHIJKLMN = wasm_u16x8_load8x8(i5 + 16);
340     i5 += 24;
341     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
342     const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
343     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi5x89ABCDEF);
344     const v128_t vxi6x89ABCDEF = wasm_u16x8_load8x8(i6 + 8);
345     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi5xGHIJKLMN);
346     const v128_t vxi6xGHIJKLMN = wasm_u16x8_load8x8(i6 + 16);
347     i6 += 24;
348 
349     vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
350     vacc89ABCDEF = wasm_i16x8_add(vacc89ABCDEF, vxi6x89ABCDEF);
351     vaccGHIJKLMN = wasm_i16x8_add(vaccGHIJKLMN, vxi6xGHIJKLMN);
352 
353     v128_t vacc0123 = wasm_v128_load(buffer);
354     v128_t vacc4567 = wasm_v128_load(buffer + 4);
355     v128_t vacc89AB = wasm_v128_load(buffer + 8);
356     v128_t vaccCDEF = wasm_v128_load(buffer + 12);
357     v128_t vaccGHIJ = wasm_v128_load(buffer + 16);
358     v128_t vaccKLMN = wasm_v128_load(buffer + 20);
359     buffer += 24;
360 
361     vacc0123 = wasm_i32x4_add(vacc0123, wasm_u32x4_extend_low_u16x8(vacc01234567));
362     vacc4567 = wasm_i32x4_add(vacc4567, wasm_u32x4_extend_high_u16x8(vacc01234567));
363     vacc89AB = wasm_i32x4_add(vacc89AB, wasm_u32x4_extend_low_u16x8(vacc89ABCDEF));
364     vaccCDEF = wasm_i32x4_add(vaccCDEF, wasm_u32x4_extend_high_u16x8(vacc89ABCDEF));
365     vaccGHIJ = wasm_i32x4_add(vaccGHIJ, wasm_u32x4_extend_low_u16x8(vaccGHIJKLMN));
366     vaccKLMN = wasm_i32x4_add(vaccKLMN, wasm_u32x4_extend_high_u16x8(vaccGHIJKLMN));
367 
368     vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
369     vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
370     vacc89AB = wasm_f32x4_convert_i32x4(vacc89AB);
371     vaccCDEF = wasm_f32x4_convert_i32x4(vaccCDEF);
372     vaccGHIJ = wasm_f32x4_convert_i32x4(vaccGHIJ);
373     vaccKLMN = wasm_f32x4_convert_i32x4(vaccKLMN);
374 
375     vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
376     vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
377     vacc89AB = wasm_f32x4_mul(vacc89AB, vscale);
378     vaccCDEF = wasm_f32x4_mul(vaccCDEF, vscale);
379     vaccGHIJ = wasm_f32x4_mul(vaccGHIJ, vscale);
380     vaccKLMN = wasm_f32x4_mul(vaccKLMN, vscale);
381 
382     vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
383     vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
384     vacc89AB = wasm_f32x4_add(vacc89AB, vmagic_bias);
385     vaccCDEF = wasm_f32x4_add(vaccCDEF, vmagic_bias);
386     vaccGHIJ = wasm_f32x4_add(vaccGHIJ, vmagic_bias);
387     vaccKLMN = wasm_f32x4_add(vaccKLMN, vmagic_bias);
388 
389     vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
390     vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
391     vacc89AB = wasm_i32x4_max(vacc89AB, vmagic_min);
392     vaccCDEF = wasm_i32x4_max(vaccCDEF, vmagic_min);
393     vaccGHIJ = wasm_i32x4_max(vaccGHIJ, vmagic_min);
394     vaccKLMN = wasm_i32x4_max(vaccKLMN, vmagic_min);
395 
396     vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
397     vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
398     vacc89AB = wasm_i32x4_sub(vacc89AB, vmagic_bias_less_output_zero_point);
399     vaccCDEF = wasm_i32x4_sub(vaccCDEF, vmagic_bias_less_output_zero_point);
400     vaccGHIJ = wasm_i32x4_sub(vaccGHIJ, vmagic_bias_less_output_zero_point);
401     vaccKLMN = wasm_i32x4_sub(vaccKLMN, vmagic_bias_less_output_zero_point);
402 
403     v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
404     v128_t vout89ABCDEF = wasm_i16x8_narrow_i32x4(vacc89AB, vaccCDEF);
405     v128_t voutGHIJKLMN = wasm_i16x8_narrow_i32x4(vaccGHIJ, vaccKLMN);
406 
407     v128_t vout0123456789ABCDEF = wasm_u8x16_narrow_i16x8(vout01234567, vout89ABCDEF);
408     v128_t voutGHIJKLMNGHIJKLMN = wasm_u8x16_narrow_i16x8(voutGHIJKLMN, voutGHIJKLMN);
409 
410     vout0123456789ABCDEF = wasm_u8x16_min(vout0123456789ABCDEF, voutput_max);
411     voutGHIJKLMNGHIJKLMN = wasm_u8x16_min(voutGHIJKLMNGHIJKLMN, voutput_max);
412 
413     wasm_v128_store(output, vout0123456789ABCDEF);
414     *((double*) (output + 16)) = wasm_f64x2_extract_lane(voutGHIJKLMNGHIJKLMN, 0);
415     output += 24;
416   }
417   if XNN_UNLIKELY(channels != 0) {
418     do {
419       const v128_t vxi0x01234567 = wasm_u16x8_load8x8(i0);
420       i0 += 8;
421       const v128_t vxi1x01234567 = wasm_u16x8_load8x8(i1);
422       i1 += 8;
423 
424       v128_t vacc01234567 = wasm_i16x8_add(vxi0x01234567, vxi1x01234567);
425       const v128_t vxi2x01234567 = wasm_u16x8_load8x8(i2);
426       i2 += 8;
427 
428       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi2x01234567);
429       const v128_t vxi3x01234567 = wasm_u16x8_load8x8(i3);
430       i3 += 8;
431       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi3x01234567);
432       const v128_t vxi4x01234567 = wasm_u16x8_load8x8(i4);
433       i4 += 8;
434       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi4x01234567);
435       const v128_t vxi5x01234567 = wasm_u16x8_load8x8(i5);
436       i5 += 8;
437       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi5x01234567);
438       const v128_t vxi6x01234567 = wasm_u16x8_load8x8(i6);
439       i6 += 8;
440 
441       vacc01234567 = wasm_i16x8_add(vacc01234567, vxi6x01234567);
442 
443       v128_t vacc0123 = wasm_v128_load(buffer);
444       v128_t vacc4567 = wasm_v128_load(buffer + 4);
445       buffer += 8;
446 
447       vacc0123 = wasm_i32x4_add(vacc0123, wasm_u32x4_extend_low_u16x8(vacc01234567));
448       vacc4567 = wasm_i32x4_add(vacc4567, wasm_u32x4_extend_high_u16x8(vacc01234567));
449 
450       vacc0123 = wasm_f32x4_convert_i32x4(vacc0123);
451       vacc4567 = wasm_f32x4_convert_i32x4(vacc4567);
452 
453       vacc0123 = wasm_f32x4_mul(vacc0123, vscale);
454       vacc4567 = wasm_f32x4_mul(vacc4567, vscale);
455 
456       vacc0123 = wasm_f32x4_add(vacc0123, vmagic_bias);
457       vacc4567 = wasm_f32x4_add(vacc4567, vmagic_bias);
458 
459       vacc0123 = wasm_i32x4_max(vacc0123, vmagic_min);
460       vacc4567 = wasm_i32x4_max(vacc4567, vmagic_min);
461 
462       vacc0123 = wasm_i32x4_sub(vacc0123, vmagic_bias_less_output_zero_point);
463       vacc4567 = wasm_i32x4_sub(vacc4567, vmagic_bias_less_output_zero_point);
464 
465       const v128_t vout01234567 = wasm_i16x8_narrow_i32x4(vacc0123, vacc4567);
466       v128_t vout0123456701234567 = wasm_u8x16_narrow_i16x8(vout01234567, vout01234567);
467       vout0123456701234567 = wasm_u8x16_min(vout0123456701234567, voutput_max);
468 
469       if XNN_LIKELY(channels >= 8) {
470         *((double*) output) = wasm_f64x2_extract_lane(vout0123456701234567, 0);
471         output += 8;
472         channels -= 8;
473       } else {
474         if (channels & 4) {
475           *((float*) output) = wasm_f32x4_extract_lane(vout0123456701234567, 0);
476           vout0123456701234567 = wasm_u64x2_shr(vout0123456701234567, 32);
477           output += 4;
478         }
479         uint32_t vout0123 = wasm_i32x4_extract_lane(vout0123456701234567, 0);
480         if (channels & 2) {
481           *((uint16_t*) output) = (uint16_t) vout0123;
482           vout0123 >>= 16;
483           output += 2;
484         }
485         if (channels & 1) {
486           *output = (uint8_t) vout0123;
487           output += 1;
488         }
489         channels = 0;
490       }
491     } while (channels != 0);
492   }
493 }
494