xref: /aosp_15_r20/external/XNNPACK/src/qu8-dwconv/gen/up8x25-minmax-rndnu-neon-mul8.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qu8-dwconv/unipass-neon-mul8.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/dwconv.h>
15 
16 
xnn_qu8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8(size_t channels,size_t output_width,const uint8_t ** input,const void * weights,uint8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const uint8_t * zero,const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qu8_dwconv_minmax_rndnu_ukernel_up8x25__neon_mul8(
18     size_t channels,
19     size_t output_width,
20     const uint8_t** input,
21     const void* weights,
22     uint8_t* output,
23     size_t input_stride,
24     size_t output_increment,
25     size_t input_offset,
26     const uint8_t* zero,
27     const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(channels != 0);
30   assert(output_width != 0);
31 
32   const uint8x8_t vkernel_zero_point = vld1_dup_u8(params->rndnu_neon.kernel_zero_point);
33   const uint16x8_t vkernel_zero_point16 = vmovl_u8(vkernel_zero_point);
34   const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
35   const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
36   const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
37   const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
38   const uint8x8_t voutput_min = vld1_dup_u8(&params->rndnu_neon.output_min);
39   const uint8x8_t voutput_max = vld1_dup_u8(&params->rndnu_neon.output_max);
40   do {
41     const uint8_t* i0 = input[0];
42     assert(i0 != NULL);
43     if XNN_UNPREDICTABLE(i0 != zero) {
44       i0 = (const uint8_t*) ((uintptr_t) i0 + input_offset);
45     }
46     const uint8_t* i1 = input[1];
47     assert(i1 != NULL);
48     if XNN_UNPREDICTABLE(i1 != zero) {
49       i1 = (const uint8_t*) ((uintptr_t) i1 + input_offset);
50     }
51     const uint8_t* i2 = input[2];
52     assert(i2 != NULL);
53     if XNN_UNPREDICTABLE(i2 != zero) {
54       i2 = (const uint8_t*) ((uintptr_t) i2 + input_offset);
55     }
56     const uint8_t* i3 = input[3];
57     assert(i3 != NULL);
58     if XNN_UNPREDICTABLE(i3 != zero) {
59       i3 = (const uint8_t*) ((uintptr_t) i3 + input_offset);
60     }
61     const uint8_t* i4 = input[4];
62     assert(i4 != NULL);
63     if XNN_UNPREDICTABLE(i4 != zero) {
64       i4 = (const uint8_t*) ((uintptr_t) i4 + input_offset);
65     }
66     const uint8_t* i5 = input[5];
67     assert(i5 != NULL);
68     if XNN_UNPREDICTABLE(i5 != zero) {
69       i5 = (const uint8_t*) ((uintptr_t) i5 + input_offset);
70     }
71     const uint8_t* i6 = input[6];
72     assert(i6 != NULL);
73     if XNN_UNPREDICTABLE(i6 != zero) {
74       i6 = (const uint8_t*) ((uintptr_t) i6 + input_offset);
75     }
76     const uint8_t* i7 = input[7];
77     assert(i7 != NULL);
78     if XNN_UNPREDICTABLE(i7 != zero) {
79       i7 = (const uint8_t*) ((uintptr_t) i7 + input_offset);
80     }
81     const uint8_t* i8 = input[8];
82     assert(i8 != NULL);
83     if XNN_UNPREDICTABLE(i8 != zero) {
84       i8 = (const uint8_t*) ((uintptr_t) i8 + input_offset);
85     }
86     const uint8_t* i9 = input[9];
87     assert(i9 != NULL);
88     if XNN_UNPREDICTABLE(i9 != zero) {
89       i9 = (const uint8_t*) ((uintptr_t) i9 + input_offset);
90     }
91     const uint8_t* i10 = input[10];
92     assert(i10 != NULL);
93     if XNN_UNPREDICTABLE(i10 != zero) {
94       i10 = (const uint8_t*) ((uintptr_t) i10 + input_offset);
95     }
96     const uint8_t* i11 = input[11];
97     assert(i11 != NULL);
98     if XNN_UNPREDICTABLE(i11 != zero) {
99       i11 = (const uint8_t*) ((uintptr_t) i11 + input_offset);
100     }
101     const uint8_t* i12 = input[12];
102     assert(i12 != NULL);
103     if XNN_UNPREDICTABLE(i12 != zero) {
104       i12 = (const uint8_t*) ((uintptr_t) i12 + input_offset);
105     }
106     const uint8_t* i13 = input[13];
107     assert(i13 != NULL);
108     if XNN_UNPREDICTABLE(i13 != zero) {
109       i13 = (const uint8_t*) ((uintptr_t) i13 + input_offset);
110     }
111     const uint8_t* i14 = input[14];
112     assert(i14 != NULL);
113     if XNN_UNPREDICTABLE(i14 != zero) {
114       i14 = (const uint8_t*) ((uintptr_t) i14 + input_offset);
115     }
116     const uint8_t* i15 = input[15];
117     assert(i15 != NULL);
118     if XNN_UNPREDICTABLE(i15 != zero) {
119       i15 = (const uint8_t*) ((uintptr_t) i15 + input_offset);
120     }
121     const uint8_t* i16 = input[16];
122     assert(i16 != NULL);
123     if XNN_UNPREDICTABLE(i16 != zero) {
124       i16 = (const uint8_t*) ((uintptr_t) i16 + input_offset);
125     }
126     const uint8_t* i17 = input[17];
127     assert(i17 != NULL);
128     if XNN_UNPREDICTABLE(i17 != zero) {
129       i17 = (const uint8_t*) ((uintptr_t) i17 + input_offset);
130     }
131     const uint8_t* i18 = input[18];
132     assert(i18 != NULL);
133     if XNN_UNPREDICTABLE(i18 != zero) {
134       i18 = (const uint8_t*) ((uintptr_t) i18 + input_offset);
135     }
136     const uint8_t* i19 = input[19];
137     assert(i19 != NULL);
138     if XNN_UNPREDICTABLE(i19 != zero) {
139       i19 = (const uint8_t*) ((uintptr_t) i19 + input_offset);
140     }
141     const uint8_t* i20 = input[20];
142     assert(i20 != NULL);
143     if XNN_UNPREDICTABLE(i20 != zero) {
144       i20 = (const uint8_t*) ((uintptr_t) i20 + input_offset);
145     }
146     const uint8_t* i21 = input[21];
147     assert(i21 != NULL);
148     if XNN_UNPREDICTABLE(i21 != zero) {
149       i21 = (const uint8_t*) ((uintptr_t) i21 + input_offset);
150     }
151     const uint8_t* i22 = input[22];
152     assert(i22 != NULL);
153     if XNN_UNPREDICTABLE(i22 != zero) {
154       i22 = (const uint8_t*) ((uintptr_t) i22 + input_offset);
155     }
156     const uint8_t* i23 = input[23];
157     assert(i23 != NULL);
158     if XNN_UNPREDICTABLE(i23 != zero) {
159       i23 = (const uint8_t*) ((uintptr_t) i23 + input_offset);
160     }
161     const uint8_t* i24 = input[24];
162     assert(i24 != NULL);
163     if XNN_UNPREDICTABLE(i24 != zero) {
164       i24 = (const uint8_t*) ((uintptr_t) i24 + input_offset);
165     }
166     input = (const uint8_t**) ((uintptr_t) input + input_stride);
167 
168 
169     size_t c = channels;
170     const void* w = weights;
171     for (; c >= 8; c -= 8) {
172       int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
173       int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
174 
175 
176       const uint8x8_t vi0x01234567 = vld1_u8(i0); i0 += 8;
177       const uint8x8_t vk0x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
178 
179       uint16x8_t vprod01234567 = vmull_u8(vi0x01234567, vk0x01234567);
180 
181       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
182       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
183       const uint8x8_t vi1x01234567 = vld1_u8(i1); i1 += 8;
184       const uint8x8_t vk1x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
185 
186       vprod01234567 = vmull_u8(vi1x01234567, vk1x01234567);
187       uint16x8_t vsum01234567 = vaddl_u8(vi0x01234567, vi1x01234567);
188 
189       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
190       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
191       const uint8x8_t vi2x01234567 = vld1_u8(i2); i2 += 8;
192       const uint8x8_t vk2x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
193 
194       vprod01234567 = vmull_u8(vi2x01234567, vk2x01234567);
195       vsum01234567 = vaddw_u8(vsum01234567, vi2x01234567);
196 
197       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
198       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
199       const uint8x8_t vi3x01234567 = vld1_u8(i3); i3 += 8;
200       const uint8x8_t vk3x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
201 
202       vprod01234567 = vmull_u8(vi3x01234567, vk3x01234567);
203       vsum01234567 = vaddw_u8(vsum01234567, vi3x01234567);
204 
205       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
206       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
207       const uint8x8_t vi4x01234567 = vld1_u8(i4); i4 += 8;
208       const uint8x8_t vk4x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
209 
210       vprod01234567 = vmull_u8(vi4x01234567, vk4x01234567);
211       vsum01234567 = vaddw_u8(vsum01234567, vi4x01234567);
212 
213       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
214       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
215       const uint8x8_t vi5x01234567 = vld1_u8(i5); i5 += 8;
216       const uint8x8_t vk5x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
217 
218       vprod01234567 = vmull_u8(vi5x01234567, vk5x01234567);
219       vsum01234567 = vaddw_u8(vsum01234567, vi5x01234567);
220 
221       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
222       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
223       const uint8x8_t vi6x01234567 = vld1_u8(i6); i6 += 8;
224       const uint8x8_t vk6x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
225 
226       vprod01234567 = vmull_u8(vi6x01234567, vk6x01234567);
227       vsum01234567 = vaddw_u8(vsum01234567, vi6x01234567);
228 
229       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
230       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
231       const uint8x8_t vi7x01234567 = vld1_u8(i7); i7 += 8;
232       const uint8x8_t vk7x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
233 
234       vprod01234567 = vmull_u8(vi7x01234567, vk7x01234567);
235       vsum01234567 = vaddw_u8(vsum01234567, vi7x01234567);
236 
237       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
238       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
239       const uint8x8_t vi8x01234567 = vld1_u8(i8); i8 += 8;
240       const uint8x8_t vk8x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
241 
242       vprod01234567 = vmull_u8(vi8x01234567, vk8x01234567);
243       vsum01234567 = vaddw_u8(vsum01234567, vi8x01234567);
244 
245       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
246       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
247       const uint8x8_t vi9x01234567 = vld1_u8(i9); i9 += 8;
248       const uint8x8_t vk9x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
249 
250       vprod01234567 = vmull_u8(vi9x01234567, vk9x01234567);
251       vsum01234567 = vaddw_u8(vsum01234567, vi9x01234567);
252 
253       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
254       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
255       const uint8x8_t vi10x01234567 = vld1_u8(i10); i10 += 8;
256       const uint8x8_t vk10x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
257 
258       vprod01234567 = vmull_u8(vi10x01234567, vk10x01234567);
259       vsum01234567 = vaddw_u8(vsum01234567, vi10x01234567);
260 
261       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
262       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
263       const uint8x8_t vi11x01234567 = vld1_u8(i11); i11 += 8;
264       const uint8x8_t vk11x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
265 
266       vprod01234567 = vmull_u8(vi11x01234567, vk11x01234567);
267       vsum01234567 = vaddw_u8(vsum01234567, vi11x01234567);
268 
269       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
270       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
271       const uint8x8_t vi12x01234567 = vld1_u8(i12); i12 += 8;
272       const uint8x8_t vk12x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
273 
274       vprod01234567 = vmull_u8(vi12x01234567, vk12x01234567);
275       vsum01234567 = vaddw_u8(vsum01234567, vi12x01234567);
276 
277       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
278       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
279       const uint8x8_t vi13x01234567 = vld1_u8(i13); i13 += 8;
280       const uint8x8_t vk13x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
281 
282       vprod01234567 = vmull_u8(vi13x01234567, vk13x01234567);
283       vsum01234567 = vaddw_u8(vsum01234567, vi13x01234567);
284 
285       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
286       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
287       const uint8x8_t vi14x01234567 = vld1_u8(i14); i14 += 8;
288       const uint8x8_t vk14x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
289 
290       vprod01234567 = vmull_u8(vi14x01234567, vk14x01234567);
291       vsum01234567 = vaddw_u8(vsum01234567, vi14x01234567);
292 
293       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
294       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
295       const uint8x8_t vi15x01234567 = vld1_u8(i15); i15 += 8;
296       const uint8x8_t vk15x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
297 
298       vprod01234567 = vmull_u8(vi15x01234567, vk15x01234567);
299       vsum01234567 = vaddw_u8(vsum01234567, vi15x01234567);
300 
301       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
302       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
303       const uint8x8_t vi16x01234567 = vld1_u8(i16); i16 += 8;
304       const uint8x8_t vk16x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
305 
306       vprod01234567 = vmull_u8(vi16x01234567, vk16x01234567);
307       vsum01234567 = vaddw_u8(vsum01234567, vi16x01234567);
308 
309       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
310       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
311       const uint8x8_t vi17x01234567 = vld1_u8(i17); i17 += 8;
312       const uint8x8_t vk17x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
313 
314       vprod01234567 = vmull_u8(vi17x01234567, vk17x01234567);
315       vsum01234567 = vaddw_u8(vsum01234567, vi17x01234567);
316 
317       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
318       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
319       const uint8x8_t vi18x01234567 = vld1_u8(i18); i18 += 8;
320       const uint8x8_t vk18x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
321 
322       vprod01234567 = vmull_u8(vi18x01234567, vk18x01234567);
323       vsum01234567 = vaddw_u8(vsum01234567, vi18x01234567);
324 
325       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
326       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
327       const uint8x8_t vi19x01234567 = vld1_u8(i19); i19 += 8;
328       const uint8x8_t vk19x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
329 
330       vprod01234567 = vmull_u8(vi19x01234567, vk19x01234567);
331       vsum01234567 = vaddw_u8(vsum01234567, vi19x01234567);
332 
333       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
334       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
335       const uint8x8_t vi20x01234567 = vld1_u8(i20); i20 += 8;
336       const uint8x8_t vk20x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
337 
338       vprod01234567 = vmull_u8(vi20x01234567, vk20x01234567);
339       vsum01234567 = vaddw_u8(vsum01234567, vi20x01234567);
340 
341       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
342       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
343       const uint8x8_t vi21x01234567 = vld1_u8(i21); i21 += 8;
344       const uint8x8_t vk21x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
345 
346       vprod01234567 = vmull_u8(vi21x01234567, vk21x01234567);
347       vsum01234567 = vaddw_u8(vsum01234567, vi21x01234567);
348 
349       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
350       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
351       const uint8x8_t vi22x01234567 = vld1_u8(i22); i22 += 8;
352       const uint8x8_t vk22x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
353 
354       vprod01234567 = vmull_u8(vi22x01234567, vk22x01234567);
355       vsum01234567 = vaddw_u8(vsum01234567, vi22x01234567);
356 
357       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
358       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
359       const uint8x8_t vi23x01234567 = vld1_u8(i23); i23 += 8;
360       const uint8x8_t vk23x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
361 
362       vprod01234567 = vmull_u8(vi23x01234567, vk23x01234567);
363       vsum01234567 = vaddw_u8(vsum01234567, vi23x01234567);
364 
365       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
366       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
367       const uint8x8_t vi24x01234567 = vld1_u8(i24); i24 += 8;
368       const uint8x8_t vk24x01234567 = vld1_u8(w); w = (const void*) ((const int8_t*) w + 8);
369 
370       vprod01234567 = vmull_u8(vi24x01234567, vk24x01234567);
371       vsum01234567 = vaddw_u8(vsum01234567, vi24x01234567);
372 
373       vacc0123 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vprod01234567)));
374       vacc4567 = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vprod01234567)));
375 
376       vacc0123 = vreinterpretq_s32_u32(vmlsl_u16(vreinterpretq_u32_s32(vacc0123), vget_low_u16(vsum01234567), vget_low_u16(vkernel_zero_point16)));
377       vacc4567 = vreinterpretq_s32_u32(vmlsl_u16(vreinterpretq_u32_s32(vacc4567), vget_high_u16(vsum01234567), vget_high_u16(vkernel_zero_point16)));
378 
379       vacc0123 = vshlq_s32(vacc0123, vright_pre_shift);
380       vacc4567 = vshlq_s32(vacc4567, vright_pre_shift);
381 
382       vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
383       vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
384 
385       vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
386       vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
387 
388 #if XNN_ARCH_ARM64
389       const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
390 
391       uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
392 #else
393       const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
394 
395       uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
396 #endif
397 
398       vout01234567 = vmax_u8(vout01234567, voutput_min);
399 
400       vout01234567 = vmin_u8(vout01234567, voutput_max);
401 
402       vst1_u8(output, vout01234567); output += 8;
403     }
404     if XNN_UNLIKELY(c != 0) {
405       {
406         int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
407         int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
408 
409         const int16x8_t vi0x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0)));
410         const int16x8_t vk0x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point));
411 
412         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
413         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
414         const int16x8_t vi1x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1)));
415         const int16x8_t vk1x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 8)), vkernel_zero_point));
416 
417         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
418         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
419         const int16x8_t vi2x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2)));
420         const int16x8_t vk2x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 16)), vkernel_zero_point));
421 
422         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
423         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
424         const int16x8_t vi3x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3)));
425         const int16x8_t vk3x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 24)), vkernel_zero_point));
426 
427         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
428         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
429         const int16x8_t vi4x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4)));
430         const int16x8_t vk4x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 32)), vkernel_zero_point));
431 
432         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
433         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
434         const int16x8_t vi5x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5)));
435         const int16x8_t vk5x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 40)), vkernel_zero_point));
436 
437         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
438         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
439         const int16x8_t vi6x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6)));
440         const int16x8_t vk6x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 48)), vkernel_zero_point));
441 
442         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
443         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
444         const int16x8_t vi7x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7)));
445         const int16x8_t vk7x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 56)), vkernel_zero_point));
446 
447         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
448         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
449         const int16x8_t vi8x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8)));
450         const int16x8_t vk8x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 64)), vkernel_zero_point));
451 
452         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
453         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
454         const int16x8_t vi9x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9)));
455         const int16x8_t vk9x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 72)), vkernel_zero_point));
456 
457         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi9x01234567), vget_low_s16(vk9x01234567));
458         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi9x01234567), vget_high_s16(vk9x01234567));
459         const int16x8_t vi10x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10)));
460         const int16x8_t vk10x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 80)), vkernel_zero_point));
461 
462         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi10x01234567), vget_low_s16(vk10x01234567));
463         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi10x01234567), vget_high_s16(vk10x01234567));
464         const int16x8_t vi11x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11)));
465         const int16x8_t vk11x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 88)), vkernel_zero_point));
466 
467         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi11x01234567), vget_low_s16(vk11x01234567));
468         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi11x01234567), vget_high_s16(vk11x01234567));
469         const int16x8_t vi12x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12)));
470         const int16x8_t vk12x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 96)), vkernel_zero_point));
471 
472         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi12x01234567), vget_low_s16(vk12x01234567));
473         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi12x01234567), vget_high_s16(vk12x01234567));
474         const int16x8_t vi13x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13)));
475         const int16x8_t vk13x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 104)), vkernel_zero_point));
476 
477         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi13x01234567), vget_low_s16(vk13x01234567));
478         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi13x01234567), vget_high_s16(vk13x01234567));
479         const int16x8_t vi14x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14)));
480         const int16x8_t vk14x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 112)), vkernel_zero_point));
481 
482         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi14x01234567), vget_low_s16(vk14x01234567));
483         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi14x01234567), vget_high_s16(vk14x01234567));
484         const int16x8_t vi15x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15)));
485         const int16x8_t vk15x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 120)), vkernel_zero_point));
486 
487         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi15x01234567), vget_low_s16(vk15x01234567));
488         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi15x01234567), vget_high_s16(vk15x01234567));
489         const int16x8_t vi16x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16)));
490         const int16x8_t vk16x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 128)), vkernel_zero_point));
491 
492         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi16x01234567), vget_low_s16(vk16x01234567));
493         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi16x01234567), vget_high_s16(vk16x01234567));
494         const int16x8_t vi17x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17)));
495         const int16x8_t vk17x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 136)), vkernel_zero_point));
496 
497         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi17x01234567), vget_low_s16(vk17x01234567));
498         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi17x01234567), vget_high_s16(vk17x01234567));
499         const int16x8_t vi18x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18)));
500         const int16x8_t vk18x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 144)), vkernel_zero_point));
501 
502         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi18x01234567), vget_low_s16(vk18x01234567));
503         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi18x01234567), vget_high_s16(vk18x01234567));
504         const int16x8_t vi19x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19)));
505         const int16x8_t vk19x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 152)), vkernel_zero_point));
506 
507         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi19x01234567), vget_low_s16(vk19x01234567));
508         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi19x01234567), vget_high_s16(vk19x01234567));
509         const int16x8_t vi20x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20)));
510         const int16x8_t vk20x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 160)), vkernel_zero_point));
511 
512         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi20x01234567), vget_low_s16(vk20x01234567));
513         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi20x01234567), vget_high_s16(vk20x01234567));
514         const int16x8_t vi21x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21)));
515         const int16x8_t vk21x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 168)), vkernel_zero_point));
516 
517         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi21x01234567), vget_low_s16(vk21x01234567));
518         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi21x01234567), vget_high_s16(vk21x01234567));
519         const int16x8_t vi22x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22)));
520         const int16x8_t vk22x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 176)), vkernel_zero_point));
521 
522         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi22x01234567), vget_low_s16(vk22x01234567));
523         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi22x01234567), vget_high_s16(vk22x01234567));
524         const int16x8_t vi23x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23)));
525         const int16x8_t vk23x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 184)), vkernel_zero_point));
526 
527         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi23x01234567), vget_low_s16(vk23x01234567));
528         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi23x01234567), vget_high_s16(vk23x01234567));
529         const int16x8_t vi24x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24)));
530         const int16x8_t vk24x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) ((const uint8_t*) w + 192)), vkernel_zero_point));
531 
532         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi24x01234567), vget_low_s16(vk24x01234567));
533         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi24x01234567), vget_high_s16(vk24x01234567));
534 
535         vacc0123 = vrshlq_s32(vacc0123, vright_pre_shift);
536         vacc4567 = vrshlq_s32(vacc4567, vright_pre_shift);
537 
538         vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
539         vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
540 
541         vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
542         vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
543 
544 #if XNN_ARCH_ARM64
545         const int16x8_t vacc01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567), voutput_zero_point);
546         uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
547 #else
548         const int16x8_t vacc01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567)), voutput_zero_point);
549         uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
550 #endif
551 
552         vout01234567 = vmax_u8(vout01234567, voutput_min);
553         vout01234567 = vmin_u8(vout01234567, voutput_max);
554 
555         if (c & 4) {
556           vst1_lane_u32((void*) output, vreinterpret_u32_u8(vout01234567), 0); output += 4;
557           vout01234567 = vext_u8(vout01234567, vout01234567, 4);
558         }
559         if (c & 2) {
560           vst1_lane_u16((void*) output, vreinterpret_u16_u8(vout01234567), 0); output += 2;
561           vout01234567 = vext_u8(vout01234567, vout01234567, 2);
562         }
563         if (c & 1) {
564           vst1_lane_u8(output, vout01234567, 0); output += 1;
565         }
566       }
567     }
568 
569     output = (uint8_t*) ((uintptr_t) output + output_increment);
570   } while (--output_width != 0);
571 }
572