xref: /aosp_15_r20/external/XNNPACK/src/qu8-dwconv/gen/up24x25-minmax-fp32-neonv8-mul16.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-dwconv/unipass-neon-mul16.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/dwconv.h>
15 #include <xnnpack/intrinsics-polyfill.h>
16 
17 
xnn_qu8_dwconv_minmax_fp32_ukernel_up24x25__neonv8_mul16(size_t channels,size_t output_width,const uint8_t ** input,const void * weights,uint8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const uint8_t * zero,const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_qu8_dwconv_minmax_fp32_ukernel_up24x25__neonv8_mul16(
19     size_t channels,
20     size_t output_width,
21     const uint8_t** input,
22     const void* weights,
23     uint8_t* output,
24     size_t input_stride,
25     size_t output_increment,
26     size_t input_offset,
27     const uint8_t* zero,
28     const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
29 {
30   assert(channels != 0);
31   assert(output_width != 0);
32 
33   const uint8x8_t vkernel_zero_point = vld1_dup_u8(params->fp32_neonv8.kernel_zero_point);
34   const float32x4_t vscale = vld1q_dup_f32(&params->fp32_neonv8.scale);
35   const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
36   const uint8x16_t voutput_min = vld1q_dup_u8(&params->fp32_neonv8.output_min);
37   const uint8x16_t voutput_max = vld1q_dup_u8(&params->fp32_neonv8.output_max);
38   do {
39     const uint8_t* i0 = input[0];
40     assert(i0 != NULL);
41     if XNN_UNPREDICTABLE(i0 != zero) {
42       i0 = (const uint8_t*) ((uintptr_t) i0 + input_offset);
43     }
44     const uint8_t* i1 = input[1];
45     assert(i1 != NULL);
46     if XNN_UNPREDICTABLE(i1 != zero) {
47       i1 = (const uint8_t*) ((uintptr_t) i1 + input_offset);
48     }
49     const uint8_t* i2 = input[2];
50     assert(i2 != NULL);
51     if XNN_UNPREDICTABLE(i2 != zero) {
52       i2 = (const uint8_t*) ((uintptr_t) i2 + input_offset);
53     }
54     const uint8_t* i3 = input[3];
55     assert(i3 != NULL);
56     if XNN_UNPREDICTABLE(i3 != zero) {
57       i3 = (const uint8_t*) ((uintptr_t) i3 + input_offset);
58     }
59     const uint8_t* i4 = input[4];
60     assert(i4 != NULL);
61     if XNN_UNPREDICTABLE(i4 != zero) {
62       i4 = (const uint8_t*) ((uintptr_t) i4 + input_offset);
63     }
64     const uint8_t* i5 = input[5];
65     assert(i5 != NULL);
66     if XNN_UNPREDICTABLE(i5 != zero) {
67       i5 = (const uint8_t*) ((uintptr_t) i5 + input_offset);
68     }
69     const uint8_t* i6 = input[6];
70     assert(i6 != NULL);
71     if XNN_UNPREDICTABLE(i6 != zero) {
72       i6 = (const uint8_t*) ((uintptr_t) i6 + input_offset);
73     }
74     const uint8_t* i7 = input[7];
75     assert(i7 != NULL);
76     if XNN_UNPREDICTABLE(i7 != zero) {
77       i7 = (const uint8_t*) ((uintptr_t) i7 + input_offset);
78     }
79     const uint8_t* i8 = input[8];
80     assert(i8 != NULL);
81     if XNN_UNPREDICTABLE(i8 != zero) {
82       i8 = (const uint8_t*) ((uintptr_t) i8 + input_offset);
83     }
84     const uint8_t* i9 = input[9];
85     assert(i9 != NULL);
86     if XNN_UNPREDICTABLE(i9 != zero) {
87       i9 = (const uint8_t*) ((uintptr_t) i9 + input_offset);
88     }
89     const uint8_t* i10 = input[10];
90     assert(i10 != NULL);
91     if XNN_UNPREDICTABLE(i10 != zero) {
92       i10 = (const uint8_t*) ((uintptr_t) i10 + input_offset);
93     }
94     const uint8_t* i11 = input[11];
95     assert(i11 != NULL);
96     if XNN_UNPREDICTABLE(i11 != zero) {
97       i11 = (const uint8_t*) ((uintptr_t) i11 + input_offset);
98     }
99     const uint8_t* i12 = input[12];
100     assert(i12 != NULL);
101     if XNN_UNPREDICTABLE(i12 != zero) {
102       i12 = (const uint8_t*) ((uintptr_t) i12 + input_offset);
103     }
104     const uint8_t* i13 = input[13];
105     assert(i13 != NULL);
106     if XNN_UNPREDICTABLE(i13 != zero) {
107       i13 = (const uint8_t*) ((uintptr_t) i13 + input_offset);
108     }
109     const uint8_t* i14 = input[14];
110     assert(i14 != NULL);
111     if XNN_UNPREDICTABLE(i14 != zero) {
112       i14 = (const uint8_t*) ((uintptr_t) i14 + input_offset);
113     }
114     const uint8_t* i15 = input[15];
115     assert(i15 != NULL);
116     if XNN_UNPREDICTABLE(i15 != zero) {
117       i15 = (const uint8_t*) ((uintptr_t) i15 + input_offset);
118     }
119     const uint8_t* i16 = input[16];
120     assert(i16 != NULL);
121     if XNN_UNPREDICTABLE(i16 != zero) {
122       i16 = (const uint8_t*) ((uintptr_t) i16 + input_offset);
123     }
124     const uint8_t* i17 = input[17];
125     assert(i17 != NULL);
126     if XNN_UNPREDICTABLE(i17 != zero) {
127       i17 = (const uint8_t*) ((uintptr_t) i17 + input_offset);
128     }
129     const uint8_t* i18 = input[18];
130     assert(i18 != NULL);
131     if XNN_UNPREDICTABLE(i18 != zero) {
132       i18 = (const uint8_t*) ((uintptr_t) i18 + input_offset);
133     }
134     const uint8_t* i19 = input[19];
135     assert(i19 != NULL);
136     if XNN_UNPREDICTABLE(i19 != zero) {
137       i19 = (const uint8_t*) ((uintptr_t) i19 + input_offset);
138     }
139     const uint8_t* i20 = input[20];
140     assert(i20 != NULL);
141     if XNN_UNPREDICTABLE(i20 != zero) {
142       i20 = (const uint8_t*) ((uintptr_t) i20 + input_offset);
143     }
144     const uint8_t* i21 = input[21];
145     assert(i21 != NULL);
146     if XNN_UNPREDICTABLE(i21 != zero) {
147       i21 = (const uint8_t*) ((uintptr_t) i21 + input_offset);
148     }
149     const uint8_t* i22 = input[22];
150     assert(i22 != NULL);
151     if XNN_UNPREDICTABLE(i22 != zero) {
152       i22 = (const uint8_t*) ((uintptr_t) i22 + input_offset);
153     }
154     const uint8_t* i23 = input[23];
155     assert(i23 != NULL);
156     if XNN_UNPREDICTABLE(i23 != zero) {
157       i23 = (const uint8_t*) ((uintptr_t) i23 + input_offset);
158     }
159     const uint8_t* i24 = input[24];
160     assert(i24 != NULL);
161     if XNN_UNPREDICTABLE(i24 != zero) {
162       i24 = (const uint8_t*) ((uintptr_t) i24 + input_offset);
163     }
164     input = (const uint8_t**) ((uintptr_t) input + input_stride);
165 
166     size_t c = channels;
167     const void* w = weights;
168     for (; c >= 24; c -= 24) {
169       int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
170       int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
171       int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
172       int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
173       int32x4_t vaccGHIJ = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
174       int32x4_t vaccKLMN = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
175 
176 
177       const int16x8_t vi0x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
178       const int16x8_t vk0x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
179       const int16x8_t vi0x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
180       const int16x8_t vk0x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
181       const int16x8_t vi0xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
182       const int16x8_t vk0xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
183 
184       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
185       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
186       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
187       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
188       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
189       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
190 
191       const int16x8_t vi1x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
192       const int16x8_t vk1x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
193       const int16x8_t vi1x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
194       const int16x8_t vk1x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
195       const int16x8_t vi1xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
196       const int16x8_t vk1xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
197 
198       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
199       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
200       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
201       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
202       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
203       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
204 
205       const int16x8_t vi2x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
206       const int16x8_t vk2x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
207       const int16x8_t vi2x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
208       const int16x8_t vk2x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
209       const int16x8_t vi2xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
210       const int16x8_t vk2xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
211 
212       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
213       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
214       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
215       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
216       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
217       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
218 
219       const int16x8_t vi3x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
220       const int16x8_t vk3x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
221       const int16x8_t vi3x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
222       const int16x8_t vk3x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
223       const int16x8_t vi3xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
224       const int16x8_t vk3xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
225 
226       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
227       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
228       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
229       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
230       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
231       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
232 
233       const int16x8_t vi4x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
234       const int16x8_t vk4x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
235       const int16x8_t vi4x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
236       const int16x8_t vk4x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
237       const int16x8_t vi4xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
238       const int16x8_t vk4xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
239 
240       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
241       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
242       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
243       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
244       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
245       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
246 
247       const int16x8_t vi5x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
248       const int16x8_t vk5x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
249       const int16x8_t vi5x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
250       const int16x8_t vk5x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
251       const int16x8_t vi5xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
252       const int16x8_t vk5xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
253 
254       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
255       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
256       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
257       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
258       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi5xGHIJKLMN), vget_low_s16(vk5xGHIJKLMN));
259       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi5xGHIJKLMN), vget_high_s16(vk5xGHIJKLMN));
260 
261       const int16x8_t vi6x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
262       const int16x8_t vk6x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
263       const int16x8_t vi6x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
264       const int16x8_t vk6x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
265       const int16x8_t vi6xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
266       const int16x8_t vk6xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
267 
268       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
269       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
270       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
271       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
272       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi6xGHIJKLMN), vget_low_s16(vk6xGHIJKLMN));
273       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi6xGHIJKLMN), vget_high_s16(vk6xGHIJKLMN));
274 
275       const int16x8_t vi7x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
276       const int16x8_t vk7x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
277       const int16x8_t vi7x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
278       const int16x8_t vk7x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
279       const int16x8_t vi7xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
280       const int16x8_t vk7xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
281 
282       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
283       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
284       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
285       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
286       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi7xGHIJKLMN), vget_low_s16(vk7xGHIJKLMN));
287       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi7xGHIJKLMN), vget_high_s16(vk7xGHIJKLMN));
288 
289       const int16x8_t vi8x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
290       const int16x8_t vk8x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
291       const int16x8_t vi8x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
292       const int16x8_t vk8x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
293       const int16x8_t vi8xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
294       const int16x8_t vk8xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
295 
296       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
297       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
298       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi8x89ABCDEF), vget_low_s16(vk8x89ABCDEF));
299       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi8x89ABCDEF), vget_high_s16(vk8x89ABCDEF));
300       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi8xGHIJKLMN), vget_low_s16(vk8xGHIJKLMN));
301       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi8xGHIJKLMN), vget_high_s16(vk8xGHIJKLMN));
302 
303       const int16x8_t vi9x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
304       const int16x8_t vk9x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
305       const int16x8_t vi9x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
306       const int16x8_t vk9x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
307       const int16x8_t vi9xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
308       const int16x8_t vk9xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
309 
310       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi9x01234567), vget_low_s16(vk9x01234567));
311       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi9x01234567), vget_high_s16(vk9x01234567));
312       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi9x89ABCDEF), vget_low_s16(vk9x89ABCDEF));
313       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi9x89ABCDEF), vget_high_s16(vk9x89ABCDEF));
314       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi9xGHIJKLMN), vget_low_s16(vk9xGHIJKLMN));
315       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi9xGHIJKLMN), vget_high_s16(vk9xGHIJKLMN));
316 
317       const int16x8_t vi10x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
318       const int16x8_t vk10x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
319       const int16x8_t vi10x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
320       const int16x8_t vk10x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
321       const int16x8_t vi10xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
322       const int16x8_t vk10xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
323 
324       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi10x01234567), vget_low_s16(vk10x01234567));
325       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi10x01234567), vget_high_s16(vk10x01234567));
326       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi10x89ABCDEF), vget_low_s16(vk10x89ABCDEF));
327       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi10x89ABCDEF), vget_high_s16(vk10x89ABCDEF));
328       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi10xGHIJKLMN), vget_low_s16(vk10xGHIJKLMN));
329       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi10xGHIJKLMN), vget_high_s16(vk10xGHIJKLMN));
330 
331       const int16x8_t vi11x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
332       const int16x8_t vk11x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
333       const int16x8_t vi11x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
334       const int16x8_t vk11x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
335       const int16x8_t vi11xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
336       const int16x8_t vk11xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
337 
338       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi11x01234567), vget_low_s16(vk11x01234567));
339       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi11x01234567), vget_high_s16(vk11x01234567));
340       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi11x89ABCDEF), vget_low_s16(vk11x89ABCDEF));
341       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi11x89ABCDEF), vget_high_s16(vk11x89ABCDEF));
342       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi11xGHIJKLMN), vget_low_s16(vk11xGHIJKLMN));
343       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi11xGHIJKLMN), vget_high_s16(vk11xGHIJKLMN));
344 
345       const int16x8_t vi12x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
346       const int16x8_t vk12x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
347       const int16x8_t vi12x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
348       const int16x8_t vk12x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
349       const int16x8_t vi12xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
350       const int16x8_t vk12xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
351 
352       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi12x01234567), vget_low_s16(vk12x01234567));
353       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi12x01234567), vget_high_s16(vk12x01234567));
354       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi12x89ABCDEF), vget_low_s16(vk12x89ABCDEF));
355       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi12x89ABCDEF), vget_high_s16(vk12x89ABCDEF));
356       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi12xGHIJKLMN), vget_low_s16(vk12xGHIJKLMN));
357       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi12xGHIJKLMN), vget_high_s16(vk12xGHIJKLMN));
358 
359       const int16x8_t vi13x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
360       const int16x8_t vk13x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
361       const int16x8_t vi13x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
362       const int16x8_t vk13x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
363       const int16x8_t vi13xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
364       const int16x8_t vk13xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
365 
366       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi13x01234567), vget_low_s16(vk13x01234567));
367       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi13x01234567), vget_high_s16(vk13x01234567));
368       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi13x89ABCDEF), vget_low_s16(vk13x89ABCDEF));
369       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi13x89ABCDEF), vget_high_s16(vk13x89ABCDEF));
370       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi13xGHIJKLMN), vget_low_s16(vk13xGHIJKLMN));
371       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi13xGHIJKLMN), vget_high_s16(vk13xGHIJKLMN));
372 
373       const int16x8_t vi14x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
374       const int16x8_t vk14x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
375       const int16x8_t vi14x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
376       const int16x8_t vk14x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
377       const int16x8_t vi14xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
378       const int16x8_t vk14xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
379 
380       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi14x01234567), vget_low_s16(vk14x01234567));
381       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi14x01234567), vget_high_s16(vk14x01234567));
382       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi14x89ABCDEF), vget_low_s16(vk14x89ABCDEF));
383       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi14x89ABCDEF), vget_high_s16(vk14x89ABCDEF));
384       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi14xGHIJKLMN), vget_low_s16(vk14xGHIJKLMN));
385       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi14xGHIJKLMN), vget_high_s16(vk14xGHIJKLMN));
386 
387       const int16x8_t vi15x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
388       const int16x8_t vk15x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
389       const int16x8_t vi15x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
390       const int16x8_t vk15x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
391       const int16x8_t vi15xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
392       const int16x8_t vk15xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
393 
394       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi15x01234567), vget_low_s16(vk15x01234567));
395       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi15x01234567), vget_high_s16(vk15x01234567));
396       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi15x89ABCDEF), vget_low_s16(vk15x89ABCDEF));
397       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi15x89ABCDEF), vget_high_s16(vk15x89ABCDEF));
398       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi15xGHIJKLMN), vget_low_s16(vk15xGHIJKLMN));
399       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi15xGHIJKLMN), vget_high_s16(vk15xGHIJKLMN));
400 
401       const int16x8_t vi16x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
402       const int16x8_t vk16x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
403       const int16x8_t vi16x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
404       const int16x8_t vk16x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
405       const int16x8_t vi16xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
406       const int16x8_t vk16xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
407 
408       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi16x01234567), vget_low_s16(vk16x01234567));
409       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi16x01234567), vget_high_s16(vk16x01234567));
410       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi16x89ABCDEF), vget_low_s16(vk16x89ABCDEF));
411       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi16x89ABCDEF), vget_high_s16(vk16x89ABCDEF));
412       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi16xGHIJKLMN), vget_low_s16(vk16xGHIJKLMN));
413       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi16xGHIJKLMN), vget_high_s16(vk16xGHIJKLMN));
414 
415       const int16x8_t vi17x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
416       const int16x8_t vk17x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
417       const int16x8_t vi17x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
418       const int16x8_t vk17x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
419       const int16x8_t vi17xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
420       const int16x8_t vk17xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
421 
422       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi17x01234567), vget_low_s16(vk17x01234567));
423       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi17x01234567), vget_high_s16(vk17x01234567));
424       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi17x89ABCDEF), vget_low_s16(vk17x89ABCDEF));
425       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi17x89ABCDEF), vget_high_s16(vk17x89ABCDEF));
426       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi17xGHIJKLMN), vget_low_s16(vk17xGHIJKLMN));
427       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi17xGHIJKLMN), vget_high_s16(vk17xGHIJKLMN));
428 
429       const int16x8_t vi18x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
430       const int16x8_t vk18x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
431       const int16x8_t vi18x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
432       const int16x8_t vk18x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
433       const int16x8_t vi18xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
434       const int16x8_t vk18xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
435 
436       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi18x01234567), vget_low_s16(vk18x01234567));
437       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi18x01234567), vget_high_s16(vk18x01234567));
438       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi18x89ABCDEF), vget_low_s16(vk18x89ABCDEF));
439       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi18x89ABCDEF), vget_high_s16(vk18x89ABCDEF));
440       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi18xGHIJKLMN), vget_low_s16(vk18xGHIJKLMN));
441       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi18xGHIJKLMN), vget_high_s16(vk18xGHIJKLMN));
442 
443       const int16x8_t vi19x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
444       const int16x8_t vk19x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
445       const int16x8_t vi19x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
446       const int16x8_t vk19x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
447       const int16x8_t vi19xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
448       const int16x8_t vk19xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
449 
450       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi19x01234567), vget_low_s16(vk19x01234567));
451       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi19x01234567), vget_high_s16(vk19x01234567));
452       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi19x89ABCDEF), vget_low_s16(vk19x89ABCDEF));
453       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi19x89ABCDEF), vget_high_s16(vk19x89ABCDEF));
454       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi19xGHIJKLMN), vget_low_s16(vk19xGHIJKLMN));
455       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi19xGHIJKLMN), vget_high_s16(vk19xGHIJKLMN));
456 
457       const int16x8_t vi20x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
458       const int16x8_t vk20x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
459       const int16x8_t vi20x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
460       const int16x8_t vk20x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
461       const int16x8_t vi20xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
462       const int16x8_t vk20xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
463 
464       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi20x01234567), vget_low_s16(vk20x01234567));
465       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi20x01234567), vget_high_s16(vk20x01234567));
466       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi20x89ABCDEF), vget_low_s16(vk20x89ABCDEF));
467       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi20x89ABCDEF), vget_high_s16(vk20x89ABCDEF));
468       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi20xGHIJKLMN), vget_low_s16(vk20xGHIJKLMN));
469       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi20xGHIJKLMN), vget_high_s16(vk20xGHIJKLMN));
470 
471       const int16x8_t vi21x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
472       const int16x8_t vk21x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
473       const int16x8_t vi21x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
474       const int16x8_t vk21x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
475       const int16x8_t vi21xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
476       const int16x8_t vk21xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
477 
478       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi21x01234567), vget_low_s16(vk21x01234567));
479       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi21x01234567), vget_high_s16(vk21x01234567));
480       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi21x89ABCDEF), vget_low_s16(vk21x89ABCDEF));
481       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi21x89ABCDEF), vget_high_s16(vk21x89ABCDEF));
482       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi21xGHIJKLMN), vget_low_s16(vk21xGHIJKLMN));
483       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi21xGHIJKLMN), vget_high_s16(vk21xGHIJKLMN));
484 
485       const int16x8_t vi22x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
486       const int16x8_t vk22x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
487       const int16x8_t vi22x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
488       const int16x8_t vk22x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
489       const int16x8_t vi22xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
490       const int16x8_t vk22xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
491 
492       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi22x01234567), vget_low_s16(vk22x01234567));
493       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi22x01234567), vget_high_s16(vk22x01234567));
494       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi22x89ABCDEF), vget_low_s16(vk22x89ABCDEF));
495       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi22x89ABCDEF), vget_high_s16(vk22x89ABCDEF));
496       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi22xGHIJKLMN), vget_low_s16(vk22xGHIJKLMN));
497       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi22xGHIJKLMN), vget_high_s16(vk22xGHIJKLMN));
498 
499       const int16x8_t vi23x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
500       const int16x8_t vk23x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
501       const int16x8_t vi23x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
502       const int16x8_t vk23x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
503       const int16x8_t vi23xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
504       const int16x8_t vk23xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
505 
506       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi23x01234567), vget_low_s16(vk23x01234567));
507       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi23x01234567), vget_high_s16(vk23x01234567));
508       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi23x89ABCDEF), vget_low_s16(vk23x89ABCDEF));
509       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi23x89ABCDEF), vget_high_s16(vk23x89ABCDEF));
510       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi23xGHIJKLMN), vget_low_s16(vk23xGHIJKLMN));
511       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi23xGHIJKLMN), vget_high_s16(vk23xGHIJKLMN));
512 
513       const int16x8_t vi24x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
514       const int16x8_t vk24x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
515       const int16x8_t vi24x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
516       const int16x8_t vk24x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
517       const int16x8_t vi24xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
518       const int16x8_t vk24xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
519 
520       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi24x01234567), vget_low_s16(vk24x01234567));
521       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi24x01234567), vget_high_s16(vk24x01234567));
522       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi24x89ABCDEF), vget_low_s16(vk24x89ABCDEF));
523       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi24x89ABCDEF), vget_high_s16(vk24x89ABCDEF));
524       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi24xGHIJKLMN), vget_low_s16(vk24xGHIJKLMN));
525       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi24xGHIJKLMN), vget_high_s16(vk24xGHIJKLMN));
526 
527       float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
528       float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
529       float32x4_t vfpacc89AB = vcvtq_f32_s32(vacc89AB);
530       float32x4_t vfpaccCDEF = vcvtq_f32_s32(vaccCDEF);
531       float32x4_t vfpaccGHIJ = vcvtq_f32_s32(vaccGHIJ);
532       float32x4_t vfpaccKLMN = vcvtq_f32_s32(vaccKLMN);
533 
534       vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
535       vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
536       vfpacc89AB = vmulq_f32(vfpacc89AB, vscale);
537       vfpaccCDEF = vmulq_f32(vfpaccCDEF, vscale);
538       vfpaccGHIJ = vmulq_f32(vfpaccGHIJ, vscale);
539       vfpaccKLMN = vmulq_f32(vfpaccKLMN, vscale);
540 
541       vacc0123 = vcvtnq_s32_f32(vfpacc0123);
542       vacc4567 = vcvtnq_s32_f32(vfpacc4567);
543       vacc89AB = vcvtnq_s32_f32(vfpacc89AB);
544       vaccCDEF = vcvtnq_s32_f32(vfpaccCDEF);
545       vaccGHIJ = vcvtnq_s32_f32(vfpaccGHIJ);
546       vaccKLMN = vcvtnq_s32_f32(vfpaccKLMN);
547 
548 #if XNN_ARCH_ARM64
549       int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
550       int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
551       int16x8_t vaccGHIJKLMN = vqmovn_high_s32(vqmovn_s32(vaccGHIJ), vaccKLMN);
552 
553       vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
554       vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
555       vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
556 
557       uint8x16_t vout0123456789ABCDEF = vqmovun_high_s16(vqmovun_s16(vacc01234567), vacc89ABCDEF);
558       uint8x8_t voutGHIJKLMN = vqmovun_s16(vaccGHIJKLMN);
559 #else  // !XNN_ARCH_ARM64
560       int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
561       int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
562       int16x8_t vaccGHIJKLMN = vcombine_s16(vqmovn_s32(vaccGHIJ), vqmovn_s32(vaccKLMN));
563 
564       vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
565       vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
566       vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
567 
568       uint8x16_t vout0123456789ABCDEF = vcombine_u8(vqmovun_s16(vacc01234567), vqmovun_s16(vacc89ABCDEF));
569       uint8x8_t voutGHIJKLMN = vqmovun_s16(vaccGHIJKLMN);
570 #endif  // !XNN_ARCH_ARM64
571 
572       vout0123456789ABCDEF = vmaxq_u8(vout0123456789ABCDEF, voutput_min);
573       voutGHIJKLMN = vmax_u8(voutGHIJKLMN, vget_low_u8(voutput_min));
574 
575       vout0123456789ABCDEF = vminq_u8(vout0123456789ABCDEF, voutput_max);
576       voutGHIJKLMN = vmin_u8(voutGHIJKLMN, vget_low_u8(voutput_max));
577 
578       vst1q_u8(output, vout0123456789ABCDEF); output += 16;
579       vst1_u8(output, voutGHIJKLMN); output += 8;
580     }
581     if XNN_UNLIKELY(c != 0) {
582       const uint8_t* k = (const uint8_t*) ((const int32_t*) w + 24);
583       do {
584         int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
585         int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
586 
587         const int16x8_t vi0x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
588         const int16x8_t vk0x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(k), vkernel_zero_point)); k += 8;
589 
590         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
591         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
592         const int16x8_t vi1x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
593         const int16x8_t vk1x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 16)), vkernel_zero_point));
594 
595         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
596         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
597         const int16x8_t vi2x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
598         const int16x8_t vk2x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 40)), vkernel_zero_point));
599 
600         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
601         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
602         const int16x8_t vi3x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
603         const int16x8_t vk3x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 64)), vkernel_zero_point));
604 
605         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
606         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
607         const int16x8_t vi4x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
608         const int16x8_t vk4x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 88)), vkernel_zero_point));
609 
610         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
611         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
612         const int16x8_t vi5x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
613         const int16x8_t vk5x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 112)), vkernel_zero_point));
614 
615         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
616         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
617         const int16x8_t vi6x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
618         const int16x8_t vk6x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 136)), vkernel_zero_point));
619 
620         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
621         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
622         const int16x8_t vi7x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
623         const int16x8_t vk7x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 160)), vkernel_zero_point));
624 
625         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
626         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
627         const int16x8_t vi8x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
628         const int16x8_t vk8x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 184)), vkernel_zero_point));
629 
630         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
631         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
632         const int16x8_t vi9x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
633         const int16x8_t vk9x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 208)), vkernel_zero_point));
634 
635         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi9x01234567), vget_low_s16(vk9x01234567));
636         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi9x01234567), vget_high_s16(vk9x01234567));
637         const int16x8_t vi10x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
638         const int16x8_t vk10x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 232)), vkernel_zero_point));
639 
640         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi10x01234567), vget_low_s16(vk10x01234567));
641         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi10x01234567), vget_high_s16(vk10x01234567));
642         const int16x8_t vi11x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
643         const int16x8_t vk11x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 256)), vkernel_zero_point));
644 
645         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi11x01234567), vget_low_s16(vk11x01234567));
646         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi11x01234567), vget_high_s16(vk11x01234567));
647         const int16x8_t vi12x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
648         const int16x8_t vk12x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 280)), vkernel_zero_point));
649 
650         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi12x01234567), vget_low_s16(vk12x01234567));
651         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi12x01234567), vget_high_s16(vk12x01234567));
652         const int16x8_t vi13x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
653         const int16x8_t vk13x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 304)), vkernel_zero_point));
654 
655         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi13x01234567), vget_low_s16(vk13x01234567));
656         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi13x01234567), vget_high_s16(vk13x01234567));
657         const int16x8_t vi14x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
658         const int16x8_t vk14x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 328)), vkernel_zero_point));
659 
660         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi14x01234567), vget_low_s16(vk14x01234567));
661         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi14x01234567), vget_high_s16(vk14x01234567));
662         const int16x8_t vi15x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
663         const int16x8_t vk15x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 352)), vkernel_zero_point));
664 
665         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi15x01234567), vget_low_s16(vk15x01234567));
666         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi15x01234567), vget_high_s16(vk15x01234567));
667         const int16x8_t vi16x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
668         const int16x8_t vk16x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 376)), vkernel_zero_point));
669 
670         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi16x01234567), vget_low_s16(vk16x01234567));
671         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi16x01234567), vget_high_s16(vk16x01234567));
672         const int16x8_t vi17x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
673         const int16x8_t vk17x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 400)), vkernel_zero_point));
674 
675         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi17x01234567), vget_low_s16(vk17x01234567));
676         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi17x01234567), vget_high_s16(vk17x01234567));
677         const int16x8_t vi18x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
678         const int16x8_t vk18x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 424)), vkernel_zero_point));
679 
680         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi18x01234567), vget_low_s16(vk18x01234567));
681         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi18x01234567), vget_high_s16(vk18x01234567));
682         const int16x8_t vi19x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
683         const int16x8_t vk19x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 448)), vkernel_zero_point));
684 
685         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi19x01234567), vget_low_s16(vk19x01234567));
686         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi19x01234567), vget_high_s16(vk19x01234567));
687         const int16x8_t vi20x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
688         const int16x8_t vk20x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 472)), vkernel_zero_point));
689 
690         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi20x01234567), vget_low_s16(vk20x01234567));
691         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi20x01234567), vget_high_s16(vk20x01234567));
692         const int16x8_t vi21x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
693         const int16x8_t vk21x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 496)), vkernel_zero_point));
694 
695         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi21x01234567), vget_low_s16(vk21x01234567));
696         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi21x01234567), vget_high_s16(vk21x01234567));
697         const int16x8_t vi22x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
698         const int16x8_t vk22x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 520)), vkernel_zero_point));
699 
700         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi22x01234567), vget_low_s16(vk22x01234567));
701         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi22x01234567), vget_high_s16(vk22x01234567));
702         const int16x8_t vi23x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
703         const int16x8_t vk23x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 544)), vkernel_zero_point));
704 
705         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi23x01234567), vget_low_s16(vk23x01234567));
706         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi23x01234567), vget_high_s16(vk23x01234567));
707         const int16x8_t vi24x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
708         const int16x8_t vk24x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 568)), vkernel_zero_point));
709 
710         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi24x01234567), vget_low_s16(vk24x01234567));
711         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi24x01234567), vget_high_s16(vk24x01234567));
712 
713         float32x4_t vfpacc0123 = vcvtq_f32_s32(vacc0123);
714         float32x4_t vfpacc4567 = vcvtq_f32_s32(vacc4567);
715 
716         vfpacc0123 = vmulq_f32(vfpacc0123, vscale);
717         vfpacc4567 = vmulq_f32(vfpacc4567, vscale);
718 
719         vacc0123 = vcvtnq_s32_f32(vfpacc0123);
720         vacc4567 = vcvtnq_s32_f32(vfpacc4567);
721 
722 #if XNN_ARCH_ARM64
723         int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
724 #else
725         int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
726 #endif
727         vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
728 
729         uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
730         vout01234567 = vmax_u8(vout01234567, vget_low_u8(voutput_min));
731         vout01234567 = vmin_u8(vout01234567, vget_low_u8(voutput_max));
732 
733         if XNN_LIKELY(c >= 8) {
734           vst1_u8(output, vout01234567); output += 8;
735           c -= 8;
736         } else {
737           if (c & 4) {
738             vst1_lane_u32((void*) output, vreinterpret_u32_u8(vout01234567), 0); output += 4;
739             vout01234567 = vext_u8(vout01234567, vout01234567, 4);
740           }
741           if (c & 2) {
742             vst1_lane_u16((void*) output, vreinterpret_u16_u8(vout01234567), 0); output += 2;
743             vout01234567 = vext_u8(vout01234567, vout01234567, 2);
744           }
745           if (c & 1) {
746             vst1_lane_u8(output, vout01234567, 0); output += 1;
747           }
748           c = 0;
749         }
750       } while (c != 0);
751     }
752 
753     output = (uint8_t*) ((uintptr_t) output + output_increment);
754   } while (--output_width != 0);
755 }
756