xref: /aosp_15_r20/external/XNNPACK/src/qu8-dwconv/gen/up24x25-minmax-rndnu-neon-mul16.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-dwconv/unipass-neon-mul16.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/dwconv.h>
15 
16 
xnn_qu8_dwconv_minmax_rndnu_ukernel_up24x25__neon_mul16(size_t channels,size_t output_width,const uint8_t ** input,const void * weights,uint8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const uint8_t * zero,const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qu8_dwconv_minmax_rndnu_ukernel_up24x25__neon_mul16(
18     size_t channels,
19     size_t output_width,
20     const uint8_t** input,
21     const void* weights,
22     uint8_t* output,
23     size_t input_stride,
24     size_t output_increment,
25     size_t input_offset,
26     const uint8_t* zero,
27     const union xnn_qu8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(channels != 0);
30   assert(output_width != 0);
31 
32   const uint8x8_t vkernel_zero_point = vld1_dup_u8(params->rndnu_neon.kernel_zero_point);
33   const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
34   const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
35   const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
36   const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
37   const uint8x16_t voutput_min = vld1q_dup_u8(&params->rndnu_neon.output_min);
38   const uint8x16_t voutput_max = vld1q_dup_u8(&params->rndnu_neon.output_max);
39   do {
40     const uint8_t* i0 = input[0];
41     assert(i0 != NULL);
42     if XNN_UNPREDICTABLE(i0 != zero) {
43       i0 = (const uint8_t*) ((uintptr_t) i0 + input_offset);
44     }
45     const uint8_t* i1 = input[1];
46     assert(i1 != NULL);
47     if XNN_UNPREDICTABLE(i1 != zero) {
48       i1 = (const uint8_t*) ((uintptr_t) i1 + input_offset);
49     }
50     const uint8_t* i2 = input[2];
51     assert(i2 != NULL);
52     if XNN_UNPREDICTABLE(i2 != zero) {
53       i2 = (const uint8_t*) ((uintptr_t) i2 + input_offset);
54     }
55     const uint8_t* i3 = input[3];
56     assert(i3 != NULL);
57     if XNN_UNPREDICTABLE(i3 != zero) {
58       i3 = (const uint8_t*) ((uintptr_t) i3 + input_offset);
59     }
60     const uint8_t* i4 = input[4];
61     assert(i4 != NULL);
62     if XNN_UNPREDICTABLE(i4 != zero) {
63       i4 = (const uint8_t*) ((uintptr_t) i4 + input_offset);
64     }
65     const uint8_t* i5 = input[5];
66     assert(i5 != NULL);
67     if XNN_UNPREDICTABLE(i5 != zero) {
68       i5 = (const uint8_t*) ((uintptr_t) i5 + input_offset);
69     }
70     const uint8_t* i6 = input[6];
71     assert(i6 != NULL);
72     if XNN_UNPREDICTABLE(i6 != zero) {
73       i6 = (const uint8_t*) ((uintptr_t) i6 + input_offset);
74     }
75     const uint8_t* i7 = input[7];
76     assert(i7 != NULL);
77     if XNN_UNPREDICTABLE(i7 != zero) {
78       i7 = (const uint8_t*) ((uintptr_t) i7 + input_offset);
79     }
80     const uint8_t* i8 = input[8];
81     assert(i8 != NULL);
82     if XNN_UNPREDICTABLE(i8 != zero) {
83       i8 = (const uint8_t*) ((uintptr_t) i8 + input_offset);
84     }
85     const uint8_t* i9 = input[9];
86     assert(i9 != NULL);
87     if XNN_UNPREDICTABLE(i9 != zero) {
88       i9 = (const uint8_t*) ((uintptr_t) i9 + input_offset);
89     }
90     const uint8_t* i10 = input[10];
91     assert(i10 != NULL);
92     if XNN_UNPREDICTABLE(i10 != zero) {
93       i10 = (const uint8_t*) ((uintptr_t) i10 + input_offset);
94     }
95     const uint8_t* i11 = input[11];
96     assert(i11 != NULL);
97     if XNN_UNPREDICTABLE(i11 != zero) {
98       i11 = (const uint8_t*) ((uintptr_t) i11 + input_offset);
99     }
100     const uint8_t* i12 = input[12];
101     assert(i12 != NULL);
102     if XNN_UNPREDICTABLE(i12 != zero) {
103       i12 = (const uint8_t*) ((uintptr_t) i12 + input_offset);
104     }
105     const uint8_t* i13 = input[13];
106     assert(i13 != NULL);
107     if XNN_UNPREDICTABLE(i13 != zero) {
108       i13 = (const uint8_t*) ((uintptr_t) i13 + input_offset);
109     }
110     const uint8_t* i14 = input[14];
111     assert(i14 != NULL);
112     if XNN_UNPREDICTABLE(i14 != zero) {
113       i14 = (const uint8_t*) ((uintptr_t) i14 + input_offset);
114     }
115     const uint8_t* i15 = input[15];
116     assert(i15 != NULL);
117     if XNN_UNPREDICTABLE(i15 != zero) {
118       i15 = (const uint8_t*) ((uintptr_t) i15 + input_offset);
119     }
120     const uint8_t* i16 = input[16];
121     assert(i16 != NULL);
122     if XNN_UNPREDICTABLE(i16 != zero) {
123       i16 = (const uint8_t*) ((uintptr_t) i16 + input_offset);
124     }
125     const uint8_t* i17 = input[17];
126     assert(i17 != NULL);
127     if XNN_UNPREDICTABLE(i17 != zero) {
128       i17 = (const uint8_t*) ((uintptr_t) i17 + input_offset);
129     }
130     const uint8_t* i18 = input[18];
131     assert(i18 != NULL);
132     if XNN_UNPREDICTABLE(i18 != zero) {
133       i18 = (const uint8_t*) ((uintptr_t) i18 + input_offset);
134     }
135     const uint8_t* i19 = input[19];
136     assert(i19 != NULL);
137     if XNN_UNPREDICTABLE(i19 != zero) {
138       i19 = (const uint8_t*) ((uintptr_t) i19 + input_offset);
139     }
140     const uint8_t* i20 = input[20];
141     assert(i20 != NULL);
142     if XNN_UNPREDICTABLE(i20 != zero) {
143       i20 = (const uint8_t*) ((uintptr_t) i20 + input_offset);
144     }
145     const uint8_t* i21 = input[21];
146     assert(i21 != NULL);
147     if XNN_UNPREDICTABLE(i21 != zero) {
148       i21 = (const uint8_t*) ((uintptr_t) i21 + input_offset);
149     }
150     const uint8_t* i22 = input[22];
151     assert(i22 != NULL);
152     if XNN_UNPREDICTABLE(i22 != zero) {
153       i22 = (const uint8_t*) ((uintptr_t) i22 + input_offset);
154     }
155     const uint8_t* i23 = input[23];
156     assert(i23 != NULL);
157     if XNN_UNPREDICTABLE(i23 != zero) {
158       i23 = (const uint8_t*) ((uintptr_t) i23 + input_offset);
159     }
160     const uint8_t* i24 = input[24];
161     assert(i24 != NULL);
162     if XNN_UNPREDICTABLE(i24 != zero) {
163       i24 = (const uint8_t*) ((uintptr_t) i24 + input_offset);
164     }
165     input = (const uint8_t**) ((uintptr_t) input + input_stride);
166 
167     size_t c = channels;
168     const void* w = weights;
169     for (; c >= 24; c -= 24) {
170       int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
171       int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
172       int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
173       int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
174       int32x4_t vaccGHIJ = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
175       int32x4_t vaccKLMN = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
176 
177 
178       const int16x8_t vi0x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
179       const int16x8_t vk0x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
180       const int16x8_t vi0x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
181       const int16x8_t vk0x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
182       const int16x8_t vi0xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
183       const int16x8_t vk0xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
184 
185       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
186       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
187       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi0x89ABCDEF), vget_low_s16(vk0x89ABCDEF));
188       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi0x89ABCDEF), vget_high_s16(vk0x89ABCDEF));
189       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi0xGHIJKLMN), vget_low_s16(vk0xGHIJKLMN));
190       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi0xGHIJKLMN), vget_high_s16(vk0xGHIJKLMN));
191 
192       const int16x8_t vi1x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
193       const int16x8_t vk1x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
194       const int16x8_t vi1x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
195       const int16x8_t vk1x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
196       const int16x8_t vi1xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
197       const int16x8_t vk1xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
198 
199       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
200       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
201       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi1x89ABCDEF), vget_low_s16(vk1x89ABCDEF));
202       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi1x89ABCDEF), vget_high_s16(vk1x89ABCDEF));
203       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi1xGHIJKLMN), vget_low_s16(vk1xGHIJKLMN));
204       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi1xGHIJKLMN), vget_high_s16(vk1xGHIJKLMN));
205 
206       const int16x8_t vi2x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
207       const int16x8_t vk2x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
208       const int16x8_t vi2x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
209       const int16x8_t vk2x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
210       const int16x8_t vi2xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
211       const int16x8_t vk2xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
212 
213       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
214       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
215       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi2x89ABCDEF), vget_low_s16(vk2x89ABCDEF));
216       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi2x89ABCDEF), vget_high_s16(vk2x89ABCDEF));
217       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi2xGHIJKLMN), vget_low_s16(vk2xGHIJKLMN));
218       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi2xGHIJKLMN), vget_high_s16(vk2xGHIJKLMN));
219 
220       const int16x8_t vi3x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
221       const int16x8_t vk3x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
222       const int16x8_t vi3x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
223       const int16x8_t vk3x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
224       const int16x8_t vi3xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
225       const int16x8_t vk3xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
226 
227       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
228       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
229       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi3x89ABCDEF), vget_low_s16(vk3x89ABCDEF));
230       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi3x89ABCDEF), vget_high_s16(vk3x89ABCDEF));
231       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi3xGHIJKLMN), vget_low_s16(vk3xGHIJKLMN));
232       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi3xGHIJKLMN), vget_high_s16(vk3xGHIJKLMN));
233 
234       const int16x8_t vi4x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
235       const int16x8_t vk4x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
236       const int16x8_t vi4x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
237       const int16x8_t vk4x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
238       const int16x8_t vi4xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
239       const int16x8_t vk4xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
240 
241       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
242       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
243       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi4x89ABCDEF), vget_low_s16(vk4x89ABCDEF));
244       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi4x89ABCDEF), vget_high_s16(vk4x89ABCDEF));
245       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi4xGHIJKLMN), vget_low_s16(vk4xGHIJKLMN));
246       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi4xGHIJKLMN), vget_high_s16(vk4xGHIJKLMN));
247 
248       const int16x8_t vi5x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
249       const int16x8_t vk5x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
250       const int16x8_t vi5x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
251       const int16x8_t vk5x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
252       const int16x8_t vi5xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
253       const int16x8_t vk5xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
254 
255       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
256       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
257       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi5x89ABCDEF), vget_low_s16(vk5x89ABCDEF));
258       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi5x89ABCDEF), vget_high_s16(vk5x89ABCDEF));
259       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi5xGHIJKLMN), vget_low_s16(vk5xGHIJKLMN));
260       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi5xGHIJKLMN), vget_high_s16(vk5xGHIJKLMN));
261 
262       const int16x8_t vi6x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
263       const int16x8_t vk6x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
264       const int16x8_t vi6x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
265       const int16x8_t vk6x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
266       const int16x8_t vi6xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
267       const int16x8_t vk6xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
268 
269       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
270       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
271       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi6x89ABCDEF), vget_low_s16(vk6x89ABCDEF));
272       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi6x89ABCDEF), vget_high_s16(vk6x89ABCDEF));
273       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi6xGHIJKLMN), vget_low_s16(vk6xGHIJKLMN));
274       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi6xGHIJKLMN), vget_high_s16(vk6xGHIJKLMN));
275 
276       const int16x8_t vi7x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
277       const int16x8_t vk7x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
278       const int16x8_t vi7x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
279       const int16x8_t vk7x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
280       const int16x8_t vi7xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
281       const int16x8_t vk7xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
282 
283       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
284       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
285       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi7x89ABCDEF), vget_low_s16(vk7x89ABCDEF));
286       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi7x89ABCDEF), vget_high_s16(vk7x89ABCDEF));
287       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi7xGHIJKLMN), vget_low_s16(vk7xGHIJKLMN));
288       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi7xGHIJKLMN), vget_high_s16(vk7xGHIJKLMN));
289 
290       const int16x8_t vi8x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
291       const int16x8_t vk8x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
292       const int16x8_t vi8x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
293       const int16x8_t vk8x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
294       const int16x8_t vi8xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
295       const int16x8_t vk8xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
296 
297       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
298       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
299       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi8x89ABCDEF), vget_low_s16(vk8x89ABCDEF));
300       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi8x89ABCDEF), vget_high_s16(vk8x89ABCDEF));
301       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi8xGHIJKLMN), vget_low_s16(vk8xGHIJKLMN));
302       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi8xGHIJKLMN), vget_high_s16(vk8xGHIJKLMN));
303 
304       const int16x8_t vi9x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
305       const int16x8_t vk9x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
306       const int16x8_t vi9x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
307       const int16x8_t vk9x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
308       const int16x8_t vi9xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
309       const int16x8_t vk9xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
310 
311       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi9x01234567), vget_low_s16(vk9x01234567));
312       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi9x01234567), vget_high_s16(vk9x01234567));
313       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi9x89ABCDEF), vget_low_s16(vk9x89ABCDEF));
314       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi9x89ABCDEF), vget_high_s16(vk9x89ABCDEF));
315       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi9xGHIJKLMN), vget_low_s16(vk9xGHIJKLMN));
316       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi9xGHIJKLMN), vget_high_s16(vk9xGHIJKLMN));
317 
318       const int16x8_t vi10x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
319       const int16x8_t vk10x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
320       const int16x8_t vi10x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
321       const int16x8_t vk10x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
322       const int16x8_t vi10xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
323       const int16x8_t vk10xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
324 
325       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi10x01234567), vget_low_s16(vk10x01234567));
326       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi10x01234567), vget_high_s16(vk10x01234567));
327       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi10x89ABCDEF), vget_low_s16(vk10x89ABCDEF));
328       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi10x89ABCDEF), vget_high_s16(vk10x89ABCDEF));
329       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi10xGHIJKLMN), vget_low_s16(vk10xGHIJKLMN));
330       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi10xGHIJKLMN), vget_high_s16(vk10xGHIJKLMN));
331 
332       const int16x8_t vi11x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
333       const int16x8_t vk11x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
334       const int16x8_t vi11x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
335       const int16x8_t vk11x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
336       const int16x8_t vi11xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
337       const int16x8_t vk11xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
338 
339       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi11x01234567), vget_low_s16(vk11x01234567));
340       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi11x01234567), vget_high_s16(vk11x01234567));
341       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi11x89ABCDEF), vget_low_s16(vk11x89ABCDEF));
342       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi11x89ABCDEF), vget_high_s16(vk11x89ABCDEF));
343       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi11xGHIJKLMN), vget_low_s16(vk11xGHIJKLMN));
344       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi11xGHIJKLMN), vget_high_s16(vk11xGHIJKLMN));
345 
346       const int16x8_t vi12x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
347       const int16x8_t vk12x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
348       const int16x8_t vi12x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
349       const int16x8_t vk12x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
350       const int16x8_t vi12xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
351       const int16x8_t vk12xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
352 
353       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi12x01234567), vget_low_s16(vk12x01234567));
354       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi12x01234567), vget_high_s16(vk12x01234567));
355       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi12x89ABCDEF), vget_low_s16(vk12x89ABCDEF));
356       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi12x89ABCDEF), vget_high_s16(vk12x89ABCDEF));
357       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi12xGHIJKLMN), vget_low_s16(vk12xGHIJKLMN));
358       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi12xGHIJKLMN), vget_high_s16(vk12xGHIJKLMN));
359 
360       const int16x8_t vi13x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
361       const int16x8_t vk13x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
362       const int16x8_t vi13x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
363       const int16x8_t vk13x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
364       const int16x8_t vi13xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
365       const int16x8_t vk13xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
366 
367       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi13x01234567), vget_low_s16(vk13x01234567));
368       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi13x01234567), vget_high_s16(vk13x01234567));
369       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi13x89ABCDEF), vget_low_s16(vk13x89ABCDEF));
370       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi13x89ABCDEF), vget_high_s16(vk13x89ABCDEF));
371       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi13xGHIJKLMN), vget_low_s16(vk13xGHIJKLMN));
372       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi13xGHIJKLMN), vget_high_s16(vk13xGHIJKLMN));
373 
374       const int16x8_t vi14x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
375       const int16x8_t vk14x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
376       const int16x8_t vi14x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
377       const int16x8_t vk14x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
378       const int16x8_t vi14xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
379       const int16x8_t vk14xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
380 
381       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi14x01234567), vget_low_s16(vk14x01234567));
382       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi14x01234567), vget_high_s16(vk14x01234567));
383       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi14x89ABCDEF), vget_low_s16(vk14x89ABCDEF));
384       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi14x89ABCDEF), vget_high_s16(vk14x89ABCDEF));
385       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi14xGHIJKLMN), vget_low_s16(vk14xGHIJKLMN));
386       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi14xGHIJKLMN), vget_high_s16(vk14xGHIJKLMN));
387 
388       const int16x8_t vi15x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
389       const int16x8_t vk15x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
390       const int16x8_t vi15x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
391       const int16x8_t vk15x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
392       const int16x8_t vi15xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
393       const int16x8_t vk15xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
394 
395       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi15x01234567), vget_low_s16(vk15x01234567));
396       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi15x01234567), vget_high_s16(vk15x01234567));
397       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi15x89ABCDEF), vget_low_s16(vk15x89ABCDEF));
398       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi15x89ABCDEF), vget_high_s16(vk15x89ABCDEF));
399       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi15xGHIJKLMN), vget_low_s16(vk15xGHIJKLMN));
400       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi15xGHIJKLMN), vget_high_s16(vk15xGHIJKLMN));
401 
402       const int16x8_t vi16x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
403       const int16x8_t vk16x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
404       const int16x8_t vi16x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
405       const int16x8_t vk16x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
406       const int16x8_t vi16xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
407       const int16x8_t vk16xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
408 
409       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi16x01234567), vget_low_s16(vk16x01234567));
410       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi16x01234567), vget_high_s16(vk16x01234567));
411       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi16x89ABCDEF), vget_low_s16(vk16x89ABCDEF));
412       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi16x89ABCDEF), vget_high_s16(vk16x89ABCDEF));
413       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi16xGHIJKLMN), vget_low_s16(vk16xGHIJKLMN));
414       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi16xGHIJKLMN), vget_high_s16(vk16xGHIJKLMN));
415 
416       const int16x8_t vi17x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
417       const int16x8_t vk17x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
418       const int16x8_t vi17x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
419       const int16x8_t vk17x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
420       const int16x8_t vi17xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
421       const int16x8_t vk17xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
422 
423       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi17x01234567), vget_low_s16(vk17x01234567));
424       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi17x01234567), vget_high_s16(vk17x01234567));
425       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi17x89ABCDEF), vget_low_s16(vk17x89ABCDEF));
426       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi17x89ABCDEF), vget_high_s16(vk17x89ABCDEF));
427       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi17xGHIJKLMN), vget_low_s16(vk17xGHIJKLMN));
428       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi17xGHIJKLMN), vget_high_s16(vk17xGHIJKLMN));
429 
430       const int16x8_t vi18x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
431       const int16x8_t vk18x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
432       const int16x8_t vi18x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
433       const int16x8_t vk18x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
434       const int16x8_t vi18xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
435       const int16x8_t vk18xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
436 
437       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi18x01234567), vget_low_s16(vk18x01234567));
438       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi18x01234567), vget_high_s16(vk18x01234567));
439       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi18x89ABCDEF), vget_low_s16(vk18x89ABCDEF));
440       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi18x89ABCDEF), vget_high_s16(vk18x89ABCDEF));
441       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi18xGHIJKLMN), vget_low_s16(vk18xGHIJKLMN));
442       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi18xGHIJKLMN), vget_high_s16(vk18xGHIJKLMN));
443 
444       const int16x8_t vi19x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
445       const int16x8_t vk19x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
446       const int16x8_t vi19x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
447       const int16x8_t vk19x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
448       const int16x8_t vi19xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
449       const int16x8_t vk19xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
450 
451       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi19x01234567), vget_low_s16(vk19x01234567));
452       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi19x01234567), vget_high_s16(vk19x01234567));
453       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi19x89ABCDEF), vget_low_s16(vk19x89ABCDEF));
454       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi19x89ABCDEF), vget_high_s16(vk19x89ABCDEF));
455       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi19xGHIJKLMN), vget_low_s16(vk19xGHIJKLMN));
456       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi19xGHIJKLMN), vget_high_s16(vk19xGHIJKLMN));
457 
458       const int16x8_t vi20x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
459       const int16x8_t vk20x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
460       const int16x8_t vi20x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
461       const int16x8_t vk20x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
462       const int16x8_t vi20xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
463       const int16x8_t vk20xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
464 
465       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi20x01234567), vget_low_s16(vk20x01234567));
466       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi20x01234567), vget_high_s16(vk20x01234567));
467       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi20x89ABCDEF), vget_low_s16(vk20x89ABCDEF));
468       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi20x89ABCDEF), vget_high_s16(vk20x89ABCDEF));
469       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi20xGHIJKLMN), vget_low_s16(vk20xGHIJKLMN));
470       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi20xGHIJKLMN), vget_high_s16(vk20xGHIJKLMN));
471 
472       const int16x8_t vi21x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
473       const int16x8_t vk21x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
474       const int16x8_t vi21x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
475       const int16x8_t vk21x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
476       const int16x8_t vi21xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
477       const int16x8_t vk21xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
478 
479       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi21x01234567), vget_low_s16(vk21x01234567));
480       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi21x01234567), vget_high_s16(vk21x01234567));
481       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi21x89ABCDEF), vget_low_s16(vk21x89ABCDEF));
482       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi21x89ABCDEF), vget_high_s16(vk21x89ABCDEF));
483       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi21xGHIJKLMN), vget_low_s16(vk21xGHIJKLMN));
484       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi21xGHIJKLMN), vget_high_s16(vk21xGHIJKLMN));
485 
486       const int16x8_t vi22x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
487       const int16x8_t vk22x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
488       const int16x8_t vi22x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
489       const int16x8_t vk22x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
490       const int16x8_t vi22xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
491       const int16x8_t vk22xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
492 
493       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi22x01234567), vget_low_s16(vk22x01234567));
494       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi22x01234567), vget_high_s16(vk22x01234567));
495       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi22x89ABCDEF), vget_low_s16(vk22x89ABCDEF));
496       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi22x89ABCDEF), vget_high_s16(vk22x89ABCDEF));
497       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi22xGHIJKLMN), vget_low_s16(vk22xGHIJKLMN));
498       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi22xGHIJKLMN), vget_high_s16(vk22xGHIJKLMN));
499 
500       const int16x8_t vi23x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
501       const int16x8_t vk23x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
502       const int16x8_t vi23x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
503       const int16x8_t vk23x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
504       const int16x8_t vi23xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
505       const int16x8_t vk23xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
506 
507       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi23x01234567), vget_low_s16(vk23x01234567));
508       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi23x01234567), vget_high_s16(vk23x01234567));
509       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi23x89ABCDEF), vget_low_s16(vk23x89ABCDEF));
510       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi23x89ABCDEF), vget_high_s16(vk23x89ABCDEF));
511       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi23xGHIJKLMN), vget_low_s16(vk23xGHIJKLMN));
512       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi23xGHIJKLMN), vget_high_s16(vk23xGHIJKLMN));
513 
514       const int16x8_t vi24x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
515       const int16x8_t vk24x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
516       const int16x8_t vi24x89ABCDEF = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
517       const int16x8_t vk24x89ABCDEF = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
518       const int16x8_t vi24xGHIJKLMN = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
519       const int16x8_t vk24xGHIJKLMN = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(w), vkernel_zero_point)); w = (const void*) ((const uint8_t*) w + 8);
520 
521       vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi24x01234567), vget_low_s16(vk24x01234567));
522       vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi24x01234567), vget_high_s16(vk24x01234567));
523       vacc89AB = vmlal_s16(vacc89AB, vget_low_s16(vi24x89ABCDEF), vget_low_s16(vk24x89ABCDEF));
524       vaccCDEF = vmlal_s16(vaccCDEF, vget_high_s16(vi24x89ABCDEF), vget_high_s16(vk24x89ABCDEF));
525       vaccGHIJ = vmlal_s16(vaccGHIJ, vget_low_s16(vi24xGHIJKLMN), vget_low_s16(vk24xGHIJKLMN));
526       vaccKLMN = vmlal_s16(vaccKLMN, vget_high_s16(vi24xGHIJKLMN), vget_high_s16(vk24xGHIJKLMN));
527 
528       vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
529       vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
530       vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
531       vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
532       vaccGHIJ = vqshlq_s32(vaccGHIJ, vright_pre_shift);
533       vaccKLMN = vqshlq_s32(vaccKLMN, vright_pre_shift);
534 
535       vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
536       vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
537       vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
538       vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
539       vaccGHIJ = vqdmulhq_s32(vaccGHIJ, vmultiplier);
540       vaccKLMN = vqdmulhq_s32(vaccKLMN, vmultiplier);
541 
542       vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
543       vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
544       vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
545       vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
546       vaccGHIJ = vrshlq_s32(vaccGHIJ, vright_post_shift);
547       vaccKLMN = vrshlq_s32(vaccKLMN, vright_post_shift);
548 
549 #if XNN_ARCH_ARM64
550       int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
551       int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
552       int16x8_t vaccGHIJKLMN = vqmovn_high_s32(vqmovn_s32(vaccGHIJ), vaccKLMN);
553 
554       vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
555       vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
556       vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
557 
558       uint8x16_t vout0123456789ABCDEF = vqmovun_high_s16(vqmovun_s16(vacc01234567), vacc89ABCDEF);
559       uint8x8_t voutGHIJKLMN = vqmovun_s16(vaccGHIJKLMN);
560 #else  // !XNN_ARCH_ARM64
561       int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
562       int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
563       int16x8_t vaccGHIJKLMN = vcombine_s16(vqmovn_s32(vaccGHIJ), vqmovn_s32(vaccKLMN));
564 
565       vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
566       vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
567       vaccGHIJKLMN = vqaddq_s16(vaccGHIJKLMN, voutput_zero_point);
568 
569       uint8x16_t vout0123456789ABCDEF = vcombine_u8(vqmovun_s16(vacc01234567), vqmovun_s16(vacc89ABCDEF));
570       uint8x8_t voutGHIJKLMN = vqmovun_s16(vaccGHIJKLMN);
571 #endif  // !XNN_ARCH_ARM64
572 
573       vout0123456789ABCDEF = vmaxq_u8(vout0123456789ABCDEF, voutput_min);
574       voutGHIJKLMN = vmax_u8(voutGHIJKLMN, vget_low_u8(voutput_min));
575 
576       vout0123456789ABCDEF = vminq_u8(vout0123456789ABCDEF, voutput_max);
577       voutGHIJKLMN = vmin_u8(voutGHIJKLMN, vget_low_u8(voutput_max));
578 
579       vst1q_u8(output, vout0123456789ABCDEF); output += 16;
580       vst1_u8(output, voutGHIJKLMN); output += 8;
581     }
582     if XNN_UNLIKELY(c != 0) {
583       const uint8_t* k = (const uint8_t*) ((const int32_t*) w + 24);
584       do {
585         int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
586         int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
587 
588         const int16x8_t vi0x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i0))); i0 += 8;
589         const int16x8_t vk0x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8(k), vkernel_zero_point)); k += 8;
590 
591         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi0x01234567), vget_low_s16(vk0x01234567));
592         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi0x01234567), vget_high_s16(vk0x01234567));
593         const int16x8_t vi1x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i1))); i1 += 8;
594         const int16x8_t vk1x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 16)), vkernel_zero_point));
595 
596         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi1x01234567), vget_low_s16(vk1x01234567));
597         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi1x01234567), vget_high_s16(vk1x01234567));
598         const int16x8_t vi2x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i2))); i2 += 8;
599         const int16x8_t vk2x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 40)), vkernel_zero_point));
600 
601         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi2x01234567), vget_low_s16(vk2x01234567));
602         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi2x01234567), vget_high_s16(vk2x01234567));
603         const int16x8_t vi3x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i3))); i3 += 8;
604         const int16x8_t vk3x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 64)), vkernel_zero_point));
605 
606         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi3x01234567), vget_low_s16(vk3x01234567));
607         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi3x01234567), vget_high_s16(vk3x01234567));
608         const int16x8_t vi4x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i4))); i4 += 8;
609         const int16x8_t vk4x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 88)), vkernel_zero_point));
610 
611         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi4x01234567), vget_low_s16(vk4x01234567));
612         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi4x01234567), vget_high_s16(vk4x01234567));
613         const int16x8_t vi5x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i5))); i5 += 8;
614         const int16x8_t vk5x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 112)), vkernel_zero_point));
615 
616         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi5x01234567), vget_low_s16(vk5x01234567));
617         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi5x01234567), vget_high_s16(vk5x01234567));
618         const int16x8_t vi6x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i6))); i6 += 8;
619         const int16x8_t vk6x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 136)), vkernel_zero_point));
620 
621         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi6x01234567), vget_low_s16(vk6x01234567));
622         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi6x01234567), vget_high_s16(vk6x01234567));
623         const int16x8_t vi7x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i7))); i7 += 8;
624         const int16x8_t vk7x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 160)), vkernel_zero_point));
625 
626         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi7x01234567), vget_low_s16(vk7x01234567));
627         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi7x01234567), vget_high_s16(vk7x01234567));
628         const int16x8_t vi8x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i8))); i8 += 8;
629         const int16x8_t vk8x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 184)), vkernel_zero_point));
630 
631         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi8x01234567), vget_low_s16(vk8x01234567));
632         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi8x01234567), vget_high_s16(vk8x01234567));
633         const int16x8_t vi9x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i9))); i9 += 8;
634         const int16x8_t vk9x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 208)), vkernel_zero_point));
635 
636         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi9x01234567), vget_low_s16(vk9x01234567));
637         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi9x01234567), vget_high_s16(vk9x01234567));
638         const int16x8_t vi10x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i10))); i10 += 8;
639         const int16x8_t vk10x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 232)), vkernel_zero_point));
640 
641         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi10x01234567), vget_low_s16(vk10x01234567));
642         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi10x01234567), vget_high_s16(vk10x01234567));
643         const int16x8_t vi11x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i11))); i11 += 8;
644         const int16x8_t vk11x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 256)), vkernel_zero_point));
645 
646         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi11x01234567), vget_low_s16(vk11x01234567));
647         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi11x01234567), vget_high_s16(vk11x01234567));
648         const int16x8_t vi12x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i12))); i12 += 8;
649         const int16x8_t vk12x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 280)), vkernel_zero_point));
650 
651         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi12x01234567), vget_low_s16(vk12x01234567));
652         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi12x01234567), vget_high_s16(vk12x01234567));
653         const int16x8_t vi13x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i13))); i13 += 8;
654         const int16x8_t vk13x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 304)), vkernel_zero_point));
655 
656         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi13x01234567), vget_low_s16(vk13x01234567));
657         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi13x01234567), vget_high_s16(vk13x01234567));
658         const int16x8_t vi14x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i14))); i14 += 8;
659         const int16x8_t vk14x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 328)), vkernel_zero_point));
660 
661         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi14x01234567), vget_low_s16(vk14x01234567));
662         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi14x01234567), vget_high_s16(vk14x01234567));
663         const int16x8_t vi15x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i15))); i15 += 8;
664         const int16x8_t vk15x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 352)), vkernel_zero_point));
665 
666         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi15x01234567), vget_low_s16(vk15x01234567));
667         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi15x01234567), vget_high_s16(vk15x01234567));
668         const int16x8_t vi16x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i16))); i16 += 8;
669         const int16x8_t vk16x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 376)), vkernel_zero_point));
670 
671         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi16x01234567), vget_low_s16(vk16x01234567));
672         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi16x01234567), vget_high_s16(vk16x01234567));
673         const int16x8_t vi17x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i17))); i17 += 8;
674         const int16x8_t vk17x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 400)), vkernel_zero_point));
675 
676         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi17x01234567), vget_low_s16(vk17x01234567));
677         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi17x01234567), vget_high_s16(vk17x01234567));
678         const int16x8_t vi18x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i18))); i18 += 8;
679         const int16x8_t vk18x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 424)), vkernel_zero_point));
680 
681         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi18x01234567), vget_low_s16(vk18x01234567));
682         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi18x01234567), vget_high_s16(vk18x01234567));
683         const int16x8_t vi19x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i19))); i19 += 8;
684         const int16x8_t vk19x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 448)), vkernel_zero_point));
685 
686         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi19x01234567), vget_low_s16(vk19x01234567));
687         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi19x01234567), vget_high_s16(vk19x01234567));
688         const int16x8_t vi20x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i20))); i20 += 8;
689         const int16x8_t vk20x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 472)), vkernel_zero_point));
690 
691         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi20x01234567), vget_low_s16(vk20x01234567));
692         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi20x01234567), vget_high_s16(vk20x01234567));
693         const int16x8_t vi21x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i21))); i21 += 8;
694         const int16x8_t vk21x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 496)), vkernel_zero_point));
695 
696         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi21x01234567), vget_low_s16(vk21x01234567));
697         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi21x01234567), vget_high_s16(vk21x01234567));
698         const int16x8_t vi22x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i22))); i22 += 8;
699         const int16x8_t vk22x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 520)), vkernel_zero_point));
700 
701         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi22x01234567), vget_low_s16(vk22x01234567));
702         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi22x01234567), vget_high_s16(vk22x01234567));
703         const int16x8_t vi23x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i23))); i23 += 8;
704         const int16x8_t vk23x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 544)), vkernel_zero_point));
705 
706         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi23x01234567), vget_low_s16(vk23x01234567));
707         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi23x01234567), vget_high_s16(vk23x01234567));
708         const int16x8_t vi24x01234567 = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(i24))); i24 += 8;
709         const int16x8_t vk24x01234567 = vreinterpretq_s16_u16(vsubl_u8(vld1_u8((const void*) (k + 568)), vkernel_zero_point));
710 
711         vacc0123 = vmlal_s16(vacc0123, vget_low_s16(vi24x01234567), vget_low_s16(vk24x01234567));
712         vacc4567 = vmlal_s16(vacc4567, vget_high_s16(vi24x01234567), vget_high_s16(vk24x01234567));
713 
714         vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
715         vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
716 
717         vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
718         vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
719 
720         vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
721         vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
722 
723 #if XNN_ARCH_ARM64
724         int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
725 #else
726         int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
727 #endif
728         vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
729 
730         uint8x8_t vout01234567 = vqmovun_s16(vacc01234567);
731         vout01234567 = vmax_u8(vout01234567, vget_low_u8(voutput_min));
732         vout01234567 = vmin_u8(vout01234567, vget_low_u8(voutput_max));
733 
734         if XNN_LIKELY(c >= 8) {
735           vst1_u8(output, vout01234567); output += 8;
736           c -= 8;
737         } else {
738           if (c & 4) {
739             vst1_lane_u32((void*) output, vreinterpret_u32_u8(vout01234567), 0); output += 4;
740             vout01234567 = vext_u8(vout01234567, vout01234567, 4);
741           }
742           if (c & 2) {
743             vst1_lane_u16((void*) output, vreinterpret_u16_u8(vout01234567), 0); output += 2;
744             vout01234567 = vext_u8(vout01234567, vout01234567, 2);
745           }
746           if (c & 1) {
747             vst1_lane_u8(output, vout01234567, 0); output += 1;
748           }
749           c = 0;
750         }
751       } while (c != 0);
752     }
753 
754     output = (uint8_t*) ((uintptr_t) output + output_increment);
755   } while (--output_width != 0);
756 }
757