1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-dwconv/unipass-neon-mul8.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/dwconv.h>
15
16
xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mul8_ld64(size_t channels,size_t output_width,const int8_t ** input,const void * weights,int8_t * output,size_t input_stride,size_t output_increment,size_t input_offset,const int8_t * zero,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_dwconv_minmax_rndnu_ukernel_up16x25__neon_mul8_ld64(
18 size_t channels,
19 size_t output_width,
20 const int8_t** input,
21 const void* weights,
22 int8_t* output,
23 size_t input_stride,
24 size_t output_increment,
25 size_t input_offset,
26 const int8_t* zero,
27 const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29 assert(channels != 0);
30 assert(output_width != 0);
31
32 const int32x4_t vright_pre_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_pre_shift);
33 const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->rndnu_neon.multiplier);
34 const int32x4_t vright_post_shift = vld1q_dup_s32(¶ms->rndnu_neon.right_post_shift);
35 const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->rndnu_neon.output_zero_point);
36 const int8x16_t voutput_min = vld1q_dup_s8(¶ms->rndnu_neon.output_min);
37 const int8x16_t voutput_max = vld1q_dup_s8(¶ms->rndnu_neon.output_max);
38 do {
39 const int8_t* i0 = input[0];
40 assert(i0 != NULL);
41 if XNN_UNPREDICTABLE(i0 != zero) {
42 i0 = (const int8_t*) ((uintptr_t) i0 + input_offset);
43 }
44 const int8_t* i1 = input[1];
45 assert(i1 != NULL);
46 if XNN_UNPREDICTABLE(i1 != zero) {
47 i1 = (const int8_t*) ((uintptr_t) i1 + input_offset);
48 }
49 const int8_t* i2 = input[2];
50 assert(i2 != NULL);
51 if XNN_UNPREDICTABLE(i2 != zero) {
52 i2 = (const int8_t*) ((uintptr_t) i2 + input_offset);
53 }
54 const int8_t* i3 = input[3];
55 assert(i3 != NULL);
56 if XNN_UNPREDICTABLE(i3 != zero) {
57 i3 = (const int8_t*) ((uintptr_t) i3 + input_offset);
58 }
59 const int8_t* i4 = input[4];
60 assert(i4 != NULL);
61 if XNN_UNPREDICTABLE(i4 != zero) {
62 i4 = (const int8_t*) ((uintptr_t) i4 + input_offset);
63 }
64 const int8_t* i5 = input[5];
65 assert(i5 != NULL);
66 if XNN_UNPREDICTABLE(i5 != zero) {
67 i5 = (const int8_t*) ((uintptr_t) i5 + input_offset);
68 }
69 const int8_t* i6 = input[6];
70 assert(i6 != NULL);
71 if XNN_UNPREDICTABLE(i6 != zero) {
72 i6 = (const int8_t*) ((uintptr_t) i6 + input_offset);
73 }
74 const int8_t* i7 = input[7];
75 assert(i7 != NULL);
76 if XNN_UNPREDICTABLE(i7 != zero) {
77 i7 = (const int8_t*) ((uintptr_t) i7 + input_offset);
78 }
79 const int8_t* i8 = input[8];
80 assert(i8 != NULL);
81 if XNN_UNPREDICTABLE(i8 != zero) {
82 i8 = (const int8_t*) ((uintptr_t) i8 + input_offset);
83 }
84 const int8_t* i9 = input[9];
85 assert(i9 != NULL);
86 if XNN_UNPREDICTABLE(i9 != zero) {
87 i9 = (const int8_t*) ((uintptr_t) i9 + input_offset);
88 }
89 const int8_t* i10 = input[10];
90 assert(i10 != NULL);
91 if XNN_UNPREDICTABLE(i10 != zero) {
92 i10 = (const int8_t*) ((uintptr_t) i10 + input_offset);
93 }
94 const int8_t* i11 = input[11];
95 assert(i11 != NULL);
96 if XNN_UNPREDICTABLE(i11 != zero) {
97 i11 = (const int8_t*) ((uintptr_t) i11 + input_offset);
98 }
99 const int8_t* i12 = input[12];
100 assert(i12 != NULL);
101 if XNN_UNPREDICTABLE(i12 != zero) {
102 i12 = (const int8_t*) ((uintptr_t) i12 + input_offset);
103 }
104 const int8_t* i13 = input[13];
105 assert(i13 != NULL);
106 if XNN_UNPREDICTABLE(i13 != zero) {
107 i13 = (const int8_t*) ((uintptr_t) i13 + input_offset);
108 }
109 const int8_t* i14 = input[14];
110 assert(i14 != NULL);
111 if XNN_UNPREDICTABLE(i14 != zero) {
112 i14 = (const int8_t*) ((uintptr_t) i14 + input_offset);
113 }
114 const int8_t* i15 = input[15];
115 assert(i15 != NULL);
116 if XNN_UNPREDICTABLE(i15 != zero) {
117 i15 = (const int8_t*) ((uintptr_t) i15 + input_offset);
118 }
119 const int8_t* i16 = input[16];
120 assert(i16 != NULL);
121 if XNN_UNPREDICTABLE(i16 != zero) {
122 i16 = (const int8_t*) ((uintptr_t) i16 + input_offset);
123 }
124 const int8_t* i17 = input[17];
125 assert(i17 != NULL);
126 if XNN_UNPREDICTABLE(i17 != zero) {
127 i17 = (const int8_t*) ((uintptr_t) i17 + input_offset);
128 }
129 const int8_t* i18 = input[18];
130 assert(i18 != NULL);
131 if XNN_UNPREDICTABLE(i18 != zero) {
132 i18 = (const int8_t*) ((uintptr_t) i18 + input_offset);
133 }
134 const int8_t* i19 = input[19];
135 assert(i19 != NULL);
136 if XNN_UNPREDICTABLE(i19 != zero) {
137 i19 = (const int8_t*) ((uintptr_t) i19 + input_offset);
138 }
139 const int8_t* i20 = input[20];
140 assert(i20 != NULL);
141 if XNN_UNPREDICTABLE(i20 != zero) {
142 i20 = (const int8_t*) ((uintptr_t) i20 + input_offset);
143 }
144 const int8_t* i21 = input[21];
145 assert(i21 != NULL);
146 if XNN_UNPREDICTABLE(i21 != zero) {
147 i21 = (const int8_t*) ((uintptr_t) i21 + input_offset);
148 }
149 const int8_t* i22 = input[22];
150 assert(i22 != NULL);
151 if XNN_UNPREDICTABLE(i22 != zero) {
152 i22 = (const int8_t*) ((uintptr_t) i22 + input_offset);
153 }
154 const int8_t* i23 = input[23];
155 assert(i23 != NULL);
156 if XNN_UNPREDICTABLE(i23 != zero) {
157 i23 = (const int8_t*) ((uintptr_t) i23 + input_offset);
158 }
159 const int8_t* i24 = input[24];
160 assert(i24 != NULL);
161 if XNN_UNPREDICTABLE(i24 != zero) {
162 i24 = (const int8_t*) ((uintptr_t) i24 + input_offset);
163 }
164 input = (const int8_t**) ((uintptr_t) input + input_stride);
165
166 size_t c = channels;
167 const void* w = weights;
168 for (; c >= 16; c -= 16) {
169 int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
170 int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
171 int32x4_t vacc89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
172 int32x4_t vaccCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
173
174 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
175 const int8x8_t vk0x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
176 const int8x8_t vi0x89ABCDEF = vld1_s8(i0); i0 += 8;
177 const int8x8_t vk0x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
178
179 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
180 int16x8_t vprod89ABCDEF = vmull_s8(vi0x89ABCDEF, vk0x89ABCDEF);
181
182 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
183 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
184 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
185 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
186 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
187 const int8x8_t vk1x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
188 const int8x8_t vi1x89ABCDEF = vld1_s8(i1); i1 += 8;
189 const int8x8_t vk1x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
190
191 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
192 vprod89ABCDEF = vmull_s8(vi1x89ABCDEF, vk1x89ABCDEF);
193
194 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
195 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
196 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
197 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
198 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
199 const int8x8_t vk2x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
200 const int8x8_t vi2x89ABCDEF = vld1_s8(i2); i2 += 8;
201 const int8x8_t vk2x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
202
203 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
204 vprod89ABCDEF = vmull_s8(vi2x89ABCDEF, vk2x89ABCDEF);
205
206 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
207 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
208 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
209 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
210 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
211 const int8x8_t vk3x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
212 const int8x8_t vi3x89ABCDEF = vld1_s8(i3); i3 += 8;
213 const int8x8_t vk3x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
214
215 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
216 vprod89ABCDEF = vmull_s8(vi3x89ABCDEF, vk3x89ABCDEF);
217
218 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
219 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
220 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
221 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
222 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
223 const int8x8_t vk4x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
224 const int8x8_t vi4x89ABCDEF = vld1_s8(i4); i4 += 8;
225 const int8x8_t vk4x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
226
227 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
228 vprod89ABCDEF = vmull_s8(vi4x89ABCDEF, vk4x89ABCDEF);
229
230 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
231 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
232 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
233 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
234 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
235 const int8x8_t vk5x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
236 const int8x8_t vi5x89ABCDEF = vld1_s8(i5); i5 += 8;
237 const int8x8_t vk5x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
238
239 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
240 vprod89ABCDEF = vmull_s8(vi5x89ABCDEF, vk5x89ABCDEF);
241
242 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
243 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
244 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
245 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
246 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
247 const int8x8_t vk6x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
248 const int8x8_t vi6x89ABCDEF = vld1_s8(i6); i6 += 8;
249 const int8x8_t vk6x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
250
251 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
252 vprod89ABCDEF = vmull_s8(vi6x89ABCDEF, vk6x89ABCDEF);
253
254 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
255 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
256 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
257 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
258 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
259 const int8x8_t vk7x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
260 const int8x8_t vi7x89ABCDEF = vld1_s8(i7); i7 += 8;
261 const int8x8_t vk7x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
262
263 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
264 vprod89ABCDEF = vmull_s8(vi7x89ABCDEF, vk7x89ABCDEF);
265
266 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
267 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
268 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
269 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
270 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
271 const int8x8_t vk8x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
272 const int8x8_t vi8x89ABCDEF = vld1_s8(i8); i8 += 8;
273 const int8x8_t vk8x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
274
275 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
276 vprod89ABCDEF = vmull_s8(vi8x89ABCDEF, vk8x89ABCDEF);
277
278 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
279 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
280 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
281 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
282 const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
283 const int8x8_t vk9x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
284 const int8x8_t vi9x89ABCDEF = vld1_s8(i9); i9 += 8;
285 const int8x8_t vk9x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
286
287 vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
288 vprod89ABCDEF = vmull_s8(vi9x89ABCDEF, vk9x89ABCDEF);
289
290 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
291 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
292 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
293 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
294 const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
295 const int8x8_t vk10x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
296 const int8x8_t vi10x89ABCDEF = vld1_s8(i10); i10 += 8;
297 const int8x8_t vk10x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
298
299 vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
300 vprod89ABCDEF = vmull_s8(vi10x89ABCDEF, vk10x89ABCDEF);
301
302 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
303 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
304 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
305 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
306 const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
307 const int8x8_t vk11x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
308 const int8x8_t vi11x89ABCDEF = vld1_s8(i11); i11 += 8;
309 const int8x8_t vk11x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
310
311 vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
312 vprod89ABCDEF = vmull_s8(vi11x89ABCDEF, vk11x89ABCDEF);
313
314 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
315 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
316 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
317 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
318 const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
319 const int8x8_t vk12x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
320 const int8x8_t vi12x89ABCDEF = vld1_s8(i12); i12 += 8;
321 const int8x8_t vk12x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
322
323 vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
324 vprod89ABCDEF = vmull_s8(vi12x89ABCDEF, vk12x89ABCDEF);
325
326 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
327 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
328 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
329 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
330 const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
331 const int8x8_t vk13x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
332 const int8x8_t vi13x89ABCDEF = vld1_s8(i13); i13 += 8;
333 const int8x8_t vk13x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
334
335 vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
336 vprod89ABCDEF = vmull_s8(vi13x89ABCDEF, vk13x89ABCDEF);
337
338 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
339 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
340 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
341 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
342 const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
343 const int8x8_t vk14x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
344 const int8x8_t vi14x89ABCDEF = vld1_s8(i14); i14 += 8;
345 const int8x8_t vk14x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
346
347 vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
348 vprod89ABCDEF = vmull_s8(vi14x89ABCDEF, vk14x89ABCDEF);
349
350 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
351 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
352 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
353 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
354 const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
355 const int8x8_t vk15x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
356 const int8x8_t vi15x89ABCDEF = vld1_s8(i15); i15 += 8;
357 const int8x8_t vk15x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
358
359 vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
360 vprod89ABCDEF = vmull_s8(vi15x89ABCDEF, vk15x89ABCDEF);
361
362 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
363 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
364 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
365 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
366 const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
367 const int8x8_t vk16x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
368 const int8x8_t vi16x89ABCDEF = vld1_s8(i16); i16 += 8;
369 const int8x8_t vk16x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
370
371 vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
372 vprod89ABCDEF = vmull_s8(vi16x89ABCDEF, vk16x89ABCDEF);
373
374 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
375 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
376 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
377 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
378 const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
379 const int8x8_t vk17x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
380 const int8x8_t vi17x89ABCDEF = vld1_s8(i17); i17 += 8;
381 const int8x8_t vk17x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
382
383 vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
384 vprod89ABCDEF = vmull_s8(vi17x89ABCDEF, vk17x89ABCDEF);
385
386 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
387 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
388 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
389 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
390 const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
391 const int8x8_t vk18x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
392 const int8x8_t vi18x89ABCDEF = vld1_s8(i18); i18 += 8;
393 const int8x8_t vk18x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
394
395 vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
396 vprod89ABCDEF = vmull_s8(vi18x89ABCDEF, vk18x89ABCDEF);
397
398 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
399 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
400 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
401 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
402 const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
403 const int8x8_t vk19x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
404 const int8x8_t vi19x89ABCDEF = vld1_s8(i19); i19 += 8;
405 const int8x8_t vk19x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
406
407 vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
408 vprod89ABCDEF = vmull_s8(vi19x89ABCDEF, vk19x89ABCDEF);
409
410 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
411 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
412 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
413 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
414 const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
415 const int8x8_t vk20x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
416 const int8x8_t vi20x89ABCDEF = vld1_s8(i20); i20 += 8;
417 const int8x8_t vk20x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
418
419 vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
420 vprod89ABCDEF = vmull_s8(vi20x89ABCDEF, vk20x89ABCDEF);
421
422 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
423 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
424 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
425 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
426 const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
427 const int8x8_t vk21x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
428 const int8x8_t vi21x89ABCDEF = vld1_s8(i21); i21 += 8;
429 const int8x8_t vk21x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
430
431 vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
432 vprod89ABCDEF = vmull_s8(vi21x89ABCDEF, vk21x89ABCDEF);
433
434 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
435 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
436 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
437 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
438 const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
439 const int8x8_t vk22x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
440 const int8x8_t vi22x89ABCDEF = vld1_s8(i22); i22 += 8;
441 const int8x8_t vk22x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
442
443 vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
444 vprod89ABCDEF = vmull_s8(vi22x89ABCDEF, vk22x89ABCDEF);
445
446 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
447 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
448 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
449 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
450 const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
451 const int8x8_t vk23x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
452 const int8x8_t vi23x89ABCDEF = vld1_s8(i23); i23 += 8;
453 const int8x8_t vk23x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
454
455 vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
456 vprod89ABCDEF = vmull_s8(vi23x89ABCDEF, vk23x89ABCDEF);
457
458 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
459 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
460 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
461 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
462 const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
463 const int8x8_t vk24x01234567 = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
464 const int8x8_t vi24x89ABCDEF = vld1_s8(i24); i24 += 8;
465 const int8x8_t vk24x89ABCDEF = vld1_s8(w); w = (const void*) ((const int8_t*) w + 8);
466
467 vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
468 vprod89ABCDEF = vmull_s8(vi24x89ABCDEF, vk24x89ABCDEF);
469
470 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
471 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
472 vacc89AB = vaddw_s16(vacc89AB, vget_low_s16(vprod89ABCDEF));
473 vaccCDEF = vaddw_s16(vaccCDEF, vget_high_s16(vprod89ABCDEF));
474
475 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
476 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
477 vacc89AB = vqshlq_s32(vacc89AB, vright_pre_shift);
478 vaccCDEF = vqshlq_s32(vaccCDEF, vright_pre_shift);
479
480 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
481 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
482 vacc89AB = vqdmulhq_s32(vacc89AB, vmultiplier);
483 vaccCDEF = vqdmulhq_s32(vaccCDEF, vmultiplier);
484
485 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
486 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
487 vacc89AB = vrshlq_s32(vacc89AB, vright_post_shift);
488 vaccCDEF = vrshlq_s32(vaccCDEF, vright_post_shift);
489
490 #if XNN_ARCH_ARM64
491 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
492 int16x8_t vacc89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc89AB), vaccCDEF);
493
494 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
495 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
496
497 int8x16_t vout0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc01234567), vacc89ABCDEF);
498 #else // !XNN_ARCH_ARM64
499 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
500 int16x8_t vacc89ABCDEF = vcombine_s16(vqmovn_s32(vacc89AB), vqmovn_s32(vaccCDEF));
501
502 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
503 vacc89ABCDEF = vqaddq_s16(vacc89ABCDEF, voutput_zero_point);
504
505 int8x16_t vout0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc01234567), vqmovn_s16(vacc89ABCDEF));
506 #endif // !XNN_ARCH_ARM64
507
508 vout0123456789ABCDEF = vmaxq_s8(vout0123456789ABCDEF, voutput_min);
509
510 vout0123456789ABCDEF = vminq_s8(vout0123456789ABCDEF, voutput_max);
511
512 vst1q_s8(output, vout0123456789ABCDEF); output += 16;
513 }
514 if XNN_UNLIKELY(c != 0) {
515 const int8_t* k = (const int8_t*) ((const int32_t*) w + 16);
516 do {
517 int32x4_t vacc0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
518 int32x4_t vacc4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
519
520 const int8x8_t vi0x01234567 = vld1_s8(i0); i0 += 8;
521 const int8x8_t vk0x01234567 = vld1_s8(k); k += 8;
522
523 int16x8_t vprod01234567 = vmull_s8(vi0x01234567, vk0x01234567);
524
525 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
526 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
527 const int8x8_t vi1x01234567 = vld1_s8(i1); i1 += 8;
528 const int8x8_t vk1x01234567 = vld1_s8((const void*) (k + 8));
529
530 vprod01234567 = vmull_s8(vi1x01234567, vk1x01234567);
531
532 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
533 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
534 const int8x8_t vi2x01234567 = vld1_s8(i2); i2 += 8;
535 const int8x8_t vk2x01234567 = vld1_s8((const void*) (k + 24));
536
537 vprod01234567 = vmull_s8(vi2x01234567, vk2x01234567);
538
539 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
540 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
541 const int8x8_t vi3x01234567 = vld1_s8(i3); i3 += 8;
542 const int8x8_t vk3x01234567 = vld1_s8((const void*) (k + 40));
543
544 vprod01234567 = vmull_s8(vi3x01234567, vk3x01234567);
545
546 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
547 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
548 const int8x8_t vi4x01234567 = vld1_s8(i4); i4 += 8;
549 const int8x8_t vk4x01234567 = vld1_s8((const void*) (k + 56));
550
551 vprod01234567 = vmull_s8(vi4x01234567, vk4x01234567);
552
553 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
554 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
555 const int8x8_t vi5x01234567 = vld1_s8(i5); i5 += 8;
556 const int8x8_t vk5x01234567 = vld1_s8((const void*) (k + 72));
557
558 vprod01234567 = vmull_s8(vi5x01234567, vk5x01234567);
559
560 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
561 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
562 const int8x8_t vi6x01234567 = vld1_s8(i6); i6 += 8;
563 const int8x8_t vk6x01234567 = vld1_s8((const void*) (k + 88));
564
565 vprod01234567 = vmull_s8(vi6x01234567, vk6x01234567);
566
567 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
568 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
569 const int8x8_t vi7x01234567 = vld1_s8(i7); i7 += 8;
570 const int8x8_t vk7x01234567 = vld1_s8((const void*) (k + 104));
571
572 vprod01234567 = vmull_s8(vi7x01234567, vk7x01234567);
573
574 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
575 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
576 const int8x8_t vi8x01234567 = vld1_s8(i8); i8 += 8;
577 const int8x8_t vk8x01234567 = vld1_s8((const void*) (k + 120));
578
579 vprod01234567 = vmull_s8(vi8x01234567, vk8x01234567);
580
581 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
582 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
583 const int8x8_t vi9x01234567 = vld1_s8(i9); i9 += 8;
584 const int8x8_t vk9x01234567 = vld1_s8((const void*) (k + 136));
585
586 vprod01234567 = vmull_s8(vi9x01234567, vk9x01234567);
587
588 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
589 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
590 const int8x8_t vi10x01234567 = vld1_s8(i10); i10 += 8;
591 const int8x8_t vk10x01234567 = vld1_s8((const void*) (k + 152));
592
593 vprod01234567 = vmull_s8(vi10x01234567, vk10x01234567);
594
595 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
596 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
597 const int8x8_t vi11x01234567 = vld1_s8(i11); i11 += 8;
598 const int8x8_t vk11x01234567 = vld1_s8((const void*) (k + 168));
599
600 vprod01234567 = vmull_s8(vi11x01234567, vk11x01234567);
601
602 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
603 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
604 const int8x8_t vi12x01234567 = vld1_s8(i12); i12 += 8;
605 const int8x8_t vk12x01234567 = vld1_s8((const void*) (k + 184));
606
607 vprod01234567 = vmull_s8(vi12x01234567, vk12x01234567);
608
609 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
610 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
611 const int8x8_t vi13x01234567 = vld1_s8(i13); i13 += 8;
612 const int8x8_t vk13x01234567 = vld1_s8((const void*) (k + 200));
613
614 vprod01234567 = vmull_s8(vi13x01234567, vk13x01234567);
615
616 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
617 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
618 const int8x8_t vi14x01234567 = vld1_s8(i14); i14 += 8;
619 const int8x8_t vk14x01234567 = vld1_s8((const void*) (k + 216));
620
621 vprod01234567 = vmull_s8(vi14x01234567, vk14x01234567);
622
623 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
624 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
625 const int8x8_t vi15x01234567 = vld1_s8(i15); i15 += 8;
626 const int8x8_t vk15x01234567 = vld1_s8((const void*) (k + 232));
627
628 vprod01234567 = vmull_s8(vi15x01234567, vk15x01234567);
629
630 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
631 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
632 const int8x8_t vi16x01234567 = vld1_s8(i16); i16 += 8;
633 const int8x8_t vk16x01234567 = vld1_s8((const void*) (k + 248));
634
635 vprod01234567 = vmull_s8(vi16x01234567, vk16x01234567);
636
637 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
638 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
639 const int8x8_t vi17x01234567 = vld1_s8(i17); i17 += 8;
640 const int8x8_t vk17x01234567 = vld1_s8((const void*) (k + 264));
641
642 vprod01234567 = vmull_s8(vi17x01234567, vk17x01234567);
643
644 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
645 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
646 const int8x8_t vi18x01234567 = vld1_s8(i18); i18 += 8;
647 const int8x8_t vk18x01234567 = vld1_s8((const void*) (k + 280));
648
649 vprod01234567 = vmull_s8(vi18x01234567, vk18x01234567);
650
651 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
652 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
653 const int8x8_t vi19x01234567 = vld1_s8(i19); i19 += 8;
654 const int8x8_t vk19x01234567 = vld1_s8((const void*) (k + 296));
655
656 vprod01234567 = vmull_s8(vi19x01234567, vk19x01234567);
657
658 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
659 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
660 const int8x8_t vi20x01234567 = vld1_s8(i20); i20 += 8;
661 const int8x8_t vk20x01234567 = vld1_s8((const void*) (k + 312));
662
663 vprod01234567 = vmull_s8(vi20x01234567, vk20x01234567);
664
665 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
666 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
667 const int8x8_t vi21x01234567 = vld1_s8(i21); i21 += 8;
668 const int8x8_t vk21x01234567 = vld1_s8((const void*) (k + 328));
669
670 vprod01234567 = vmull_s8(vi21x01234567, vk21x01234567);
671
672 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
673 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
674 const int8x8_t vi22x01234567 = vld1_s8(i22); i22 += 8;
675 const int8x8_t vk22x01234567 = vld1_s8((const void*) (k + 344));
676
677 vprod01234567 = vmull_s8(vi22x01234567, vk22x01234567);
678
679 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
680 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
681 const int8x8_t vi23x01234567 = vld1_s8(i23); i23 += 8;
682 const int8x8_t vk23x01234567 = vld1_s8((const void*) (k + 360));
683
684 vprod01234567 = vmull_s8(vi23x01234567, vk23x01234567);
685
686 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
687 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
688 const int8x8_t vi24x01234567 = vld1_s8(i24); i24 += 8;
689 const int8x8_t vk24x01234567 = vld1_s8((const void*) (k + 376));
690
691 vprod01234567 = vmull_s8(vi24x01234567, vk24x01234567);
692
693 vacc0123 = vaddw_s16(vacc0123, vget_low_s16(vprod01234567));
694 vacc4567 = vaddw_s16(vacc4567, vget_high_s16(vprod01234567));
695
696 vacc0123 = vqshlq_s32(vacc0123, vright_pre_shift);
697 vacc4567 = vqshlq_s32(vacc4567, vright_pre_shift);
698
699 vacc0123 = vqdmulhq_s32(vacc0123, vmultiplier);
700 vacc4567 = vqdmulhq_s32(vacc4567, vmultiplier);
701
702 vacc0123 = vrshlq_s32(vacc0123, vright_post_shift);
703 vacc4567 = vrshlq_s32(vacc4567, vright_post_shift);
704
705 #if XNN_ARCH_ARM64
706 int16x8_t vacc01234567 = vqmovn_high_s32(vqmovn_s32(vacc0123), vacc4567);
707 #else
708 int16x8_t vacc01234567 = vcombine_s16(vqmovn_s32(vacc0123), vqmovn_s32(vacc4567));
709 #endif
710 vacc01234567 = vqaddq_s16(vacc01234567, voutput_zero_point);
711
712 int8x8_t vout01234567 = vqmovn_s16(vacc01234567);
713 vout01234567 = vmax_s8(vout01234567, vget_low_s8(voutput_min));
714 vout01234567 = vmin_s8(vout01234567, vget_low_s8(voutput_max));
715
716 if XNN_LIKELY(c >= 8) {
717 vst1_s8(output, vout01234567); output += 8;
718 c -= 8;
719 } else {
720 if (c & 4) {
721 vst1_lane_u32((void*) output, vreinterpret_u32_s8(vout01234567), 0); output += 4;
722 vout01234567 = vext_s8(vout01234567, vout01234567, 4);
723 }
724 if (c & 2) {
725 vst1_lane_u16((void*) output, vreinterpret_u16_s8(vout01234567), 0); output += 2;
726 vout01234567 = vext_s8(vout01234567, vout01234567, 2);
727 }
728 if (c & 1) {
729 vst1_lane_s8(output, vout01234567, 0); output += 1;
730 }
731 c = 0;
732 }
733 } while (c != 0);
734 }
735
736 output = (int8_t*) ((uintptr_t) output + output_increment);
737 } while (--output_width != 0);
738 }
739