xref: /aosp_15_r20/external/XNNPACK/src/qs8-gemm/gen/4x16c2-minmax-rndnu-neon-mlal-dup.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-gemm/c2-neon-mull-dup.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/gemm.h>
15 #include <xnnpack/math.h>
16 
xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c2__neon_mlal_dup(size_t mr,size_t nc,size_t kc,const int8_t * restrict a,size_t a_stride,const void * restrict w,int8_t * restrict c,size_t cm_stride,size_t cn_stride,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_gemm_minmax_rndnu_ukernel_4x16c2__neon_mlal_dup(
18     size_t mr,
19     size_t nc,
20     size_t kc,
21     const int8_t* restrict a,
22     size_t a_stride,
23     const void* restrict w,
24     int8_t* restrict c,
25     size_t cm_stride,
26     size_t cn_stride,
27     const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(mr != 0);
30   assert(mr <= 4);
31   assert(nc != 0);
32   assert(kc != 0);
33   assert(kc % sizeof(int8_t) == 0);
34   assert(a != NULL);
35   assert(w != NULL);
36   assert(c != NULL);
37 
38   kc = round_up_po2(kc, 2 * sizeof(int8_t));
39   const int8_t* a0 = a;
40   int8_t* c0 = c;
41   const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
42   int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
43   if XNN_UNPREDICTABLE(mr < 2) {
44     a1 = a0;
45     c1 = c0;
46   }
47   const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
48   int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
49   if XNN_UNPREDICTABLE(mr <= 2) {
50     a2 = a1;
51     c2 = c1;
52   }
53   const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
54   int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
55   if XNN_UNPREDICTABLE(mr != 4) {
56     a3 = a2;
57     c3 = c2;
58   }
59 
60   do {
61     int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
62     int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
63     int32x4_t vacc0x89AB = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
64     int32x4_t vacc0xCDEF = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
65     int32x4_t vacc1x0123 = vacc0x0123;
66     int32x4_t vacc1x4567 = vacc0x4567;
67     int32x4_t vacc1x89AB = vacc0x89AB;
68     int32x4_t vacc1xCDEF = vacc0xCDEF;
69     int32x4_t vacc2x0123 = vacc0x0123;
70     int32x4_t vacc2x4567 = vacc0x4567;
71     int32x4_t vacc2x89AB = vacc0x89AB;
72     int32x4_t vacc2xCDEF = vacc0xCDEF;
73     int32x4_t vacc3x0123 = vacc0x0123;
74     int32x4_t vacc3x4567 = vacc0x4567;
75     int32x4_t vacc3x89AB = vacc0x89AB;
76     int32x4_t vacc3xCDEF = vacc0xCDEF;
77 
78     size_t k = kc;
79 
80     while (k >= 16 * sizeof(int8_t)) {
81       const int8x8_t va0x0 = vld1_s8(a0); a0 += 8;
82       const int8x8_t va0x1 = vld1_s8(a0); a0 += 8;
83       const int8x8_t va1x0 = vld1_s8(a1); a1 += 8;
84       const int8x8_t va1x1 = vld1_s8(a1); a1 += 8;
85       const int8x8_t va2x0 = vld1_s8(a2); a2 += 8;
86       const int8x8_t va2x1 = vld1_s8(a2); a2 += 8;
87       const int8x8_t va3x0 = vld1_s8(a3); a3 += 8;
88       const int8x8_t va3x1 = vld1_s8(a3); a3 += 8;
89       const int8x8_t vb0123c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
90       const int8x8_t vb4567c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
91       const int8x8_t vb89ABc0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
92       const int8x8_t vbCDEFc0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
93       const int8x8_t vb0123c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
94       const int8x8_t vb4567c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
95       const int8x8_t vb89ABc1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
96       const int8x8_t vbCDEFc1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
97       const int8x8_t vb0123c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
98       const int8x8_t vb4567c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
99       const int8x8_t vb89ABc2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
100       const int8x8_t vbCDEFc2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
101       const int8x8_t vb0123c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
102       const int8x8_t vb4567c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
103       const int8x8_t vb89ABc3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
104       const int8x8_t vbCDEFc3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
105 
106       const int8x8_t va0c0x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x0), 0));
107       const int8x8_t va0c0x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x1), 0));
108       const int8x8_t va1c0x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x0), 0));
109       const int8x8_t va1c0x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x1), 0));
110       const int8x8_t va2c0x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x0), 0));
111       const int8x8_t va2c0x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x1), 0));
112       const int8x8_t va3c0x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x0), 0));
113       const int8x8_t va3c0x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x1), 0));
114 
115       int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0x0, va0c0x0);
116       int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0x0, va1c0x0);
117       int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0x0, va2c0x0);
118       int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0x0, va3c0x0);
119       const int8x8_t vb0123c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
120       vprod0x0123c0 = vmlal_s8(vprod0x0123c0, vb0123c0x1, va0c0x1);
121       vprod1x0123c0 = vmlal_s8(vprod1x0123c0, vb0123c0x1, va1c0x1);
122       vprod2x0123c0 = vmlal_s8(vprod2x0123c0, vb0123c0x1, va2c0x1);
123       vprod3x0123c0 = vmlal_s8(vprod3x0123c0, vb0123c0x1, va3c0x1);
124       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
125       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
126       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
127       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
128       int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0x0, va0c0x0);
129       int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0x0, va1c0x0);
130       int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0x0, va2c0x0);
131       int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0x0, va3c0x0);
132       const int8x8_t vb4567c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
133       vprod0x4567c0 = vmlal_s8(vprod0x4567c0, vb4567c0x1, va0c0x1);
134       vprod1x4567c0 = vmlal_s8(vprod1x4567c0, vb4567c0x1, va1c0x1);
135       vprod2x4567c0 = vmlal_s8(vprod2x4567c0, vb4567c0x1, va2c0x1);
136       vprod3x4567c0 = vmlal_s8(vprod3x4567c0, vb4567c0x1, va3c0x1);
137       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
138       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
139       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
140       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
141       int16x8_t vprod0x89ABc0 = vmull_s8(vb89ABc0x0, va0c0x0);
142       int16x8_t vprod1x89ABc0 = vmull_s8(vb89ABc0x0, va1c0x0);
143       int16x8_t vprod2x89ABc0 = vmull_s8(vb89ABc0x0, va2c0x0);
144       int16x8_t vprod3x89ABc0 = vmull_s8(vb89ABc0x0, va3c0x0);
145       const int8x8_t vb89ABc0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
146       vprod0x89ABc0 = vmlal_s8(vprod0x89ABc0, vb89ABc0x1, va0c0x1);
147       vprod1x89ABc0 = vmlal_s8(vprod1x89ABc0, vb89ABc0x1, va1c0x1);
148       vprod2x89ABc0 = vmlal_s8(vprod2x89ABc0, vb89ABc0x1, va2c0x1);
149       vprod3x89ABc0 = vmlal_s8(vprod3x89ABc0, vb89ABc0x1, va3c0x1);
150       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc0);
151       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc0);
152       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc0);
153       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc0);
154       int16x8_t vprod0xCDEFc0 = vmull_s8(vbCDEFc0x0, va0c0x0);
155       int16x8_t vprod1xCDEFc0 = vmull_s8(vbCDEFc0x0, va1c0x0);
156       int16x8_t vprod2xCDEFc0 = vmull_s8(vbCDEFc0x0, va2c0x0);
157       int16x8_t vprod3xCDEFc0 = vmull_s8(vbCDEFc0x0, va3c0x0);
158       const int8x8_t vbCDEFc0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
159       vprod0xCDEFc0 = vmlal_s8(vprod0xCDEFc0, vbCDEFc0x1, va0c0x1);
160       vprod1xCDEFc0 = vmlal_s8(vprod1xCDEFc0, vbCDEFc0x1, va1c0x1);
161       vprod2xCDEFc0 = vmlal_s8(vprod2xCDEFc0, vbCDEFc0x1, va2c0x1);
162       vprod3xCDEFc0 = vmlal_s8(vprod3xCDEFc0, vbCDEFc0x1, va3c0x1);
163       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc0);
164       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc0);
165       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc0);
166       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc0);
167       const int8x8_t va0c1x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x0), 1));
168       const int8x8_t va0c1x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x1), 1));
169       const int8x8_t va1c1x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x0), 1));
170       const int8x8_t va1c1x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x1), 1));
171       const int8x8_t va2c1x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x0), 1));
172       const int8x8_t va2c1x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x1), 1));
173       const int8x8_t va3c1x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x0), 1));
174       const int8x8_t va3c1x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x1), 1));
175 
176       int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1x0, va0c1x0);
177       int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1x0, va1c1x0);
178       int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1x0, va2c1x0);
179       int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1x0, va3c1x0);
180       const int8x8_t vb0123c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
181       vprod0x0123c1 = vmlal_s8(vprod0x0123c1, vb0123c1x1, va0c1x1);
182       vprod1x0123c1 = vmlal_s8(vprod1x0123c1, vb0123c1x1, va1c1x1);
183       vprod2x0123c1 = vmlal_s8(vprod2x0123c1, vb0123c1x1, va2c1x1);
184       vprod3x0123c1 = vmlal_s8(vprod3x0123c1, vb0123c1x1, va3c1x1);
185       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
186       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
187       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
188       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
189       int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1x0, va0c1x0);
190       int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1x0, va1c1x0);
191       int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1x0, va2c1x0);
192       int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1x0, va3c1x0);
193       const int8x8_t vb4567c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
194       vprod0x4567c1 = vmlal_s8(vprod0x4567c1, vb4567c1x1, va0c1x1);
195       vprod1x4567c1 = vmlal_s8(vprod1x4567c1, vb4567c1x1, va1c1x1);
196       vprod2x4567c1 = vmlal_s8(vprod2x4567c1, vb4567c1x1, va2c1x1);
197       vprod3x4567c1 = vmlal_s8(vprod3x4567c1, vb4567c1x1, va3c1x1);
198       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
199       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
200       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
201       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
202       int16x8_t vprod0x89ABc1 = vmull_s8(vb89ABc1x0, va0c1x0);
203       int16x8_t vprod1x89ABc1 = vmull_s8(vb89ABc1x0, va1c1x0);
204       int16x8_t vprod2x89ABc1 = vmull_s8(vb89ABc1x0, va2c1x0);
205       int16x8_t vprod3x89ABc1 = vmull_s8(vb89ABc1x0, va3c1x0);
206       const int8x8_t vb89ABc1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
207       vprod0x89ABc1 = vmlal_s8(vprod0x89ABc1, vb89ABc1x1, va0c1x1);
208       vprod1x89ABc1 = vmlal_s8(vprod1x89ABc1, vb89ABc1x1, va1c1x1);
209       vprod2x89ABc1 = vmlal_s8(vprod2x89ABc1, vb89ABc1x1, va2c1x1);
210       vprod3x89ABc1 = vmlal_s8(vprod3x89ABc1, vb89ABc1x1, va3c1x1);
211       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc1);
212       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc1);
213       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc1);
214       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc1);
215       int16x8_t vprod0xCDEFc1 = vmull_s8(vbCDEFc1x0, va0c1x0);
216       int16x8_t vprod1xCDEFc1 = vmull_s8(vbCDEFc1x0, va1c1x0);
217       int16x8_t vprod2xCDEFc1 = vmull_s8(vbCDEFc1x0, va2c1x0);
218       int16x8_t vprod3xCDEFc1 = vmull_s8(vbCDEFc1x0, va3c1x0);
219       const int8x8_t vbCDEFc1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
220       vprod0xCDEFc1 = vmlal_s8(vprod0xCDEFc1, vbCDEFc1x1, va0c1x1);
221       vprod1xCDEFc1 = vmlal_s8(vprod1xCDEFc1, vbCDEFc1x1, va1c1x1);
222       vprod2xCDEFc1 = vmlal_s8(vprod2xCDEFc1, vbCDEFc1x1, va2c1x1);
223       vprod3xCDEFc1 = vmlal_s8(vprod3xCDEFc1, vbCDEFc1x1, va3c1x1);
224       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc1);
225       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc1);
226       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc1);
227       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc1);
228       const int8x8_t va0c2x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x0), 2));
229       const int8x8_t va0c2x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x1), 2));
230       const int8x8_t va1c2x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x0), 2));
231       const int8x8_t va1c2x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x1), 2));
232       const int8x8_t va2c2x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x0), 2));
233       const int8x8_t va2c2x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x1), 2));
234       const int8x8_t va3c2x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x0), 2));
235       const int8x8_t va3c2x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x1), 2));
236 
237       int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2x0, va0c2x0);
238       int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2x0, va1c2x0);
239       int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2x0, va2c2x0);
240       int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2x0, va3c2x0);
241       const int8x8_t vb0123c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
242       vprod0x0123c2 = vmlal_s8(vprod0x0123c2, vb0123c2x1, va0c2x1);
243       vprod1x0123c2 = vmlal_s8(vprod1x0123c2, vb0123c2x1, va1c2x1);
244       vprod2x0123c2 = vmlal_s8(vprod2x0123c2, vb0123c2x1, va2c2x1);
245       vprod3x0123c2 = vmlal_s8(vprod3x0123c2, vb0123c2x1, va3c2x1);
246       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
247       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
248       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
249       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
250       int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2x0, va0c2x0);
251       int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2x0, va1c2x0);
252       int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2x0, va2c2x0);
253       int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2x0, va3c2x0);
254       const int8x8_t vb4567c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
255       vprod0x4567c2 = vmlal_s8(vprod0x4567c2, vb4567c2x1, va0c2x1);
256       vprod1x4567c2 = vmlal_s8(vprod1x4567c2, vb4567c2x1, va1c2x1);
257       vprod2x4567c2 = vmlal_s8(vprod2x4567c2, vb4567c2x1, va2c2x1);
258       vprod3x4567c2 = vmlal_s8(vprod3x4567c2, vb4567c2x1, va3c2x1);
259       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
260       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
261       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
262       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
263       int16x8_t vprod0x89ABc2 = vmull_s8(vb89ABc2x0, va0c2x0);
264       int16x8_t vprod1x89ABc2 = vmull_s8(vb89ABc2x0, va1c2x0);
265       int16x8_t vprod2x89ABc2 = vmull_s8(vb89ABc2x0, va2c2x0);
266       int16x8_t vprod3x89ABc2 = vmull_s8(vb89ABc2x0, va3c2x0);
267       const int8x8_t vb89ABc2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
268       vprod0x89ABc2 = vmlal_s8(vprod0x89ABc2, vb89ABc2x1, va0c2x1);
269       vprod1x89ABc2 = vmlal_s8(vprod1x89ABc2, vb89ABc2x1, va1c2x1);
270       vprod2x89ABc2 = vmlal_s8(vprod2x89ABc2, vb89ABc2x1, va2c2x1);
271       vprod3x89ABc2 = vmlal_s8(vprod3x89ABc2, vb89ABc2x1, va3c2x1);
272       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);
273       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);
274       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);
275       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc2);
276       int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2x0, va0c2x0);
277       int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2x0, va1c2x0);
278       int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2x0, va2c2x0);
279       int16x8_t vprod3xCDEFc2 = vmull_s8(vbCDEFc2x0, va3c2x0);
280       const int8x8_t vbCDEFc2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
281       vprod0xCDEFc2 = vmlal_s8(vprod0xCDEFc2, vbCDEFc2x1, va0c2x1);
282       vprod1xCDEFc2 = vmlal_s8(vprod1xCDEFc2, vbCDEFc2x1, va1c2x1);
283       vprod2xCDEFc2 = vmlal_s8(vprod2xCDEFc2, vbCDEFc2x1, va2c2x1);
284       vprod3xCDEFc2 = vmlal_s8(vprod3xCDEFc2, vbCDEFc2x1, va3c2x1);
285       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);
286       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);
287       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);
288       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc2);
289       const int8x8_t va0c3x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x0), 3));
290       const int8x8_t va0c3x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0x1), 3));
291       const int8x8_t va1c3x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x0), 3));
292       const int8x8_t va1c3x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1x1), 3));
293       const int8x8_t va2c3x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x0), 3));
294       const int8x8_t va2c3x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2x1), 3));
295       const int8x8_t va3c3x0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x0), 3));
296       const int8x8_t va3c3x1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3x1), 3));
297 
298       int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3x0, va0c3x0);
299       int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3x0, va1c3x0);
300       int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3x0, va2c3x0);
301       int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3x0, va3c3x0);
302       const int8x8_t vb0123c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
303       vprod0x0123c3 = vmlal_s8(vprod0x0123c3, vb0123c3x1, va0c3x1);
304       vprod1x0123c3 = vmlal_s8(vprod1x0123c3, vb0123c3x1, va1c3x1);
305       vprod2x0123c3 = vmlal_s8(vprod2x0123c3, vb0123c3x1, va2c3x1);
306       vprod3x0123c3 = vmlal_s8(vprod3x0123c3, vb0123c3x1, va3c3x1);
307       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
308       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
309       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
310       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
311       int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3x0, va0c3x0);
312       int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3x0, va1c3x0);
313       int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3x0, va2c3x0);
314       int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3x0, va3c3x0);
315       const int8x8_t vb4567c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
316       vprod0x4567c3 = vmlal_s8(vprod0x4567c3, vb4567c3x1, va0c3x1);
317       vprod1x4567c3 = vmlal_s8(vprod1x4567c3, vb4567c3x1, va1c3x1);
318       vprod2x4567c3 = vmlal_s8(vprod2x4567c3, vb4567c3x1, va2c3x1);
319       vprod3x4567c3 = vmlal_s8(vprod3x4567c3, vb4567c3x1, va3c3x1);
320       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
321       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
322       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
323       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
324       int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3x0, va0c3x0);
325       int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3x0, va1c3x0);
326       int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3x0, va2c3x0);
327       int16x8_t vprod3x89ABc3 = vmull_s8(vb89ABc3x0, va3c3x0);
328       const int8x8_t vb89ABc3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
329       vprod0x89ABc3 = vmlal_s8(vprod0x89ABc3, vb89ABc3x1, va0c3x1);
330       vprod1x89ABc3 = vmlal_s8(vprod1x89ABc3, vb89ABc3x1, va1c3x1);
331       vprod2x89ABc3 = vmlal_s8(vprod2x89ABc3, vb89ABc3x1, va2c3x1);
332       vprod3x89ABc3 = vmlal_s8(vprod3x89ABc3, vb89ABc3x1, va3c3x1);
333       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
334       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
335       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
336       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc3);
337       int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3x0, va0c3x0);
338       int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3x0, va1c3x0);
339       int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3x0, va2c3x0);
340       int16x8_t vprod3xCDEFc3 = vmull_s8(vbCDEFc3x0, va3c3x0);
341       const int8x8_t vbCDEFc3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
342       vprod0xCDEFc3 = vmlal_s8(vprod0xCDEFc3, vbCDEFc3x1, va0c3x1);
343       vprod1xCDEFc3 = vmlal_s8(vprod1xCDEFc3, vbCDEFc3x1, va1c3x1);
344       vprod2xCDEFc3 = vmlal_s8(vprod2xCDEFc3, vbCDEFc3x1, va2c3x1);
345       vprod3xCDEFc3 = vmlal_s8(vprod3xCDEFc3, vbCDEFc3x1, va3c3x1);
346       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
347       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
348       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
349       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc3);
350 
351       k -= 16 * sizeof(int8_t);
352     }
353 
354     if (k >= 8 * sizeof(int8_t)) {
355       const int8x8_t va0 = vld1_s8(a0); a0 += 8;
356       const int8x8_t va1 = vld1_s8(a1); a1 += 8;
357       const int8x8_t va2 = vld1_s8(a2); a2 += 8;
358       const int8x8_t va3 = vld1_s8(a3); a3 += 8;
359 
360       const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
361       const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
362       const int8x8_t vb89ABc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
363       const int8x8_t vbCDEFc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
364       const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
365       const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
366       const int8x8_t vb89ABc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
367       const int8x8_t vbCDEFc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
368       const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
369       const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
370       const int8x8_t vb89ABc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
371       const int8x8_t vbCDEFc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
372       const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
373       const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
374       const int8x8_t vb89ABc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
375       const int8x8_t vbCDEFc3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
376 
377       const int8x8_t va0c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0));
378       const int8x8_t va1c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0));
379       const int8x8_t va2c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 0));
380       const int8x8_t va3c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 0));
381 
382       const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);
383       const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);
384       const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);
385       const int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0, va3c0);
386       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
387       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
388       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
389       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
390       const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);
391       const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);
392       const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);
393       const int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0, va3c0);
394       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
395       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
396       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
397       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
398       const int16x8_t vprod0x89ABc0 = vmull_s8(vb89ABc0, va0c0);
399       const int16x8_t vprod1x89ABc0 = vmull_s8(vb89ABc0, va1c0);
400       const int16x8_t vprod2x89ABc0 = vmull_s8(vb89ABc0, va2c0);
401       const int16x8_t vprod3x89ABc0 = vmull_s8(vb89ABc0, va3c0);
402       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc0);
403       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc0);
404       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc0);
405       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc0);
406       const int16x8_t vprod0xCDEFc0 = vmull_s8(vbCDEFc0, va0c0);
407       const int16x8_t vprod1xCDEFc0 = vmull_s8(vbCDEFc0, va1c0);
408       const int16x8_t vprod2xCDEFc0 = vmull_s8(vbCDEFc0, va2c0);
409       const int16x8_t vprod3xCDEFc0 = vmull_s8(vbCDEFc0, va3c0);
410       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc0);
411       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc0);
412       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc0);
413       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc0);
414       const int8x8_t va0c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1));
415       const int8x8_t va1c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1));
416       const int8x8_t va2c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 1));
417       const int8x8_t va3c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 1));
418 
419       const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);
420       const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);
421       const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);
422       const int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1, va3c1);
423       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
424       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
425       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
426       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
427       const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);
428       const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);
429       const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);
430       const int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1, va3c1);
431       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
432       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
433       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
434       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
435       const int16x8_t vprod0x89ABc1 = vmull_s8(vb89ABc1, va0c1);
436       const int16x8_t vprod1x89ABc1 = vmull_s8(vb89ABc1, va1c1);
437       const int16x8_t vprod2x89ABc1 = vmull_s8(vb89ABc1, va2c1);
438       const int16x8_t vprod3x89ABc1 = vmull_s8(vb89ABc1, va3c1);
439       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc1);
440       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc1);
441       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc1);
442       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc1);
443       const int16x8_t vprod0xCDEFc1 = vmull_s8(vbCDEFc1, va0c1);
444       const int16x8_t vprod1xCDEFc1 = vmull_s8(vbCDEFc1, va1c1);
445       const int16x8_t vprod2xCDEFc1 = vmull_s8(vbCDEFc1, va2c1);
446       const int16x8_t vprod3xCDEFc1 = vmull_s8(vbCDEFc1, va3c1);
447       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc1);
448       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc1);
449       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc1);
450       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc1);
451       const int8x8_t va0c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2));
452       const int8x8_t va1c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2));
453       const int8x8_t va2c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2));
454       const int8x8_t va3c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2));
455 
456       const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);
457       const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);
458       const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);
459       const int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2, va3c2);
460       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
461       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
462       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
463       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
464       const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);
465       const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);
466       const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);
467       const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, va3c2);
468       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
469       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
470       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
471       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
472       const int16x8_t vprod0x89ABc2 = vmull_s8(vb89ABc2, va0c2);
473       const int16x8_t vprod1x89ABc2 = vmull_s8(vb89ABc2, va1c2);
474       const int16x8_t vprod2x89ABc2 = vmull_s8(vb89ABc2, va2c2);
475       const int16x8_t vprod3x89ABc2 = vmull_s8(vb89ABc2, va3c2);
476       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);
477       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);
478       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);
479       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc2);
480       const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, va0c2);
481       const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, va1c2);
482       const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, va2c2);
483       const int16x8_t vprod3xCDEFc2 = vmull_s8(vbCDEFc2, va3c2);
484       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);
485       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);
486       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);
487       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc2);
488       const int8x8_t va0c3 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 3));
489       const int8x8_t va1c3 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 3));
490       const int8x8_t va2c3 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 3));
491       const int8x8_t va3c3 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 3));
492 
493       const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, va0c3);
494       const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, va1c3);
495       const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, va2c3);
496       const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, va3c3);
497       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
498       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
499       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
500       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
501       const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, va0c3);
502       const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, va1c3);
503       const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, va2c3);
504       const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, va3c3);
505       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
506       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
507       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
508       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
509       const int16x8_t vprod0x89ABc3 = vmull_s8(vb89ABc3, va0c3);
510       const int16x8_t vprod1x89ABc3 = vmull_s8(vb89ABc3, va1c3);
511       const int16x8_t vprod2x89ABc3 = vmull_s8(vb89ABc3, va2c3);
512       const int16x8_t vprod3x89ABc3 = vmull_s8(vb89ABc3, va3c3);
513       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc3);
514       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc3);
515       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc3);
516       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc3);
517       const int16x8_t vprod0xCDEFc3 = vmull_s8(vbCDEFc3, va0c3);
518       const int16x8_t vprod1xCDEFc3 = vmull_s8(vbCDEFc3, va1c3);
519       const int16x8_t vprod2xCDEFc3 = vmull_s8(vbCDEFc3, va2c3);
520       const int16x8_t vprod3xCDEFc3 = vmull_s8(vbCDEFc3, va3c3);
521       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc3);
522       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc3);
523       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc3);
524       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc3);
525 
526       k -= 8 * sizeof(int8_t);
527     }
528 
529     if XNN_UNLIKELY(k != 0) {
530       const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);
531       const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);
532       const int8x8_t va2 = vld1_s8(a2); a2 = (const int8_t*) ((uintptr_t) a2 + k);
533       const int8x8_t va3 = vld1_s8(a3); a3 = (const int8_t*) ((uintptr_t) a3 + k);
534 
535       const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
536       const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
537       const int8x8_t vb89ABc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
538       const int8x8_t vbCDEFc0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
539 
540       const int8x8_t va0c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0));
541       const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);
542       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
543       const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);
544       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
545       const int16x8_t vprod0x89ABc0 = vmull_s8(vb89ABc0, va0c0);
546       vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc0);
547       const int16x8_t vprod0xCDEFc0 = vmull_s8(vbCDEFc0, va0c0);
548       vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc0);
549       const int8x8_t va1c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0));
550       const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);
551       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
552       const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);
553       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
554       const int16x8_t vprod1x89ABc0 = vmull_s8(vb89ABc0, va1c0);
555       vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc0);
556       const int16x8_t vprod1xCDEFc0 = vmull_s8(vbCDEFc0, va1c0);
557       vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc0);
558       const int8x8_t va2c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 0));
559       const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);
560       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
561       const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);
562       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
563       const int16x8_t vprod2x89ABc0 = vmull_s8(vb89ABc0, va2c0);
564       vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc0);
565       const int16x8_t vprod2xCDEFc0 = vmull_s8(vbCDEFc0, va2c0);
566       vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc0);
567       const int8x8_t va3c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 0));
568       const int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0, va3c0);
569       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
570       const int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0, va3c0);
571       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
572       const int16x8_t vprod3x89ABc0 = vmull_s8(vb89ABc0, va3c0);
573       vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc0);
574       const int16x8_t vprod3xCDEFc0 = vmull_s8(vbCDEFc0, va3c0);
575       vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc0);
576 
577       if (k > 2 * sizeof(int8_t)) {
578         const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
579         const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
580         const int8x8_t vb89ABc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
581         const int8x8_t vbCDEFc1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
582 
583         const int8x8_t va0c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1));
584         const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);
585         vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
586         const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);
587         vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
588         const int16x8_t vprod0x89ABc1 = vmull_s8(vb89ABc1, va0c1);
589         vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc1);
590         const int16x8_t vprod0xCDEFc1 = vmull_s8(vbCDEFc1, va0c1);
591         vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc1);
592         const int8x8_t va1c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1));
593         const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);
594         vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
595         const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);
596         vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
597         const int16x8_t vprod1x89ABc1 = vmull_s8(vb89ABc1, va1c1);
598         vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc1);
599         const int16x8_t vprod1xCDEFc1 = vmull_s8(vbCDEFc1, va1c1);
600         vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc1);
601         const int8x8_t va2c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 1));
602         const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);
603         vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
604         const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);
605         vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
606         const int16x8_t vprod2x89ABc1 = vmull_s8(vb89ABc1, va2c1);
607         vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc1);
608         const int16x8_t vprod2xCDEFc1 = vmull_s8(vbCDEFc1, va2c1);
609         vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc1);
610         const int8x8_t va3c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 1));
611         const int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1, va3c1);
612         vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
613         const int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1, va3c1);
614         vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
615         const int16x8_t vprod3x89ABc1 = vmull_s8(vb89ABc1, va3c1);
616         vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc1);
617         const int16x8_t vprod3xCDEFc1 = vmull_s8(vbCDEFc1, va3c1);
618         vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc1);
619 
620         if (k > 4 * sizeof(int8_t)) {
621           const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
622           const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
623           const int8x8_t vb89ABc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
624           const int8x8_t vbCDEFc2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
625 
626           const int8x8_t va0c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2));
627           const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);
628           vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
629           const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);
630           vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
631           const int16x8_t vprod0x89ABc2 = vmull_s8(vb89ABc2, va0c2);
632           vacc0x89AB = vpadalq_s16(vacc0x89AB, vprod0x89ABc2);
633           const int16x8_t vprod0xCDEFc2 = vmull_s8(vbCDEFc2, va0c2);
634           vacc0xCDEF = vpadalq_s16(vacc0xCDEF, vprod0xCDEFc2);
635           const int8x8_t va1c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2));
636           const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);
637           vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
638           const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);
639           vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
640           const int16x8_t vprod1x89ABc2 = vmull_s8(vb89ABc2, va1c2);
641           vacc1x89AB = vpadalq_s16(vacc1x89AB, vprod1x89ABc2);
642           const int16x8_t vprod1xCDEFc2 = vmull_s8(vbCDEFc2, va1c2);
643           vacc1xCDEF = vpadalq_s16(vacc1xCDEF, vprod1xCDEFc2);
644           const int8x8_t va2c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2));
645           const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);
646           vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
647           const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);
648           vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
649           const int16x8_t vprod2x89ABc2 = vmull_s8(vb89ABc2, va2c2);
650           vacc2x89AB = vpadalq_s16(vacc2x89AB, vprod2x89ABc2);
651           const int16x8_t vprod2xCDEFc2 = vmull_s8(vbCDEFc2, va2c2);
652           vacc2xCDEF = vpadalq_s16(vacc2xCDEF, vprod2xCDEFc2);
653           const int8x8_t va3c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2));
654           const int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2, va3c2);
655           vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
656           const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, va3c2);
657           vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
658           const int16x8_t vprod3x89ABc2 = vmull_s8(vb89ABc2, va3c2);
659           vacc3x89AB = vpadalq_s16(vacc3x89AB, vprod3x89ABc2);
660           const int16x8_t vprod3xCDEFc2 = vmull_s8(vbCDEFc2, va3c2);
661           vacc3xCDEF = vpadalq_s16(vacc3xCDEF, vprod3xCDEFc2);
662         }
663       }
664     }
665 
666     const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
667     const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
668     const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
669 
670     vacc0x0123 = vqshlq_s32(vacc0x0123, vright_pre_shift);
671     vacc0x4567 = vqshlq_s32(vacc0x4567, vright_pre_shift);
672     vacc0x89AB = vqshlq_s32(vacc0x89AB, vright_pre_shift);
673     vacc0xCDEF = vqshlq_s32(vacc0xCDEF, vright_pre_shift);
674     vacc1x0123 = vqshlq_s32(vacc1x0123, vright_pre_shift);
675     vacc1x4567 = vqshlq_s32(vacc1x4567, vright_pre_shift);
676     vacc1x89AB = vqshlq_s32(vacc1x89AB, vright_pre_shift);
677     vacc1xCDEF = vqshlq_s32(vacc1xCDEF, vright_pre_shift);
678     vacc2x0123 = vqshlq_s32(vacc2x0123, vright_pre_shift);
679     vacc2x4567 = vqshlq_s32(vacc2x4567, vright_pre_shift);
680     vacc2x89AB = vqshlq_s32(vacc2x89AB, vright_pre_shift);
681     vacc2xCDEF = vqshlq_s32(vacc2xCDEF, vright_pre_shift);
682     vacc3x0123 = vqshlq_s32(vacc3x0123, vright_pre_shift);
683     vacc3x4567 = vqshlq_s32(vacc3x4567, vright_pre_shift);
684     vacc3x89AB = vqshlq_s32(vacc3x89AB, vright_pre_shift);
685     vacc3xCDEF = vqshlq_s32(vacc3xCDEF, vright_pre_shift);
686 
687     vacc0x0123 = vqdmulhq_s32(vacc0x0123, vmultiplier);
688     vacc0x4567 = vqdmulhq_s32(vacc0x4567, vmultiplier);
689     vacc0x89AB = vqdmulhq_s32(vacc0x89AB, vmultiplier);
690     vacc0xCDEF = vqdmulhq_s32(vacc0xCDEF, vmultiplier);
691     vacc1x0123 = vqdmulhq_s32(vacc1x0123, vmultiplier);
692     vacc1x4567 = vqdmulhq_s32(vacc1x4567, vmultiplier);
693     vacc1x89AB = vqdmulhq_s32(vacc1x89AB, vmultiplier);
694     vacc1xCDEF = vqdmulhq_s32(vacc1xCDEF, vmultiplier);
695     vacc2x0123 = vqdmulhq_s32(vacc2x0123, vmultiplier);
696     vacc2x4567 = vqdmulhq_s32(vacc2x4567, vmultiplier);
697     vacc2x89AB = vqdmulhq_s32(vacc2x89AB, vmultiplier);
698     vacc2xCDEF = vqdmulhq_s32(vacc2xCDEF, vmultiplier);
699     vacc3x0123 = vqdmulhq_s32(vacc3x0123, vmultiplier);
700     vacc3x4567 = vqdmulhq_s32(vacc3x4567, vmultiplier);
701     vacc3x89AB = vqdmulhq_s32(vacc3x89AB, vmultiplier);
702     vacc3xCDEF = vqdmulhq_s32(vacc3xCDEF, vmultiplier);
703 
704     vacc0x0123 = vrshlq_s32(vacc0x0123, vright_post_shift);
705     vacc0x4567 = vrshlq_s32(vacc0x4567, vright_post_shift);
706     vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_post_shift);
707     vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_post_shift);
708     vacc1x0123 = vrshlq_s32(vacc1x0123, vright_post_shift);
709     vacc1x4567 = vrshlq_s32(vacc1x4567, vright_post_shift);
710     vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_post_shift);
711     vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_post_shift);
712     vacc2x0123 = vrshlq_s32(vacc2x0123, vright_post_shift);
713     vacc2x4567 = vrshlq_s32(vacc2x4567, vright_post_shift);
714     vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_post_shift);
715     vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_post_shift);
716     vacc3x0123 = vrshlq_s32(vacc3x0123, vright_post_shift);
717     vacc3x4567 = vrshlq_s32(vacc3x4567, vright_post_shift);
718     vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_post_shift);
719     vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_post_shift);
720 
721     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
722 #if XNN_ARCH_ARM64
723     int16x8_t vacc0x01234567 = vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567);
724     int16x8_t vacc0x89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF);
725     int16x8_t vacc1x01234567 = vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567);
726     int16x8_t vacc1x89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc1x89AB), vacc1xCDEF);
727     int16x8_t vacc2x01234567 = vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567);
728     int16x8_t vacc2x89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc2x89AB), vacc2xCDEF);
729     int16x8_t vacc3x01234567 = vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567);
730     int16x8_t vacc3x89ABCDEF = vqmovn_high_s32(vqmovn_s32(vacc3x89AB), vacc3xCDEF);
731 
732     vacc0x01234567 = vqaddq_s16(vacc0x01234567, voutput_zero_point);
733     vacc0x89ABCDEF = vqaddq_s16(vacc0x89ABCDEF, voutput_zero_point);
734     vacc1x01234567 = vqaddq_s16(vacc1x01234567, voutput_zero_point);
735     vacc1x89ABCDEF = vqaddq_s16(vacc1x89ABCDEF, voutput_zero_point);
736     vacc2x01234567 = vqaddq_s16(vacc2x01234567, voutput_zero_point);
737     vacc2x89ABCDEF = vqaddq_s16(vacc2x89ABCDEF, voutput_zero_point);
738     vacc3x01234567 = vqaddq_s16(vacc3x01234567, voutput_zero_point);
739     vacc3x89ABCDEF = vqaddq_s16(vacc3x89ABCDEF, voutput_zero_point);
740 
741     int8x16_t vout0x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
742     int8x16_t vout1x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc1x01234567), vacc1x89ABCDEF);
743     int8x16_t vout2x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc2x89ABCDEF);
744     int8x16_t vout3x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc3x01234567), vacc3x89ABCDEF);
745 #else
746     int16x8_t vacc0x01234567 = vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567));
747     int16x8_t vacc0x89ABCDEF = vcombine_s16(vqmovn_s32(vacc0x89AB), vqmovn_s32(vacc0xCDEF));
748     int16x8_t vacc1x01234567 = vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567));
749     int16x8_t vacc1x89ABCDEF = vcombine_s16(vqmovn_s32(vacc1x89AB), vqmovn_s32(vacc1xCDEF));
750     int16x8_t vacc2x01234567 = vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567));
751     int16x8_t vacc2x89ABCDEF = vcombine_s16(vqmovn_s32(vacc2x89AB), vqmovn_s32(vacc2xCDEF));
752     int16x8_t vacc3x01234567 = vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567));
753     int16x8_t vacc3x89ABCDEF = vcombine_s16(vqmovn_s32(vacc3x89AB), vqmovn_s32(vacc3xCDEF));
754 
755     vacc0x01234567 = vqaddq_s16(vacc0x01234567, voutput_zero_point);
756     vacc0x89ABCDEF = vqaddq_s16(vacc0x89ABCDEF, voutput_zero_point);
757     vacc1x01234567 = vqaddq_s16(vacc1x01234567, voutput_zero_point);
758     vacc1x89ABCDEF = vqaddq_s16(vacc1x89ABCDEF, voutput_zero_point);
759     vacc2x01234567 = vqaddq_s16(vacc2x01234567, voutput_zero_point);
760     vacc2x89ABCDEF = vqaddq_s16(vacc2x89ABCDEF, voutput_zero_point);
761     vacc3x01234567 = vqaddq_s16(vacc3x01234567, voutput_zero_point);
762     vacc3x89ABCDEF = vqaddq_s16(vacc3x89ABCDEF, voutput_zero_point);
763 
764     int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
765     int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
766     int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
767     int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
768 #endif
769 
770     const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
771     vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
772     vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
773     vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
774     vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
775 
776     const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
777     vout0x0123456789ABCDEF = vminq_s8(vout0x0123456789ABCDEF, voutput_max);
778     vout1x0123456789ABCDEF = vminq_s8(vout1x0123456789ABCDEF, voutput_max);
779     vout2x0123456789ABCDEF = vminq_s8(vout2x0123456789ABCDEF, voutput_max);
780     vout3x0123456789ABCDEF = vminq_s8(vout3x0123456789ABCDEF, voutput_max);
781 
782     if (nc >= 16) {
783       vst1q_s8(c0 + 0, vout0x0123456789ABCDEF);
784       vst1q_s8(c1 + 0, vout1x0123456789ABCDEF);
785       vst1q_s8(c2 + 0, vout2x0123456789ABCDEF);
786       vst1q_s8(c3 + 0, vout3x0123456789ABCDEF);
787 
788       c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
789       c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
790       c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
791       c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
792 
793       a0 = (const int8_t*) ((uintptr_t) a0 - kc);
794       a1 = (const int8_t*) ((uintptr_t) a1 - kc);
795       a2 = (const int8_t*) ((uintptr_t) a2 - kc);
796       a3 = (const int8_t*) ((uintptr_t) a3 - kc);
797 
798       nc -= 16;
799     } else {
800       // Final case where not all of the 16 columns fit in the destination.
801       int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vget_low_s8(vout0x0123456789ABCDEF), vget_low_s8(vout1x0123456789ABCDEF));
802       int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vget_low_s8(vout2x0123456789ABCDEF), vget_low_s8(vout3x0123456789ABCDEF));
803       if (nc & 8) {
804         vst1_s8(c0, vget_low_s8(vout0x01234567_1x01234567)); c0 += 8;
805         vst1_s8(c1, vget_high_s8(vout0x01234567_1x01234567)); c1 += 8;
806         vst1_s8(c2, vget_low_s8(vout2x01234567_3x01234567)); c2 += 8;
807         vst1_s8(c3, vget_high_s8(vout2x01234567_3x01234567)); c3 += 8;
808         vout0x01234567_1x01234567 = vcombine_s8(vget_high_s8(vout0x0123456789ABCDEF), vget_high_s8(vout1x0123456789ABCDEF));
809         vout2x01234567_3x01234567 = vcombine_s8(vget_high_s8(vout2x0123456789ABCDEF), vget_high_s8(vout3x0123456789ABCDEF));
810       }
811       if (nc & 4) {
812         vst1q_lane_u32((void*) c0, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;
813         vst1q_lane_u32((void*) c1, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;
814         vst1q_lane_u32((void*) c2, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4;
815         vst1q_lane_u32((void*) c3, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4;
816         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);
817         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4);
818       }
819       if (nc & 2) {
820         vst1q_lane_u16((void*) c0, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;
821         vst1q_lane_u16((void*) c1, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;
822         vst1q_lane_u16((void*) c2, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2;
823         vst1q_lane_u16((void*) c3, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2;
824         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);
825         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2);
826       }
827       if (nc & 1) {
828         vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);
829         vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);
830         vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0);
831         vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8);
832       }
833 
834       nc = 0;
835     }
836   } while (nc != 0);
837 }
838