xref: /aosp_15_r20/external/XNNPACK/src/qs8-gemm/gen/4x8c2-minmax-rndnu-neon-mlal-ld1r.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-gemm/c2-neon-mull-dup.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/gemm.h>
15 #include <xnnpack/math.h>
16 
xnn_qs8_gemm_minmax_rndnu_ukernel_4x8c2__neon_mlal_ld1r(size_t mr,size_t nc,size_t kc,const int8_t * restrict a,size_t a_stride,const void * restrict w,int8_t * restrict c,size_t cm_stride,size_t cn_stride,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_gemm_minmax_rndnu_ukernel_4x8c2__neon_mlal_ld1r(
18     size_t mr,
19     size_t nc,
20     size_t kc,
21     const int8_t* restrict a,
22     size_t a_stride,
23     const void* restrict w,
24     int8_t* restrict c,
25     size_t cm_stride,
26     size_t cn_stride,
27     const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(mr != 0);
30   assert(mr <= 4);
31   assert(nc != 0);
32   assert(kc != 0);
33   assert(kc % sizeof(int8_t) == 0);
34   assert(a != NULL);
35   assert(w != NULL);
36   assert(c != NULL);
37 
38   kc = round_up_po2(kc, 2 * sizeof(int8_t));
39   const int8_t* a0 = a;
40   int8_t* c0 = c;
41   const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
42   int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
43   if XNN_UNPREDICTABLE(mr < 2) {
44     a1 = a0;
45     c1 = c0;
46   }
47   const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
48   int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
49   if XNN_UNPREDICTABLE(mr <= 2) {
50     a2 = a1;
51     c2 = c1;
52   }
53   const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
54   int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
55   if XNN_UNPREDICTABLE(mr != 4) {
56     a3 = a2;
57     c3 = c2;
58   }
59 
60   do {
61     int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
62     int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
63     int32x4_t vacc1x0123 = vacc0x0123;
64     int32x4_t vacc1x4567 = vacc0x4567;
65     int32x4_t vacc2x0123 = vacc0x0123;
66     int32x4_t vacc2x4567 = vacc0x4567;
67     int32x4_t vacc3x0123 = vacc0x0123;
68     int32x4_t vacc3x4567 = vacc0x4567;
69 
70     size_t k = kc;
71 
72     while (k >= 16 * sizeof(int8_t)) {
73       const int16x4_t va00x0 = vld1_dup_s16((const void*)a0);
74       const int16x4_t va01x0 = vld1_dup_s16((const void*)(a0 + 2));
75       const int16x4_t va02x0 = vld1_dup_s16((const void*)(a0 + 4));
76       const int16x4_t va03x0 = vld1_dup_s16((const void*)(a0 + 6)); a0 += 8;
77       const int16x4_t va00x1 = vld1_dup_s16((const void*)a0);
78       const int16x4_t va01x1 = vld1_dup_s16((const void*)(a0 + 2));
79       const int16x4_t va02x1 = vld1_dup_s16((const void*)(a0 + 4));
80       const int16x4_t va03x1 = vld1_dup_s16((const void*)(a0 + 6)); a0 += 8;
81       const int16x4_t va10x0 = vld1_dup_s16((const void*)a1);
82       const int16x4_t va11x0 = vld1_dup_s16((const void*)(a1 + 2));
83       const int16x4_t va12x0 = vld1_dup_s16((const void*)(a1 + 4));
84       const int16x4_t va13x0 = vld1_dup_s16((const void*)(a1 + 6)); a1 += 8;
85       const int16x4_t va10x1 = vld1_dup_s16((const void*)a1);
86       const int16x4_t va11x1 = vld1_dup_s16((const void*)(a1 + 2));
87       const int16x4_t va12x1 = vld1_dup_s16((const void*)(a1 + 4));
88       const int16x4_t va13x1 = vld1_dup_s16((const void*)(a1 + 6)); a1 += 8;
89       const int16x4_t va20x0 = vld1_dup_s16((const void*)a2);
90       const int16x4_t va21x0 = vld1_dup_s16((const void*)(a2 + 2));
91       const int16x4_t va22x0 = vld1_dup_s16((const void*)(a2 + 4));
92       const int16x4_t va23x0 = vld1_dup_s16((const void*)(a2 + 6)); a2 += 8;
93       const int16x4_t va20x1 = vld1_dup_s16((const void*)a2);
94       const int16x4_t va21x1 = vld1_dup_s16((const void*)(a2 + 2));
95       const int16x4_t va22x1 = vld1_dup_s16((const void*)(a2 + 4));
96       const int16x4_t va23x1 = vld1_dup_s16((const void*)(a2 + 6)); a2 += 8;
97       const int16x4_t va30x0 = vld1_dup_s16((const void*)a3);
98       const int16x4_t va31x0 = vld1_dup_s16((const void*)(a3 + 2));
99       const int16x4_t va32x0 = vld1_dup_s16((const void*)(a3 + 4));
100       const int16x4_t va33x0 = vld1_dup_s16((const void*)(a3 + 6)); a3 += 8;
101       const int16x4_t va30x1 = vld1_dup_s16((const void*)a3);
102       const int16x4_t va31x1 = vld1_dup_s16((const void*)(a3 + 2));
103       const int16x4_t va32x1 = vld1_dup_s16((const void*)(a3 + 4));
104       const int16x4_t va33x1 = vld1_dup_s16((const void*)(a3 + 6)); a3 += 8;
105       const int8x8_t vb0123c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
106       const int8x8_t vb4567c0x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
107       const int8x8_t vb0123c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
108       const int8x8_t vb4567c1x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
109       const int8x8_t vb0123c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
110       const int8x8_t vb4567c2x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
111       const int8x8_t vb0123c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
112       const int8x8_t vb4567c3x0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
113 
114       const int8x8_t va0c0x0 = vreinterpret_s8_s16(va00x0);
115       const int8x8_t va0c0x1 = vreinterpret_s8_s16(va00x1);
116       const int8x8_t va1c0x0 = vreinterpret_s8_s16(va10x0);
117       const int8x8_t va1c0x1 = vreinterpret_s8_s16(va10x1);
118       const int8x8_t va2c0x0 = vreinterpret_s8_s16(va20x0);
119       const int8x8_t va2c0x1 = vreinterpret_s8_s16(va20x1);
120       const int8x8_t va3c0x0 = vreinterpret_s8_s16(va30x0);
121       const int8x8_t va3c0x1 = vreinterpret_s8_s16(va30x1);
122 
123       int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0x0, va0c0x0);
124       int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0x0, va1c0x0);
125       int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0x0, va2c0x0);
126       int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0x0, va3c0x0);
127       const int8x8_t vb0123c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
128       vprod0x0123c0 = vmlal_s8(vprod0x0123c0, vb0123c0x1, va0c0x1);
129       vprod1x0123c0 = vmlal_s8(vprod1x0123c0, vb0123c0x1, va1c0x1);
130       vprod2x0123c0 = vmlal_s8(vprod2x0123c0, vb0123c0x1, va2c0x1);
131       vprod3x0123c0 = vmlal_s8(vprod3x0123c0, vb0123c0x1, va3c0x1);
132       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
133       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
134       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
135       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
136       int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0x0, va0c0x0);
137       int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0x0, va1c0x0);
138       int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0x0, va2c0x0);
139       int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0x0, va3c0x0);
140       const int8x8_t vb4567c0x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
141       vprod0x4567c0 = vmlal_s8(vprod0x4567c0, vb4567c0x1, va0c0x1);
142       vprod1x4567c0 = vmlal_s8(vprod1x4567c0, vb4567c0x1, va1c0x1);
143       vprod2x4567c0 = vmlal_s8(vprod2x4567c0, vb4567c0x1, va2c0x1);
144       vprod3x4567c0 = vmlal_s8(vprod3x4567c0, vb4567c0x1, va3c0x1);
145       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
146       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
147       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
148       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
149       const int8x8_t va0c1x0 = vreinterpret_s8_s16(va01x0);
150       const int8x8_t va0c1x1 = vreinterpret_s8_s16(va01x1);
151       const int8x8_t va1c1x0 = vreinterpret_s8_s16(va11x0);
152       const int8x8_t va1c1x1 = vreinterpret_s8_s16(va11x1);
153       const int8x8_t va2c1x0 = vreinterpret_s8_s16(va21x0);
154       const int8x8_t va2c1x1 = vreinterpret_s8_s16(va21x1);
155       const int8x8_t va3c1x0 = vreinterpret_s8_s16(va31x0);
156       const int8x8_t va3c1x1 = vreinterpret_s8_s16(va31x1);
157 
158       int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1x0, va0c1x0);
159       int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1x0, va1c1x0);
160       int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1x0, va2c1x0);
161       int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1x0, va3c1x0);
162       const int8x8_t vb0123c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
163       vprod0x0123c1 = vmlal_s8(vprod0x0123c1, vb0123c1x1, va0c1x1);
164       vprod1x0123c1 = vmlal_s8(vprod1x0123c1, vb0123c1x1, va1c1x1);
165       vprod2x0123c1 = vmlal_s8(vprod2x0123c1, vb0123c1x1, va2c1x1);
166       vprod3x0123c1 = vmlal_s8(vprod3x0123c1, vb0123c1x1, va3c1x1);
167       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
168       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
169       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
170       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
171       int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1x0, va0c1x0);
172       int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1x0, va1c1x0);
173       int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1x0, va2c1x0);
174       int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1x0, va3c1x0);
175       const int8x8_t vb4567c1x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
176       vprod0x4567c1 = vmlal_s8(vprod0x4567c1, vb4567c1x1, va0c1x1);
177       vprod1x4567c1 = vmlal_s8(vprod1x4567c1, vb4567c1x1, va1c1x1);
178       vprod2x4567c1 = vmlal_s8(vprod2x4567c1, vb4567c1x1, va2c1x1);
179       vprod3x4567c1 = vmlal_s8(vprod3x4567c1, vb4567c1x1, va3c1x1);
180       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
181       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
182       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
183       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
184       const int8x8_t va0c2x0 = vreinterpret_s8_s16(va02x0);
185       const int8x8_t va0c2x1 = vreinterpret_s8_s16(va02x1);
186       const int8x8_t va1c2x0 = vreinterpret_s8_s16(va12x0);
187       const int8x8_t va1c2x1 = vreinterpret_s8_s16(va12x1);
188       const int8x8_t va2c2x0 = vreinterpret_s8_s16(va22x0);
189       const int8x8_t va2c2x1 = vreinterpret_s8_s16(va22x1);
190       const int8x8_t va3c2x0 = vreinterpret_s8_s16(va32x0);
191       const int8x8_t va3c2x1 = vreinterpret_s8_s16(va32x1);
192 
193       int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2x0, va0c2x0);
194       int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2x0, va1c2x0);
195       int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2x0, va2c2x0);
196       int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2x0, va3c2x0);
197       const int8x8_t vb0123c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
198       vprod0x0123c2 = vmlal_s8(vprod0x0123c2, vb0123c2x1, va0c2x1);
199       vprod1x0123c2 = vmlal_s8(vprod1x0123c2, vb0123c2x1, va1c2x1);
200       vprod2x0123c2 = vmlal_s8(vprod2x0123c2, vb0123c2x1, va2c2x1);
201       vprod3x0123c2 = vmlal_s8(vprod3x0123c2, vb0123c2x1, va3c2x1);
202       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
203       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
204       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
205       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
206       int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2x0, va0c2x0);
207       int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2x0, va1c2x0);
208       int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2x0, va2c2x0);
209       int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2x0, va3c2x0);
210       const int8x8_t vb4567c2x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
211       vprod0x4567c2 = vmlal_s8(vprod0x4567c2, vb4567c2x1, va0c2x1);
212       vprod1x4567c2 = vmlal_s8(vprod1x4567c2, vb4567c2x1, va1c2x1);
213       vprod2x4567c2 = vmlal_s8(vprod2x4567c2, vb4567c2x1, va2c2x1);
214       vprod3x4567c2 = vmlal_s8(vprod3x4567c2, vb4567c2x1, va3c2x1);
215       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
216       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
217       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
218       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
219       const int8x8_t va0c3x0 = vreinterpret_s8_s16(va03x0);
220       const int8x8_t va0c3x1 = vreinterpret_s8_s16(va03x1);
221       const int8x8_t va1c3x0 = vreinterpret_s8_s16(va13x0);
222       const int8x8_t va1c3x1 = vreinterpret_s8_s16(va13x1);
223       const int8x8_t va2c3x0 = vreinterpret_s8_s16(va23x0);
224       const int8x8_t va2c3x1 = vreinterpret_s8_s16(va23x1);
225       const int8x8_t va3c3x0 = vreinterpret_s8_s16(va33x0);
226       const int8x8_t va3c3x1 = vreinterpret_s8_s16(va33x1);
227 
228       int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3x0, va0c3x0);
229       int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3x0, va1c3x0);
230       int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3x0, va2c3x0);
231       int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3x0, va3c3x0);
232       const int8x8_t vb0123c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
233       vprod0x0123c3 = vmlal_s8(vprod0x0123c3, vb0123c3x1, va0c3x1);
234       vprod1x0123c3 = vmlal_s8(vprod1x0123c3, vb0123c3x1, va1c3x1);
235       vprod2x0123c3 = vmlal_s8(vprod2x0123c3, vb0123c3x1, va2c3x1);
236       vprod3x0123c3 = vmlal_s8(vprod3x0123c3, vb0123c3x1, va3c3x1);
237       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
238       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
239       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
240       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
241       int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3x0, va0c3x0);
242       int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3x0, va1c3x0);
243       int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3x0, va2c3x0);
244       int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3x0, va3c3x0);
245       const int8x8_t vb4567c3x1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
246       vprod0x4567c3 = vmlal_s8(vprod0x4567c3, vb4567c3x1, va0c3x1);
247       vprod1x4567c3 = vmlal_s8(vprod1x4567c3, vb4567c3x1, va1c3x1);
248       vprod2x4567c3 = vmlal_s8(vprod2x4567c3, vb4567c3x1, va2c3x1);
249       vprod3x4567c3 = vmlal_s8(vprod3x4567c3, vb4567c3x1, va3c3x1);
250       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
251       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
252       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
253       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
254 
255       k -= 16 * sizeof(int8_t);
256     }
257 
258     if (k >= 8 * sizeof(int8_t)) {
259       const int16x4_t va00 = vld1_dup_s16((const void*)a0);
260       const int16x4_t va01 = vld1_dup_s16((const void*)(a0 + 2));
261       const int16x4_t va02 = vld1_dup_s16((const void*)(a0 + 4));
262       const int16x4_t va03 = vld1_dup_s16((const void*)(a0 + 6)); a0 += 8;
263       const int16x4_t va10 = vld1_dup_s16((const void*)a1);
264       const int16x4_t va11 = vld1_dup_s16((const void*)(a1 + 2));
265       const int16x4_t va12 = vld1_dup_s16((const void*)(a1 + 4));
266       const int16x4_t va13 = vld1_dup_s16((const void*)(a1 + 6)); a1 += 8;
267       const int16x4_t va20 = vld1_dup_s16((const void*)a2);
268       const int16x4_t va21 = vld1_dup_s16((const void*)(a2 + 2));
269       const int16x4_t va22 = vld1_dup_s16((const void*)(a2 + 4));
270       const int16x4_t va23 = vld1_dup_s16((const void*)(a2 + 6)); a2 += 8;
271       const int16x4_t va30 = vld1_dup_s16((const void*)a3);
272       const int16x4_t va31 = vld1_dup_s16((const void*)(a3 + 2));
273       const int16x4_t va32 = vld1_dup_s16((const void*)(a3 + 4));
274       const int16x4_t va33 = vld1_dup_s16((const void*)(a3 + 6)); a3 += 8;
275 
276       const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
277       const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
278       const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
279       const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
280       const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
281       const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
282       const int8x8_t vb0123c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
283       const int8x8_t vb4567c3 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
284 
285       const int8x8_t va0c0 = vreinterpret_s8_s16(va00);
286       const int8x8_t va1c0 = vreinterpret_s8_s16(va10);
287       const int8x8_t va2c0 = vreinterpret_s8_s16(va20);
288       const int8x8_t va3c0 = vreinterpret_s8_s16(va30);
289 
290       const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);
291       const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);
292       const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);
293       const int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0, va3c0);
294       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
295       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
296       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
297       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
298       const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);
299       const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);
300       const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);
301       const int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0, va3c0);
302       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
303       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
304       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
305       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
306       const int8x8_t va0c1 = vreinterpret_s8_s16(va01);
307       const int8x8_t va1c1 = vreinterpret_s8_s16(va11);
308       const int8x8_t va2c1 = vreinterpret_s8_s16(va21);
309       const int8x8_t va3c1 = vreinterpret_s8_s16(va31);
310 
311       const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);
312       const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);
313       const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);
314       const int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1, va3c1);
315       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
316       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
317       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
318       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
319       const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);
320       const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);
321       const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);
322       const int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1, va3c1);
323       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
324       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
325       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
326       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
327       const int8x8_t va0c2 = vreinterpret_s8_s16(va02);
328       const int8x8_t va1c2 = vreinterpret_s8_s16(va12);
329       const int8x8_t va2c2 = vreinterpret_s8_s16(va22);
330       const int8x8_t va3c2 = vreinterpret_s8_s16(va32);
331 
332       const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);
333       const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);
334       const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);
335       const int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2, va3c2);
336       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
337       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
338       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
339       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
340       const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);
341       const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);
342       const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);
343       const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, va3c2);
344       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
345       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
346       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
347       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
348       const int8x8_t va0c3 = vreinterpret_s8_s16(va03);
349       const int8x8_t va1c3 = vreinterpret_s8_s16(va13);
350       const int8x8_t va2c3 = vreinterpret_s8_s16(va23);
351       const int8x8_t va3c3 = vreinterpret_s8_s16(va33);
352 
353       const int16x8_t vprod0x0123c3 = vmull_s8(vb0123c3, va0c3);
354       const int16x8_t vprod1x0123c3 = vmull_s8(vb0123c3, va1c3);
355       const int16x8_t vprod2x0123c3 = vmull_s8(vb0123c3, va2c3);
356       const int16x8_t vprod3x0123c3 = vmull_s8(vb0123c3, va3c3);
357       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c3);
358       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c3);
359       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c3);
360       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c3);
361       const int16x8_t vprod0x4567c3 = vmull_s8(vb4567c3, va0c3);
362       const int16x8_t vprod1x4567c3 = vmull_s8(vb4567c3, va1c3);
363       const int16x8_t vprod2x4567c3 = vmull_s8(vb4567c3, va2c3);
364       const int16x8_t vprod3x4567c3 = vmull_s8(vb4567c3, va3c3);
365       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c3);
366       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c3);
367       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c3);
368       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c3);
369 
370       k -= 8 * sizeof(int8_t);
371     }
372 
373     if XNN_UNLIKELY(k != 0) {
374       const int8x8_t va0 = vld1_s8(a0); a0 = (const int8_t*) ((uintptr_t) a0 + k);
375       const int8x8_t va1 = vld1_s8(a1); a1 = (const int8_t*) ((uintptr_t) a1 + k);
376       const int8x8_t va2 = vld1_s8(a2); a2 = (const int8_t*) ((uintptr_t) a2 + k);
377       const int8x8_t va3 = vld1_s8(a3); a3 = (const int8_t*) ((uintptr_t) a3 + k);
378 
379       const int8x8_t vb0123c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
380       const int8x8_t vb4567c0 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
381 
382       const int8x8_t va0c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 0));
383       const int16x8_t vprod0x0123c0 = vmull_s8(vb0123c0, va0c0);
384       vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c0);
385       const int16x8_t vprod0x4567c0 = vmull_s8(vb4567c0, va0c0);
386       vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c0);
387       const int8x8_t va1c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 0));
388       const int16x8_t vprod1x0123c0 = vmull_s8(vb0123c0, va1c0);
389       vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c0);
390       const int16x8_t vprod1x4567c0 = vmull_s8(vb4567c0, va1c0);
391       vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c0);
392       const int8x8_t va2c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 0));
393       const int16x8_t vprod2x0123c0 = vmull_s8(vb0123c0, va2c0);
394       vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c0);
395       const int16x8_t vprod2x4567c0 = vmull_s8(vb4567c0, va2c0);
396       vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c0);
397       const int8x8_t va3c0 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 0));
398       const int16x8_t vprod3x0123c0 = vmull_s8(vb0123c0, va3c0);
399       vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c0);
400       const int16x8_t vprod3x4567c0 = vmull_s8(vb4567c0, va3c0);
401       vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c0);
402 
403       if (k > 2 * sizeof(int8_t)) {
404         const int8x8_t vb0123c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
405         const int8x8_t vb4567c1 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
406 
407         const int8x8_t va0c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 1));
408         const int16x8_t vprod0x0123c1 = vmull_s8(vb0123c1, va0c1);
409         vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c1);
410         const int16x8_t vprod0x4567c1 = vmull_s8(vb4567c1, va0c1);
411         vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c1);
412         const int8x8_t va1c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 1));
413         const int16x8_t vprod1x0123c1 = vmull_s8(vb0123c1, va1c1);
414         vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c1);
415         const int16x8_t vprod1x4567c1 = vmull_s8(vb4567c1, va1c1);
416         vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c1);
417         const int8x8_t va2c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 1));
418         const int16x8_t vprod2x0123c1 = vmull_s8(vb0123c1, va2c1);
419         vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c1);
420         const int16x8_t vprod2x4567c1 = vmull_s8(vb4567c1, va2c1);
421         vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c1);
422         const int8x8_t va3c1 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 1));
423         const int16x8_t vprod3x0123c1 = vmull_s8(vb0123c1, va3c1);
424         vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c1);
425         const int16x8_t vprod3x4567c1 = vmull_s8(vb4567c1, va3c1);
426         vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c1);
427 
428         if (k > 4 * sizeof(int8_t)) {
429           const int8x8_t vb0123c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
430           const int8x8_t vb4567c2 = vld1_s8(w); w = (const void*) ((uintptr_t) w + 8 * sizeof(int8_t));
431 
432           const int8x8_t va0c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va0), 2));
433           const int16x8_t vprod0x0123c2 = vmull_s8(vb0123c2, va0c2);
434           vacc0x0123 = vpadalq_s16(vacc0x0123, vprod0x0123c2);
435           const int16x8_t vprod0x4567c2 = vmull_s8(vb4567c2, va0c2);
436           vacc0x4567 = vpadalq_s16(vacc0x4567, vprod0x4567c2);
437           const int8x8_t va1c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va1), 2));
438           const int16x8_t vprod1x0123c2 = vmull_s8(vb0123c2, va1c2);
439           vacc1x0123 = vpadalq_s16(vacc1x0123, vprod1x0123c2);
440           const int16x8_t vprod1x4567c2 = vmull_s8(vb4567c2, va1c2);
441           vacc1x4567 = vpadalq_s16(vacc1x4567, vprod1x4567c2);
442           const int8x8_t va2c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va2), 2));
443           const int16x8_t vprod2x0123c2 = vmull_s8(vb0123c2, va2c2);
444           vacc2x0123 = vpadalq_s16(vacc2x0123, vprod2x0123c2);
445           const int16x8_t vprod2x4567c2 = vmull_s8(vb4567c2, va2c2);
446           vacc2x4567 = vpadalq_s16(vacc2x4567, vprod2x4567c2);
447           const int8x8_t va3c2 = vreinterpret_s8_s16(vdup_lane_s16(vreinterpret_s16_s8(va3), 2));
448           const int16x8_t vprod3x0123c2 = vmull_s8(vb0123c2, va3c2);
449           vacc3x0123 = vpadalq_s16(vacc3x0123, vprod3x0123c2);
450           const int16x8_t vprod3x4567c2 = vmull_s8(vb4567c2, va3c2);
451           vacc3x4567 = vpadalq_s16(vacc3x4567, vprod3x4567c2);
452         }
453       }
454     }
455 
456     const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
457     const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
458     const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
459 
460     vacc0x0123 = vqshlq_s32(vacc0x0123, vright_pre_shift);
461     vacc0x4567 = vqshlq_s32(vacc0x4567, vright_pre_shift);
462     vacc1x0123 = vqshlq_s32(vacc1x0123, vright_pre_shift);
463     vacc1x4567 = vqshlq_s32(vacc1x4567, vright_pre_shift);
464     vacc2x0123 = vqshlq_s32(vacc2x0123, vright_pre_shift);
465     vacc2x4567 = vqshlq_s32(vacc2x4567, vright_pre_shift);
466     vacc3x0123 = vqshlq_s32(vacc3x0123, vright_pre_shift);
467     vacc3x4567 = vqshlq_s32(vacc3x4567, vright_pre_shift);
468 
469     vacc0x0123 = vqdmulhq_s32(vacc0x0123, vmultiplier);
470     vacc0x4567 = vqdmulhq_s32(vacc0x4567, vmultiplier);
471     vacc1x0123 = vqdmulhq_s32(vacc1x0123, vmultiplier);
472     vacc1x4567 = vqdmulhq_s32(vacc1x4567, vmultiplier);
473     vacc2x0123 = vqdmulhq_s32(vacc2x0123, vmultiplier);
474     vacc2x4567 = vqdmulhq_s32(vacc2x4567, vmultiplier);
475     vacc3x0123 = vqdmulhq_s32(vacc3x0123, vmultiplier);
476     vacc3x4567 = vqdmulhq_s32(vacc3x4567, vmultiplier);
477 
478     vacc0x0123 = vrshlq_s32(vacc0x0123, vright_post_shift);
479     vacc0x4567 = vrshlq_s32(vacc0x4567, vright_post_shift);
480     vacc1x0123 = vrshlq_s32(vacc1x0123, vright_post_shift);
481     vacc1x4567 = vrshlq_s32(vacc1x4567, vright_post_shift);
482     vacc2x0123 = vrshlq_s32(vacc2x0123, vright_post_shift);
483     vacc2x4567 = vrshlq_s32(vacc2x4567, vright_post_shift);
484     vacc3x0123 = vrshlq_s32(vacc3x0123, vright_post_shift);
485     vacc3x4567 = vrshlq_s32(vacc3x4567, vright_post_shift);
486 
487     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
488 #if XNN_ARCH_ARM64
489     int16x8_t vacc0x01234567 = vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567);
490     int16x8_t vacc1x01234567 = vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567);
491     int16x8_t vacc2x01234567 = vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567);
492     int16x8_t vacc3x01234567 = vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567);
493 
494     vacc0x01234567 = vqaddq_s16(vacc0x01234567, voutput_zero_point);
495     vacc1x01234567 = vqaddq_s16(vacc1x01234567, voutput_zero_point);
496     vacc2x01234567 = vqaddq_s16(vacc2x01234567, voutput_zero_point);
497     vacc3x01234567 = vqaddq_s16(vacc3x01234567, voutput_zero_point);
498 
499     int8x16_t vout0x01234567_1x01234567 = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc1x01234567);
500     int8x16_t vout2x01234567_3x01234567 = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc3x01234567);
501 #else
502     int16x8_t vacc0x01234567 = vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567));
503     int16x8_t vacc1x01234567 = vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567));
504     int16x8_t vacc2x01234567 = vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567));
505     int16x8_t vacc3x01234567 = vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567));
506 
507     vacc0x01234567 = vqaddq_s16(vacc0x01234567, voutput_zero_point);
508     vacc1x01234567 = vqaddq_s16(vacc1x01234567, voutput_zero_point);
509     vacc2x01234567 = vqaddq_s16(vacc2x01234567, voutput_zero_point);
510     vacc3x01234567 = vqaddq_s16(vacc3x01234567, voutput_zero_point);
511 
512     int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
513     int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
514 #endif
515 
516     const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
517     vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
518     vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
519 
520     const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
521     vout0x01234567_1x01234567 = vminq_s8(vout0x01234567_1x01234567, voutput_max);
522     vout2x01234567_3x01234567 = vminq_s8(vout2x01234567_3x01234567, voutput_max);
523 
524     if (nc >= 8) {
525       vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567));
526       vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567));
527       vst1_s8(c2 + 0, vget_low_s8(vout2x01234567_3x01234567));
528       vst1_s8(c3 + 0, vget_high_s8(vout2x01234567_3x01234567));
529 
530       c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
531       c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
532       c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
533       c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
534 
535       a0 = (const int8_t*) ((uintptr_t) a0 - kc);
536       a1 = (const int8_t*) ((uintptr_t) a1 - kc);
537       a2 = (const int8_t*) ((uintptr_t) a2 - kc);
538       a3 = (const int8_t*) ((uintptr_t) a3 - kc);
539 
540       nc -= 8;
541     } else {
542       // Final case where not all of the 8 columns fit in the destination.
543       if (nc & 4) {
544         vst1q_lane_u32((void*) c0, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;
545         vst1q_lane_u32((void*) c1, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;
546         vst1q_lane_u32((void*) c2, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4;
547         vst1q_lane_u32((void*) c3, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4;
548         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);
549         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4);
550       }
551       if (nc & 2) {
552         vst1q_lane_u16((void*) c0, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;
553         vst1q_lane_u16((void*) c1, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;
554         vst1q_lane_u16((void*) c2, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2;
555         vst1q_lane_u16((void*) c3, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2;
556         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);
557         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2);
558       }
559       if (nc & 1) {
560         vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);
561         vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);
562         vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0);
563         vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8);
564       }
565 
566       nc = 0;
567     }
568   } while (nc != 0);
569 }
570