xref: /aosp_15_r20/external/XNNPACK/src/qs8-igemm/gen/8x16c4-minmax-rndnu-neondot.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/qs8-igemm/c4-neondot.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/igemm.h>
15 #include <xnnpack/math.h>
16 
17 
xnn_qs8_igemm_minmax_rndnu_ukernel_8x16c4__neondot(size_t mr,size_t nc,size_t kc,size_t ks,const int8_t ** restrict a,const void * restrict w,int8_t * restrict c,size_t cm_stride,size_t cn_stride,size_t a_offset,const int8_t * zero,const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])18 void xnn_qs8_igemm_minmax_rndnu_ukernel_8x16c4__neondot(
19     size_t mr,
20     size_t nc,
21     size_t kc,
22     size_t ks,
23     const int8_t** restrict a,
24     const void* restrict w,
25     int8_t* restrict c,
26     size_t cm_stride,
27     size_t cn_stride,
28     size_t a_offset,
29     const int8_t* zero,
30     const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31 {
32   assert(mr != 0);
33   assert(mr <= 8);
34   assert(nc != 0);
35   assert(kc != 0);
36   assert(ks != 0);
37   assert(ks % (8 * sizeof(void*)) == 0);
38   assert(a_offset % sizeof(int8_t) == 0);
39   assert(a != NULL);
40   assert(w != NULL);
41   assert(c != NULL);
42 
43   kc = round_up_po2(kc, 4 * sizeof(int8_t));
44   int8_t* c0 = c;
45   int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
46   if XNN_UNPREDICTABLE(mr < 2) {
47     c1 = c0;
48   }
49   int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
50   if XNN_UNPREDICTABLE(mr <= 2) {
51     c2 = c1;
52   }
53   int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
54   if XNN_UNPREDICTABLE(mr < 4) {
55     c3 = c2;
56   }
57   int8_t* c4 = (int8_t*) ((uintptr_t) c3 + cm_stride);
58   if XNN_UNPREDICTABLE(mr <= 4) {
59     c4 = c3;
60   }
61   int8_t* c5 = (int8_t*) ((uintptr_t) c4 + cm_stride);
62   if XNN_UNPREDICTABLE(mr < 6) {
63     c5 = c4;
64   }
65   int8_t* c6 = (int8_t*) ((uintptr_t) c5 + cm_stride);
66   if XNN_UNPREDICTABLE(mr <= 6) {
67     c6 = c5;
68   }
69   int8_t* c7 = (int8_t*) ((uintptr_t) c6 + cm_stride);
70   if XNN_UNPREDICTABLE(mr != 8) {
71     c7 = c6;
72   }
73 
74   do {
75     int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
76     int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
77     int32x4_t vacc0x89AB = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
78     int32x4_t vacc0xCDEF = vld1q_s32(w); w = (const void*) ((const int32_t*) w + 4);
79     int32x4_t vacc1x0123 = vacc0x0123;
80     int32x4_t vacc1x4567 = vacc0x4567;
81     int32x4_t vacc1x89AB = vacc0x89AB;
82     int32x4_t vacc1xCDEF = vacc0xCDEF;
83     int32x4_t vacc2x0123 = vacc0x0123;
84     int32x4_t vacc2x4567 = vacc0x4567;
85     int32x4_t vacc2x89AB = vacc0x89AB;
86     int32x4_t vacc2xCDEF = vacc0xCDEF;
87     int32x4_t vacc3x0123 = vacc0x0123;
88     int32x4_t vacc3x4567 = vacc0x4567;
89     int32x4_t vacc3x89AB = vacc0x89AB;
90     int32x4_t vacc3xCDEF = vacc0xCDEF;
91     int32x4_t vacc4x0123 = vacc0x0123;
92     int32x4_t vacc4x4567 = vacc0x4567;
93     int32x4_t vacc4x89AB = vacc0x89AB;
94     int32x4_t vacc4xCDEF = vacc0xCDEF;
95     int32x4_t vacc5x0123 = vacc0x0123;
96     int32x4_t vacc5x4567 = vacc0x4567;
97     int32x4_t vacc5x89AB = vacc0x89AB;
98     int32x4_t vacc5xCDEF = vacc0xCDEF;
99     int32x4_t vacc6x0123 = vacc0x0123;
100     int32x4_t vacc6x4567 = vacc0x4567;
101     int32x4_t vacc6x89AB = vacc0x89AB;
102     int32x4_t vacc6xCDEF = vacc0xCDEF;
103     int32x4_t vacc7x0123 = vacc0x0123;
104     int32x4_t vacc7x4567 = vacc0x4567;
105     int32x4_t vacc7x89AB = vacc0x89AB;
106     int32x4_t vacc7xCDEF = vacc0xCDEF;
107 
108     size_t p = ks;
109     do {
110       const int8_t* restrict a0 = a[0];
111       if XNN_UNPREDICTABLE(a0 != zero) {
112         a0 = (const int8_t*) ((uintptr_t) a0 + a_offset);
113       }
114       const int8_t* restrict a1 = a[1];
115       if XNN_UNPREDICTABLE(a1 != zero) {
116         a1 = (const int8_t*) ((uintptr_t) a1 + a_offset);
117       }
118       const int8_t* restrict a2 = a[2];
119       if XNN_UNPREDICTABLE(a2 != zero) {
120         a2 = (const int8_t*) ((uintptr_t) a2 + a_offset);
121       }
122       const int8_t* restrict a3 = a[3];
123       if XNN_UNPREDICTABLE(a3 != zero) {
124         a3 = (const int8_t*) ((uintptr_t) a3 + a_offset);
125       }
126       const int8_t* restrict a4 = a[4];
127       if XNN_UNPREDICTABLE(a4 != zero) {
128         a4 = (const int8_t*) ((uintptr_t) a4 + a_offset);
129       }
130       const int8_t* restrict a5 = a[5];
131       if XNN_UNPREDICTABLE(a5 != zero) {
132         a5 = (const int8_t*) ((uintptr_t) a5 + a_offset);
133       }
134       const int8_t* restrict a6 = a[6];
135       if XNN_UNPREDICTABLE(a6 != zero) {
136         a6 = (const int8_t*) ((uintptr_t) a6 + a_offset);
137       }
138       const int8_t* restrict a7 = a[7];
139       if XNN_UNPREDICTABLE(a7 != zero) {
140         a7 = (const int8_t*) ((uintptr_t) a7 + a_offset);
141       }
142       a += 8;
143 
144       // Inner accumulation loop along the 16 columns.
145       size_t k = kc;
146       // 2x partial unrolled loop to load 8 bytes at a time.
147       while (k >= 8 * sizeof(int8_t)) {
148         // Load a 8x8 block of activations.
149         const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 8;
150         const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 8;
151         const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 8;
152         const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 8;
153         const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 8;
154         const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 8;
155         const int8x8_t va6x01234567 = vld1_s8(a6); a6 += 8;
156         const int8x8_t va7x01234567 = vld1_s8(a7); a7 += 8;
157 
158         // Load a 8x16 block of weights.
159         const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
160         const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
161         const int8x16_t vb0123x89AB = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
162         const int8x16_t vb0123xCDEF = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
163         const int8x16_t vb4567x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
164         const int8x16_t vb4567x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
165         const int8x16_t vb4567x89AB = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
166         const int8x16_t vb4567xCDEF = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
167 
168         // Multiply-accumulate: 8x8 * 8x16 --> 8x16.
169         vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
170         vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
171         vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb0123x89AB, va0x01234567, 0);
172         vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb0123xCDEF, va0x01234567, 0);
173         vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
174         vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
175         vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb0123x89AB, va1x01234567, 0);
176         vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb0123xCDEF, va1x01234567, 0);
177         vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
178         vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
179         vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb0123x89AB, va2x01234567, 0);
180         vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb0123xCDEF, va2x01234567, 0);
181         vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
182         vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
183         vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb0123x89AB, va3x01234567, 0);
184         vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb0123xCDEF, va3x01234567, 0);
185         vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0);
186         vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0);
187         vacc4x89AB = vdotq_lane_s32(vacc4x89AB, vb0123x89AB, va4x01234567, 0);
188         vacc4xCDEF = vdotq_lane_s32(vacc4xCDEF, vb0123xCDEF, va4x01234567, 0);
189         vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0);
190         vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0);
191         vacc5x89AB = vdotq_lane_s32(vacc5x89AB, vb0123x89AB, va5x01234567, 0);
192         vacc5xCDEF = vdotq_lane_s32(vacc5xCDEF, vb0123xCDEF, va5x01234567, 0);
193         vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0);
194         vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0);
195         vacc6x89AB = vdotq_lane_s32(vacc6x89AB, vb0123x89AB, va6x01234567, 0);
196         vacc6xCDEF = vdotq_lane_s32(vacc6xCDEF, vb0123xCDEF, va6x01234567, 0);
197         vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0);
198         vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0);
199         vacc7x89AB = vdotq_lane_s32(vacc7x89AB, vb0123x89AB, va7x01234567, 0);
200         vacc7xCDEF = vdotq_lane_s32(vacc7xCDEF, vb0123xCDEF, va7x01234567, 0);
201         vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb4567x0123, va0x01234567, 1);
202         vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb4567x4567, va0x01234567, 1);
203         vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb4567x89AB, va0x01234567, 1);
204         vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb4567xCDEF, va0x01234567, 1);
205         vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb4567x0123, va1x01234567, 1);
206         vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb4567x4567, va1x01234567, 1);
207         vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb4567x89AB, va1x01234567, 1);
208         vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb4567xCDEF, va1x01234567, 1);
209         vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb4567x0123, va2x01234567, 1);
210         vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb4567x4567, va2x01234567, 1);
211         vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb4567x89AB, va2x01234567, 1);
212         vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb4567xCDEF, va2x01234567, 1);
213         vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb4567x0123, va3x01234567, 1);
214         vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb4567x4567, va3x01234567, 1);
215         vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb4567x89AB, va3x01234567, 1);
216         vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb4567xCDEF, va3x01234567, 1);
217         vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb4567x0123, va4x01234567, 1);
218         vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb4567x4567, va4x01234567, 1);
219         vacc4x89AB = vdotq_lane_s32(vacc4x89AB, vb4567x89AB, va4x01234567, 1);
220         vacc4xCDEF = vdotq_lane_s32(vacc4xCDEF, vb4567xCDEF, va4x01234567, 1);
221         vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb4567x0123, va5x01234567, 1);
222         vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb4567x4567, va5x01234567, 1);
223         vacc5x89AB = vdotq_lane_s32(vacc5x89AB, vb4567x89AB, va5x01234567, 1);
224         vacc5xCDEF = vdotq_lane_s32(vacc5xCDEF, vb4567xCDEF, va5x01234567, 1);
225         vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb4567x0123, va6x01234567, 1);
226         vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb4567x4567, va6x01234567, 1);
227         vacc6x89AB = vdotq_lane_s32(vacc6x89AB, vb4567x89AB, va6x01234567, 1);
228         vacc6xCDEF = vdotq_lane_s32(vacc6xCDEF, vb4567xCDEF, va6x01234567, 1);
229         vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb4567x0123, va7x01234567, 1);
230         vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb4567x4567, va7x01234567, 1);
231         vacc7x89AB = vdotq_lane_s32(vacc7x89AB, vb4567x89AB, va7x01234567, 1);
232         vacc7xCDEF = vdotq_lane_s32(vacc7xCDEF, vb4567xCDEF, va7x01234567, 1);
233 
234         k -= 8 * sizeof(int8_t);
235       }
236       // Handle up to 4 final positions of `k`
237       if XNN_UNLIKELY(k != 0) {
238         // Load a 8x4 block of activations.
239         const int8x8_t va0x01234567 = vld1_s8(a0);
240         const int8x8_t va1x01234567 = vld1_s8(a1);
241         const int8x8_t va2x01234567 = vld1_s8(a2);
242         const int8x8_t va3x01234567 = vld1_s8(a3);
243         const int8x8_t va4x01234567 = vld1_s8(a4);
244         const int8x8_t va5x01234567 = vld1_s8(a5);
245         const int8x8_t va6x01234567 = vld1_s8(a6);
246         const int8x8_t va7x01234567 = vld1_s8(a7);
247 
248         // Load a 4x16 block of weights.
249         const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
250         const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
251         const int8x16_t vb0123x89AB = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
252         const int8x16_t vb0123xCDEF = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
253 
254         // Multiply-accumulate: 8x4 * 4x16 --> 8x16.
255         vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
256         vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
257         vacc0x89AB = vdotq_lane_s32(vacc0x89AB, vb0123x89AB, va0x01234567, 0);
258         vacc0xCDEF = vdotq_lane_s32(vacc0xCDEF, vb0123xCDEF, va0x01234567, 0);
259         vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
260         vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
261         vacc1x89AB = vdotq_lane_s32(vacc1x89AB, vb0123x89AB, va1x01234567, 0);
262         vacc1xCDEF = vdotq_lane_s32(vacc1xCDEF, vb0123xCDEF, va1x01234567, 0);
263         vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
264         vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
265         vacc2x89AB = vdotq_lane_s32(vacc2x89AB, vb0123x89AB, va2x01234567, 0);
266         vacc2xCDEF = vdotq_lane_s32(vacc2xCDEF, vb0123xCDEF, va2x01234567, 0);
267         vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
268         vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
269         vacc3x89AB = vdotq_lane_s32(vacc3x89AB, vb0123x89AB, va3x01234567, 0);
270         vacc3xCDEF = vdotq_lane_s32(vacc3xCDEF, vb0123xCDEF, va3x01234567, 0);
271         vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0);
272         vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0);
273         vacc4x89AB = vdotq_lane_s32(vacc4x89AB, vb0123x89AB, va4x01234567, 0);
274         vacc4xCDEF = vdotq_lane_s32(vacc4xCDEF, vb0123xCDEF, va4x01234567, 0);
275         vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0);
276         vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0);
277         vacc5x89AB = vdotq_lane_s32(vacc5x89AB, vb0123x89AB, va5x01234567, 0);
278         vacc5xCDEF = vdotq_lane_s32(vacc5xCDEF, vb0123xCDEF, va5x01234567, 0);
279         vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0);
280         vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0);
281         vacc6x89AB = vdotq_lane_s32(vacc6x89AB, vb0123x89AB, va6x01234567, 0);
282         vacc6xCDEF = vdotq_lane_s32(vacc6xCDEF, vb0123xCDEF, va6x01234567, 0);
283         vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0);
284         vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0);
285         vacc7x89AB = vdotq_lane_s32(vacc7x89AB, vb0123x89AB, va7x01234567, 0);
286         vacc7xCDEF = vdotq_lane_s32(vacc7xCDEF, vb0123xCDEF, va7x01234567, 0);
287       }
288       p -= 8 * sizeof(void*);
289     } while (p != 0);
290 
291     const int32x4_t vright_pre_shift = vld1q_dup_s32(&params->rndnu_neon.right_pre_shift);
292     const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
293     const int32x4_t vright_post_shift = vld1q_dup_s32(&params->rndnu_neon.right_post_shift);
294 
295     vacc0x0123 = vshlq_s32(vacc0x0123, vright_pre_shift);
296     vacc0x4567 = vshlq_s32(vacc0x4567, vright_pre_shift);
297     vacc0x89AB = vshlq_s32(vacc0x89AB, vright_pre_shift);
298     vacc0xCDEF = vshlq_s32(vacc0xCDEF, vright_pre_shift);
299     vacc1x0123 = vshlq_s32(vacc1x0123, vright_pre_shift);
300     vacc1x4567 = vshlq_s32(vacc1x4567, vright_pre_shift);
301     vacc1x89AB = vshlq_s32(vacc1x89AB, vright_pre_shift);
302     vacc1xCDEF = vshlq_s32(vacc1xCDEF, vright_pre_shift);
303     vacc2x0123 = vshlq_s32(vacc2x0123, vright_pre_shift);
304     vacc2x4567 = vshlq_s32(vacc2x4567, vright_pre_shift);
305     vacc2x89AB = vshlq_s32(vacc2x89AB, vright_pre_shift);
306     vacc2xCDEF = vshlq_s32(vacc2xCDEF, vright_pre_shift);
307     vacc3x0123 = vshlq_s32(vacc3x0123, vright_pre_shift);
308     vacc3x4567 = vshlq_s32(vacc3x4567, vright_pre_shift);
309     vacc3x89AB = vshlq_s32(vacc3x89AB, vright_pre_shift);
310     vacc3xCDEF = vshlq_s32(vacc3xCDEF, vright_pre_shift);
311     vacc4x0123 = vshlq_s32(vacc4x0123, vright_pre_shift);
312     vacc4x4567 = vshlq_s32(vacc4x4567, vright_pre_shift);
313     vacc4x89AB = vshlq_s32(vacc4x89AB, vright_pre_shift);
314     vacc4xCDEF = vshlq_s32(vacc4xCDEF, vright_pre_shift);
315     vacc5x0123 = vshlq_s32(vacc5x0123, vright_pre_shift);
316     vacc5x4567 = vshlq_s32(vacc5x4567, vright_pre_shift);
317     vacc5x89AB = vshlq_s32(vacc5x89AB, vright_pre_shift);
318     vacc5xCDEF = vshlq_s32(vacc5xCDEF, vright_pre_shift);
319     vacc6x0123 = vshlq_s32(vacc6x0123, vright_pre_shift);
320     vacc6x4567 = vshlq_s32(vacc6x4567, vright_pre_shift);
321     vacc6x89AB = vshlq_s32(vacc6x89AB, vright_pre_shift);
322     vacc6xCDEF = vshlq_s32(vacc6xCDEF, vright_pre_shift);
323     vacc7x0123 = vshlq_s32(vacc7x0123, vright_pre_shift);
324     vacc7x4567 = vshlq_s32(vacc7x4567, vright_pre_shift);
325     vacc7x89AB = vshlq_s32(vacc7x89AB, vright_pre_shift);
326     vacc7xCDEF = vshlq_s32(vacc7xCDEF, vright_pre_shift);
327 
328     vacc0x0123 = vqdmulhq_s32(vacc0x0123, vmultiplier);
329     vacc0x4567 = vqdmulhq_s32(vacc0x4567, vmultiplier);
330     vacc0x89AB = vqdmulhq_s32(vacc0x89AB, vmultiplier);
331     vacc0xCDEF = vqdmulhq_s32(vacc0xCDEF, vmultiplier);
332     vacc1x0123 = vqdmulhq_s32(vacc1x0123, vmultiplier);
333     vacc1x4567 = vqdmulhq_s32(vacc1x4567, vmultiplier);
334     vacc1x89AB = vqdmulhq_s32(vacc1x89AB, vmultiplier);
335     vacc1xCDEF = vqdmulhq_s32(vacc1xCDEF, vmultiplier);
336     vacc2x0123 = vqdmulhq_s32(vacc2x0123, vmultiplier);
337     vacc2x4567 = vqdmulhq_s32(vacc2x4567, vmultiplier);
338     vacc2x89AB = vqdmulhq_s32(vacc2x89AB, vmultiplier);
339     vacc2xCDEF = vqdmulhq_s32(vacc2xCDEF, vmultiplier);
340     vacc3x0123 = vqdmulhq_s32(vacc3x0123, vmultiplier);
341     vacc3x4567 = vqdmulhq_s32(vacc3x4567, vmultiplier);
342     vacc3x89AB = vqdmulhq_s32(vacc3x89AB, vmultiplier);
343     vacc3xCDEF = vqdmulhq_s32(vacc3xCDEF, vmultiplier);
344     vacc4x0123 = vqdmulhq_s32(vacc4x0123, vmultiplier);
345     vacc4x4567 = vqdmulhq_s32(vacc4x4567, vmultiplier);
346     vacc4x89AB = vqdmulhq_s32(vacc4x89AB, vmultiplier);
347     vacc4xCDEF = vqdmulhq_s32(vacc4xCDEF, vmultiplier);
348     vacc5x0123 = vqdmulhq_s32(vacc5x0123, vmultiplier);
349     vacc5x4567 = vqdmulhq_s32(vacc5x4567, vmultiplier);
350     vacc5x89AB = vqdmulhq_s32(vacc5x89AB, vmultiplier);
351     vacc5xCDEF = vqdmulhq_s32(vacc5xCDEF, vmultiplier);
352     vacc6x0123 = vqdmulhq_s32(vacc6x0123, vmultiplier);
353     vacc6x4567 = vqdmulhq_s32(vacc6x4567, vmultiplier);
354     vacc6x89AB = vqdmulhq_s32(vacc6x89AB, vmultiplier);
355     vacc6xCDEF = vqdmulhq_s32(vacc6xCDEF, vmultiplier);
356     vacc7x0123 = vqdmulhq_s32(vacc7x0123, vmultiplier);
357     vacc7x4567 = vqdmulhq_s32(vacc7x4567, vmultiplier);
358     vacc7x89AB = vqdmulhq_s32(vacc7x89AB, vmultiplier);
359     vacc7xCDEF = vqdmulhq_s32(vacc7xCDEF, vmultiplier);
360 
361     vacc0x0123 = vrshlq_s32(vacc0x0123, vright_post_shift);
362     vacc0x4567 = vrshlq_s32(vacc0x4567, vright_post_shift);
363     vacc0x89AB = vrshlq_s32(vacc0x89AB, vright_post_shift);
364     vacc0xCDEF = vrshlq_s32(vacc0xCDEF, vright_post_shift);
365     vacc1x0123 = vrshlq_s32(vacc1x0123, vright_post_shift);
366     vacc1x4567 = vrshlq_s32(vacc1x4567, vright_post_shift);
367     vacc1x89AB = vrshlq_s32(vacc1x89AB, vright_post_shift);
368     vacc1xCDEF = vrshlq_s32(vacc1xCDEF, vright_post_shift);
369     vacc2x0123 = vrshlq_s32(vacc2x0123, vright_post_shift);
370     vacc2x4567 = vrshlq_s32(vacc2x4567, vright_post_shift);
371     vacc2x89AB = vrshlq_s32(vacc2x89AB, vright_post_shift);
372     vacc2xCDEF = vrshlq_s32(vacc2xCDEF, vright_post_shift);
373     vacc3x0123 = vrshlq_s32(vacc3x0123, vright_post_shift);
374     vacc3x4567 = vrshlq_s32(vacc3x4567, vright_post_shift);
375     vacc3x89AB = vrshlq_s32(vacc3x89AB, vright_post_shift);
376     vacc3xCDEF = vrshlq_s32(vacc3xCDEF, vright_post_shift);
377     vacc4x0123 = vrshlq_s32(vacc4x0123, vright_post_shift);
378     vacc4x4567 = vrshlq_s32(vacc4x4567, vright_post_shift);
379     vacc4x89AB = vrshlq_s32(vacc4x89AB, vright_post_shift);
380     vacc4xCDEF = vrshlq_s32(vacc4xCDEF, vright_post_shift);
381     vacc5x0123 = vrshlq_s32(vacc5x0123, vright_post_shift);
382     vacc5x4567 = vrshlq_s32(vacc5x4567, vright_post_shift);
383     vacc5x89AB = vrshlq_s32(vacc5x89AB, vright_post_shift);
384     vacc5xCDEF = vrshlq_s32(vacc5xCDEF, vright_post_shift);
385     vacc6x0123 = vrshlq_s32(vacc6x0123, vright_post_shift);
386     vacc6x4567 = vrshlq_s32(vacc6x4567, vright_post_shift);
387     vacc6x89AB = vrshlq_s32(vacc6x89AB, vright_post_shift);
388     vacc6xCDEF = vrshlq_s32(vacc6xCDEF, vright_post_shift);
389     vacc7x0123 = vrshlq_s32(vacc7x0123, vright_post_shift);
390     vacc7x4567 = vrshlq_s32(vacc7x4567, vright_post_shift);
391     vacc7x89AB = vrshlq_s32(vacc7x89AB, vright_post_shift);
392     vacc7xCDEF = vrshlq_s32(vacc7xCDEF, vright_post_shift);
393 
394     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
395 #if XNN_ARCH_ARM64
396     const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
397     const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x89AB), vacc0xCDEF), voutput_zero_point);
398     const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
399     const int16x8_t vacc1x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x89AB), vacc1xCDEF), voutput_zero_point);
400     const int16x8_t vacc2x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567), voutput_zero_point);
401     const int16x8_t vacc2x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x89AB), vacc2xCDEF), voutput_zero_point);
402     const int16x8_t vacc3x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567), voutput_zero_point);
403     const int16x8_t vacc3x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc3x89AB), vacc3xCDEF), voutput_zero_point);
404     const int16x8_t vacc4x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc4x0123), vacc4x4567), voutput_zero_point);
405     const int16x8_t vacc4x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc4x89AB), vacc4xCDEF), voutput_zero_point);
406     const int16x8_t vacc5x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc5x0123), vacc5x4567), voutput_zero_point);
407     const int16x8_t vacc5x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc5x89AB), vacc5xCDEF), voutput_zero_point);
408     const int16x8_t vacc6x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc6x0123), vacc6x4567), voutput_zero_point);
409     const int16x8_t vacc6x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc6x89AB), vacc6xCDEF), voutput_zero_point);
410     const int16x8_t vacc7x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc7x0123), vacc7x4567), voutput_zero_point);
411     const int16x8_t vacc7x89ABCDEF = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc7x89AB), vacc7xCDEF), voutput_zero_point);
412 
413     int8x16_t vout0x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc0x89ABCDEF);
414     int8x16_t vout1x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc1x01234567), vacc1x89ABCDEF);
415     int8x16_t vout2x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc2x89ABCDEF);
416     int8x16_t vout3x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc3x01234567), vacc3x89ABCDEF);
417     int8x16_t vout4x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc4x01234567), vacc4x89ABCDEF);
418     int8x16_t vout5x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc5x01234567), vacc5x89ABCDEF);
419     int8x16_t vout6x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc6x01234567), vacc6x89ABCDEF);
420     int8x16_t vout7x0123456789ABCDEF = vqmovn_high_s16(vqmovn_s16(vacc7x01234567), vacc7x89ABCDEF);
421 #else
422     const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point);
423     const int16x8_t vacc0x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x89AB), vqmovn_s32(vacc0xCDEF)), voutput_zero_point);
424     const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point);
425     const int16x8_t vacc1x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x89AB), vqmovn_s32(vacc1xCDEF)), voutput_zero_point);
426     const int16x8_t vacc2x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567)), voutput_zero_point);
427     const int16x8_t vacc2x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x89AB), vqmovn_s32(vacc2xCDEF)), voutput_zero_point);
428     const int16x8_t vacc3x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567)), voutput_zero_point);
429     const int16x8_t vacc3x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc3x89AB), vqmovn_s32(vacc3xCDEF)), voutput_zero_point);
430     const int16x8_t vacc4x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc4x0123), vqmovn_s32(vacc4x4567)), voutput_zero_point);
431     const int16x8_t vacc4x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc4x89AB), vqmovn_s32(vacc4xCDEF)), voutput_zero_point);
432     const int16x8_t vacc5x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc5x0123), vqmovn_s32(vacc5x4567)), voutput_zero_point);
433     const int16x8_t vacc5x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc5x89AB), vqmovn_s32(vacc5xCDEF)), voutput_zero_point);
434     const int16x8_t vacc6x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc6x0123), vqmovn_s32(vacc6x4567)), voutput_zero_point);
435     const int16x8_t vacc6x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc6x89AB), vqmovn_s32(vacc6xCDEF)), voutput_zero_point);
436     const int16x8_t vacc7x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc7x0123), vqmovn_s32(vacc7x4567)), voutput_zero_point);
437     const int16x8_t vacc7x89ABCDEF = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc7x89AB), vqmovn_s32(vacc7xCDEF)), voutput_zero_point);
438 
439     int8x16_t vout0x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc0x89ABCDEF));
440     int8x16_t vout1x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc1x01234567), vqmovn_s16(vacc1x89ABCDEF));
441     int8x16_t vout2x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc2x89ABCDEF));
442     int8x16_t vout3x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc3x01234567), vqmovn_s16(vacc3x89ABCDEF));
443     int8x16_t vout4x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc4x89ABCDEF));
444     int8x16_t vout5x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc5x01234567), vqmovn_s16(vacc5x89ABCDEF));
445     int8x16_t vout6x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc6x89ABCDEF));
446     int8x16_t vout7x0123456789ABCDEF = vcombine_s8(vqmovn_s16(vacc7x01234567), vqmovn_s16(vacc7x89ABCDEF));
447 #endif
448     const int8x16_t voutput_min = vld1q_dup_s8(&params->rndnu_neon.output_min);
449     const int8x16_t voutput_max = vld1q_dup_s8(&params->rndnu_neon.output_max);
450 
451     vout7x0123456789ABCDEF = vmaxq_s8(vout7x0123456789ABCDEF, voutput_min);
452     vout6x0123456789ABCDEF = vmaxq_s8(vout6x0123456789ABCDEF, voutput_min);
453     vout5x0123456789ABCDEF = vmaxq_s8(vout5x0123456789ABCDEF, voutput_min);
454     vout4x0123456789ABCDEF = vmaxq_s8(vout4x0123456789ABCDEF, voutput_min);
455     vout3x0123456789ABCDEF = vmaxq_s8(vout3x0123456789ABCDEF, voutput_min);
456     vout2x0123456789ABCDEF = vmaxq_s8(vout2x0123456789ABCDEF, voutput_min);
457     vout1x0123456789ABCDEF = vmaxq_s8(vout1x0123456789ABCDEF, voutput_min);
458     vout0x0123456789ABCDEF = vmaxq_s8(vout0x0123456789ABCDEF, voutput_min);
459 
460     vout7x0123456789ABCDEF = vminq_s8(vout7x0123456789ABCDEF, voutput_max);
461     vout6x0123456789ABCDEF = vminq_s8(vout6x0123456789ABCDEF, voutput_max);
462     vout5x0123456789ABCDEF = vminq_s8(vout5x0123456789ABCDEF, voutput_max);
463     vout4x0123456789ABCDEF = vminq_s8(vout4x0123456789ABCDEF, voutput_max);
464     vout3x0123456789ABCDEF = vminq_s8(vout3x0123456789ABCDEF, voutput_max);
465     vout2x0123456789ABCDEF = vminq_s8(vout2x0123456789ABCDEF, voutput_max);
466     vout1x0123456789ABCDEF = vminq_s8(vout1x0123456789ABCDEF, voutput_max);
467     vout0x0123456789ABCDEF = vminq_s8(vout0x0123456789ABCDEF, voutput_max);
468 
469     if (nc >= 16) {
470       vst1q_s8(c7 + 0, vout7x0123456789ABCDEF);
471       vst1q_s8(c6 + 0, vout6x0123456789ABCDEF);
472       vst1q_s8(c5 + 0, vout5x0123456789ABCDEF);
473       vst1q_s8(c4 + 0, vout4x0123456789ABCDEF);
474       vst1q_s8(c3 + 0, vout3x0123456789ABCDEF);
475       vst1q_s8(c2 + 0, vout2x0123456789ABCDEF);
476       vst1q_s8(c1 + 0, vout1x0123456789ABCDEF);
477       vst1q_s8(c0 + 0, vout0x0123456789ABCDEF);
478 
479       c7 = (int8_t*) ((uintptr_t) c7 + cn_stride);
480       c6 = (int8_t*) ((uintptr_t) c6 + cn_stride);
481       c5 = (int8_t*) ((uintptr_t) c5 + cn_stride);
482       c4 = (int8_t*) ((uintptr_t) c4 + cn_stride);
483       c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
484       c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
485       c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
486       c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
487 
488       a = (const int8_t**restrict) ((uintptr_t) a - ks);
489 
490       nc -= 16;
491     } else {
492       int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vget_low_s8(vout6x0123456789ABCDEF), vget_low_s8(vout7x0123456789ABCDEF));
493       int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vget_low_s8(vout4x0123456789ABCDEF), vget_low_s8(vout5x0123456789ABCDEF));
494       int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vget_low_s8(vout2x0123456789ABCDEF), vget_low_s8(vout3x0123456789ABCDEF));
495       int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vget_low_s8(vout0x0123456789ABCDEF), vget_low_s8(vout1x0123456789ABCDEF));
496       if (nc & 8) {
497         vst1_s8(c7, vget_high_s8(vout6x01234567_7x01234567)); c7 += 8;
498         vst1_s8(c6, vget_low_s8(vout6x01234567_7x01234567)); c6 += 8;
499         vst1_s8(c5, vget_high_s8(vout4x01234567_5x01234567)); c5 += 8;
500         vst1_s8(c4, vget_low_s8(vout4x01234567_5x01234567)); c4 += 8;
501         vst1_s8(c3, vget_high_s8(vout2x01234567_3x01234567)); c3 += 8;
502         vst1_s8(c2, vget_low_s8(vout2x01234567_3x01234567)); c2 += 8;
503         vst1_s8(c1, vget_high_s8(vout0x01234567_1x01234567)); c1 += 8;
504         vst1_s8(c0, vget_low_s8(vout0x01234567_1x01234567)); c0 += 8;
505         vout6x01234567_7x01234567 = vcombine_s8(vget_high_s8(vout6x0123456789ABCDEF), vget_high_s8(vout7x0123456789ABCDEF));
506         vout4x01234567_5x01234567 = vcombine_s8(vget_high_s8(vout4x0123456789ABCDEF), vget_high_s8(vout5x0123456789ABCDEF));
507         vout2x01234567_3x01234567 = vcombine_s8(vget_high_s8(vout2x0123456789ABCDEF), vget_high_s8(vout3x0123456789ABCDEF));
508         vout0x01234567_1x01234567 = vcombine_s8(vget_high_s8(vout0x0123456789ABCDEF), vget_high_s8(vout1x0123456789ABCDEF));
509       }
510       if (nc & 4) {
511         vst1q_lane_u32((void*) c7, vreinterpretq_u32_s8(vout6x01234567_7x01234567), 2); c7 += 4;
512         vst1q_lane_u32((void*) c6, vreinterpretq_u32_s8(vout6x01234567_7x01234567), 0); c6 += 4;
513         vst1q_lane_u32((void*) c5, vreinterpretq_u32_s8(vout4x01234567_5x01234567), 2); c5 += 4;
514         vst1q_lane_u32((void*) c4, vreinterpretq_u32_s8(vout4x01234567_5x01234567), 0); c4 += 4;
515         vst1q_lane_u32((void*) c3, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4;
516         vst1q_lane_u32((void*) c2, vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4;
517         vst1q_lane_u32((void*) c1, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;
518         vst1q_lane_u32((void*) c0, vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;
519         vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 4);
520         vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 4);
521         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4);
522         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);
523       }
524       if (nc & 2) {
525         vst1q_lane_u16((void*) c7, vreinterpretq_u16_s8(vout6x01234567_7x01234567), 4); c7 += 2;
526         vst1q_lane_u16((void*) c6, vreinterpretq_u16_s8(vout6x01234567_7x01234567), 0); c6 += 2;
527         vst1q_lane_u16((void*) c5, vreinterpretq_u16_s8(vout4x01234567_5x01234567), 4); c5 += 2;
528         vst1q_lane_u16((void*) c4, vreinterpretq_u16_s8(vout4x01234567_5x01234567), 0); c4 += 2;
529         vst1q_lane_u16((void*) c3, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2;
530         vst1q_lane_u16((void*) c2, vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2;
531         vst1q_lane_u16((void*) c1, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;
532         vst1q_lane_u16((void*) c0, vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;
533         vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 2);
534         vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 2);
535         vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2);
536         vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);
537       }
538       if (nc & 1) {
539         vst1q_lane_s8(c7, vout6x01234567_7x01234567, 8);
540         vst1q_lane_s8(c6, vout6x01234567_7x01234567, 0);
541         vst1q_lane_s8(c5, vout4x01234567_5x01234567, 8);
542         vst1q_lane_s8(c4, vout4x01234567_5x01234567, 0);
543         vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8);
544         vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0);
545         vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);
546         vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);
547       }
548 
549       nc = 0;
550     }
551   } while (nc != 0);
552 }
553