xref: /aosp_15_r20/external/XNNPACK/src/x8-transposec/gen/16x16-reuse-dec-zip-neon.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/x32-transposec/neon-zip.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <arm_neon.h>
11 
12 #include <assert.h>
13 
14 #include <xnnpack/common.h>
15 #include <xnnpack/math.h>
16 #include <xnnpack/transpose.h>
17 
xnn_x8_transposec_ukernel__16x16_reuse_dec_zip_neon(const uint8_t * input,uint8_t * output,size_t input_stride,size_t output_stride,size_t block_width,size_t block_height)18 void xnn_x8_transposec_ukernel__16x16_reuse_dec_zip_neon(
19     const uint8_t* input,
20     uint8_t* output,
21     size_t input_stride,
22     size_t output_stride,
23     size_t block_width,
24     size_t block_height) XNN_OOB_READS
25 {
26   assert(output_stride >= block_height * sizeof(uint8_t));
27   assert(input_stride >= block_width * sizeof(uint8_t));
28 
29   const size_t tile_height = 16;
30   const size_t tile_width = 16;
31   const size_t tile_hbytes = tile_height * sizeof(uint8_t);
32   const size_t tile_wbytes = tile_width * sizeof(uint8_t);
33   const size_t input_reset = tile_wbytes - round_down_po2(block_height, tile_height) * input_stride;
34   const size_t output_reset = tile_width * output_stride - round_down_po2(block_height, 2) * sizeof(uint8_t) - tile_hbytes;
35 
36   const uint8_t* i0 = input;
37   uint8_t* o = (uint8_t*) ((uintptr_t) output - tile_hbytes);
38   const size_t minus_output_stride = -output_stride;
39 
40   do {
41     const size_t rem = min(block_width - 1, 15);
42     const size_t oN_stride = rem * output_stride;
43     const size_t oN_offset = oN_stride + tile_hbytes;
44     size_t bh = block_height;
45     for (; bh >= 16; bh -= 16) {
46       const uint8x16_t v4_0 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
47       const uint8x16_t v4_1 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
48       const uint8x16_t v4_2 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
49       const uint8x16_t v4_3 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
50       const uint8x16_t v4_4 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
51       const uint8x16_t v4_5 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
52       const uint8x16_t v4_6 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
53       const uint8x16_t v4_7 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
54       const uint8x16_t v4_8 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
55       const uint8x16_t v4_9 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
56       const uint8x16_t v4_10 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
57       const uint8x16_t v4_11 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
58       const uint8x16_t v4_12 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
59       const uint8x16_t v4_13 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
60       const uint8x16_t v4_14 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
61       const uint8x16_t v4_15 = vld1q_u8(i0); i0 = (uint8_t*) ((uintptr_t) i0 + input_stride);
62 
63       const uint8x16x2_t v3_0 = vzipq_u8(v4_0, v4_8);
64       const uint8x16x2_t v3_1 = vzipq_u8(v4_1, v4_9);
65       const uint8x16x2_t v3_2 = vzipq_u8(v4_2, v4_10);
66       const uint8x16x2_t v3_3 = vzipq_u8(v4_3, v4_11);
67       const uint8x16x2_t v3_4 = vzipq_u8(v4_4, v4_12);
68       const uint8x16x2_t v3_5 = vzipq_u8(v4_5, v4_13);
69       const uint8x16x2_t v3_6 = vzipq_u8(v4_6, v4_14);
70       const uint8x16x2_t v3_7 = vzipq_u8(v4_7, v4_15);
71 
72       const uint8x16x2_t v2_0 = vzipq_u8(v3_0.val[0], v3_4.val[0]);
73       const uint8x16x2_t v2_1 = vzipq_u8(v3_0.val[1], v3_4.val[1]);
74       const uint8x16x2_t v2_2 = vzipq_u8(v3_1.val[0], v3_5.val[0]);
75       const uint8x16x2_t v2_3 = vzipq_u8(v3_1.val[1], v3_5.val[1]);
76       const uint8x16x2_t v2_4 = vzipq_u8(v3_2.val[0], v3_6.val[0]);
77       const uint8x16x2_t v2_5 = vzipq_u8(v3_2.val[1], v3_6.val[1]);
78       const uint8x16x2_t v2_6 = vzipq_u8(v3_3.val[0], v3_7.val[0]);
79       const uint8x16x2_t v2_7 = vzipq_u8(v3_3.val[1], v3_7.val[1]);
80       const uint8x16x2_t v1_0 = vzipq_u8(v2_0.val[0], v2_4.val[0]);
81       const uint8x16x2_t v1_1 = vzipq_u8(v2_0.val[1], v2_4.val[1]);
82       const uint8x16x2_t v1_2 = vzipq_u8(v2_1.val[0], v2_5.val[0]);
83       const uint8x16x2_t v1_3 = vzipq_u8(v2_1.val[1], v2_5.val[1]);
84       const uint8x16x2_t v1_4 = vzipq_u8(v2_2.val[0], v2_6.val[0]);
85       const uint8x16x2_t v1_5 = vzipq_u8(v2_2.val[1], v2_6.val[1]);
86       const uint8x16x2_t v1_6 = vzipq_u8(v2_3.val[0], v2_7.val[0]);
87       const uint8x16x2_t v1_7 = vzipq_u8(v2_3.val[1], v2_7.val[1]);
88       const uint8x16x2_t v0_0 = vzipq_u8(v1_0.val[0], v1_4.val[0]);
89       const uint8x16x2_t v0_1 = vzipq_u8(v1_0.val[1], v1_4.val[1]);
90       const uint8x16x2_t v0_2 = vzipq_u8(v1_1.val[0], v1_5.val[0]);
91       const uint8x16x2_t v0_3 = vzipq_u8(v1_1.val[1], v1_5.val[1]);
92       const uint8x16x2_t v0_4 = vzipq_u8(v1_2.val[0], v1_6.val[0]);
93       const uint8x16x2_t v0_5 = vzipq_u8(v1_2.val[1], v1_6.val[1]);
94       const uint8x16x2_t v0_6 = vzipq_u8(v1_3.val[0], v1_7.val[0]);
95       const uint8x16x2_t v0_7 = vzipq_u8(v1_3.val[1], v1_7.val[1]);
96 
97       o = (uint8_t*) ((uintptr_t) o + oN_offset);
98       vst1q_u8(o, v0_7.val[1]);
99       if XNN_UNPREDICTABLE(block_width > 15) {
100         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
101       }
102       vst1q_u8(o, v0_7.val[0]);
103       if XNN_UNPREDICTABLE(block_width >= 15) {
104         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
105       }
106       vst1q_u8(o, v0_6.val[1]);
107       if XNN_UNPREDICTABLE(block_width > 13) {
108         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
109       }
110       vst1q_u8(o, v0_6.val[0]);
111       if XNN_UNPREDICTABLE(block_width >= 13) {
112         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
113       }
114       vst1q_u8(o, v0_5.val[1]);
115       if XNN_UNPREDICTABLE(block_width > 11) {
116         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
117       }
118       vst1q_u8(o, v0_5.val[0]);
119       if XNN_UNPREDICTABLE(block_width >= 11) {
120         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
121       }
122       vst1q_u8(o, v0_4.val[1]);
123       if XNN_UNPREDICTABLE(block_width > 9) {
124         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
125       }
126       vst1q_u8(o, v0_4.val[0]);
127       if XNN_UNPREDICTABLE(block_width >= 9) {
128         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
129       }
130       vst1q_u8(o, v0_3.val[1]);
131       if XNN_UNPREDICTABLE(block_width > 7) {
132         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
133       }
134       vst1q_u8(o, v0_3.val[0]);
135       if XNN_UNPREDICTABLE(block_width >= 7) {
136         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
137       }
138       vst1q_u8(o, v0_2.val[1]);
139       if XNN_UNPREDICTABLE(block_width > 5) {
140         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
141       }
142       vst1q_u8(o, v0_2.val[0]);
143       if XNN_UNPREDICTABLE(block_width >= 5) {
144         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
145       }
146       vst1q_u8(o, v0_1.val[1]);
147       if XNN_UNPREDICTABLE(block_width > 3) {
148         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
149       }
150       vst1q_u8(o, v0_1.val[0]);
151       if XNN_UNPREDICTABLE(block_width >= 3) {
152         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
153       }
154       vst1q_u8(o, v0_0.val[1]);
155       if XNN_UNPREDICTABLE(block_width > 1) {
156         o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
157       }
158       vst1q_u8(o, v0_0.val[0]);
159     }
160     o = (uint8_t*) ((uintptr_t) o + tile_hbytes);
161 
162     if (bh != 0) {
163       const uint8x16_t v4_0 = vld1q_u8(i0);
164       const uint8_t *i1 = (const uint8_t*) ((uintptr_t) i0 + input_stride);
165       if XNN_UNPREDICTABLE(bh < 2) {
166         i1 = i0;
167       }
168       const uint8x16_t v4_1 = vld1q_u8(i1);
169       const uint8_t *i2 = (const uint8_t*) ((uintptr_t) i1 + input_stride);
170       if XNN_UNPREDICTABLE(bh <= 2) {
171         i2 = i1;
172       }
173       const uint8x16_t v4_2 = vld1q_u8(i2);
174       const uint8_t *i3 = (const uint8_t*) ((uintptr_t) i2 + input_stride);
175       if XNN_UNPREDICTABLE(bh < 4) {
176         i3 = i2;
177       }
178       const uint8x16_t v4_3 = vld1q_u8(i3);
179       const uint8_t *i4 = (const uint8_t*) ((uintptr_t) i3 + input_stride);
180       if XNN_UNPREDICTABLE(bh <= 4) {
181         i4 = i3;
182       }
183       const uint8x16_t v4_4 = vld1q_u8(i4);
184       const uint8_t *i5 = (const uint8_t*) ((uintptr_t) i4 + input_stride);
185       if XNN_UNPREDICTABLE(bh < 6) {
186         i5 = i4;
187       }
188       const uint8x16_t v4_5 = vld1q_u8(i5);
189       const uint8_t *i6 = (const uint8_t*) ((uintptr_t) i5 + input_stride);
190       if XNN_UNPREDICTABLE(bh <= 6) {
191         i6 = i5;
192       }
193       const uint8x16_t v4_6 = vld1q_u8(i6);
194       const uint8_t *i7 = (const uint8_t*) ((uintptr_t) i6 + input_stride);
195       if XNN_UNPREDICTABLE(bh < 8) {
196         i7 = i6;
197       }
198       const uint8x16_t v4_7 = vld1q_u8(i7);
199       const uint8_t *i8 = (const uint8_t*) ((uintptr_t) i7 + input_stride);
200       if XNN_UNPREDICTABLE(bh <= 8) {
201         i8 = i7;
202       }
203       const uint8x16_t v4_8 = vld1q_u8(i8);
204       const uint8_t *i9 = (const uint8_t*) ((uintptr_t) i8 + input_stride);
205       if XNN_UNPREDICTABLE(bh < 10) {
206         i9 = i8;
207       }
208       const uint8x16_t v4_9 = vld1q_u8(i9);
209       const uint8_t *i10 = (const uint8_t*) ((uintptr_t) i9 + input_stride);
210       if XNN_UNPREDICTABLE(bh <= 10) {
211         i10 = i9;
212       }
213       const uint8x16_t v4_10 = vld1q_u8(i10);
214       const uint8_t *i11 = (const uint8_t*) ((uintptr_t) i10 + input_stride);
215       if XNN_UNPREDICTABLE(bh < 12) {
216         i11 = i10;
217       }
218       const uint8x16_t v4_11 = vld1q_u8(i11);
219       const uint8_t *i12 = (const uint8_t*) ((uintptr_t) i11 + input_stride);
220       if XNN_UNPREDICTABLE(bh <= 12) {
221         i12 = i11;
222       }
223       const uint8x16_t v4_12 = vld1q_u8(i12);
224       const uint8_t *i13 = (const uint8_t*) ((uintptr_t) i12 + input_stride);
225       if XNN_UNPREDICTABLE(bh < 14) {
226         i13 = i12;
227       }
228       const uint8x16_t v4_13 = vld1q_u8(i13);
229       const uint8_t *i14 = (const uint8_t*) ((uintptr_t) i13 + input_stride);
230       if XNN_UNPREDICTABLE(bh <= 14) {
231         i14 = i13;
232       }
233       const uint8x16_t v4_14 = vld1q_u8(i14);
234       const uint8x16_t v4_15 = vmovq_n_u8(0);
235 
236       const uint8x16x2_t v3_0 = vzipq_u8(v4_0, v4_8);
237       const uint8x16x2_t v3_1 = vzipq_u8(v4_1, v4_9);
238       const uint8x16x2_t v3_2 = vzipq_u8(v4_2, v4_10);
239       const uint8x16x2_t v3_3 = vzipq_u8(v4_3, v4_11);
240       const uint8x16x2_t v3_4 = vzipq_u8(v4_4, v4_12);
241       const uint8x16x2_t v3_5 = vzipq_u8(v4_5, v4_13);
242       const uint8x16x2_t v3_6 = vzipq_u8(v4_6, v4_14);
243       const uint8x16x2_t v3_7 = vzipq_u8(v4_7, v4_15);
244 
245       const uint8x16x2_t v2_0 = vzipq_u8(v3_0.val[0], v3_4.val[0]);
246       const uint8x16x2_t v2_1 = vzipq_u8(v3_0.val[1], v3_4.val[1]);
247       const uint8x16x2_t v2_2 = vzipq_u8(v3_1.val[0], v3_5.val[0]);
248       const uint8x16x2_t v2_3 = vzipq_u8(v3_1.val[1], v3_5.val[1]);
249       const uint8x16x2_t v2_4 = vzipq_u8(v3_2.val[0], v3_6.val[0]);
250       const uint8x16x2_t v2_5 = vzipq_u8(v3_2.val[1], v3_6.val[1]);
251       const uint8x16x2_t v2_6 = vzipq_u8(v3_3.val[0], v3_7.val[0]);
252       const uint8x16x2_t v2_7 = vzipq_u8(v3_3.val[1], v3_7.val[1]);
253       const uint8x16x2_t v1_0 = vzipq_u8(v2_0.val[0], v2_4.val[0]);
254       const uint8x16x2_t v1_1 = vzipq_u8(v2_0.val[1], v2_4.val[1]);
255       const uint8x16x2_t v1_2 = vzipq_u8(v2_1.val[0], v2_5.val[0]);
256       const uint8x16x2_t v1_3 = vzipq_u8(v2_1.val[1], v2_5.val[1]);
257       const uint8x16x2_t v1_4 = vzipq_u8(v2_2.val[0], v2_6.val[0]);
258       const uint8x16x2_t v1_5 = vzipq_u8(v2_2.val[1], v2_6.val[1]);
259       const uint8x16x2_t v1_6 = vzipq_u8(v2_3.val[0], v2_7.val[0]);
260       const uint8x16x2_t v1_7 = vzipq_u8(v2_3.val[1], v2_7.val[1]);
261       const uint8x16x2_t v0_0 = vzipq_u8(v1_0.val[0], v1_4.val[0]);
262       const uint8x16x2_t v0_1 = vzipq_u8(v1_0.val[1], v1_4.val[1]);
263       const uint8x16x2_t v0_2 = vzipq_u8(v1_1.val[0], v1_5.val[0]);
264       const uint8x16x2_t v0_3 = vzipq_u8(v1_1.val[1], v1_5.val[1]);
265       const uint8x16x2_t v0_4 = vzipq_u8(v1_2.val[0], v1_6.val[0]);
266       const uint8x16x2_t v0_5 = vzipq_u8(v1_2.val[1], v1_6.val[1]);
267       const uint8x16x2_t v0_6 = vzipq_u8(v1_3.val[0], v1_7.val[0]);
268       const uint8x16x2_t v0_7 = vzipq_u8(v1_3.val[1], v1_7.val[1]);
269 
270       uint8x8_t v0_low = vget_low_u8(v0_0.val[0]);
271       uint8x8_t v1_low = vget_low_u8(v0_0.val[1]);
272       uint8x8_t v2_low = vget_low_u8(v0_1.val[0]);
273       uint8x8_t v3_low = vget_low_u8(v0_1.val[1]);
274       uint8x8_t v4_low = vget_low_u8(v0_2.val[0]);
275       uint8x8_t v5_low = vget_low_u8(v0_2.val[1]);
276       uint8x8_t v6_low = vget_low_u8(v0_3.val[0]);
277       uint8x8_t v7_low = vget_low_u8(v0_3.val[1]);
278       uint8x8_t v8_low = vget_low_u8(v0_4.val[0]);
279       uint8x8_t v9_low = vget_low_u8(v0_4.val[1]);
280       uint8x8_t v10_low = vget_low_u8(v0_5.val[0]);
281       uint8x8_t v11_low = vget_low_u8(v0_5.val[1]);
282       uint8x8_t v12_low = vget_low_u8(v0_6.val[0]);
283       uint8x8_t v13_low = vget_low_u8(v0_6.val[1]);
284       uint8x8_t v14_low = vget_low_u8(v0_7.val[0]);
285       uint8x8_t v15_low = vget_low_u8(v0_7.val[1]);
286 
287       if (bh & 8) {
288         o = (uint8_t*) ((uintptr_t) o + oN_stride);
289         vst1_u8(o, v15_low);
290         if XNN_UNPREDICTABLE(block_width > 15) {
291           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
292         }
293         vst1_u8(o, v14_low);
294         if XNN_UNPREDICTABLE(block_width >= 15) {
295           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
296         }
297         vst1_u8(o, v13_low);
298         if XNN_UNPREDICTABLE(block_width > 13) {
299           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
300         }
301         vst1_u8(o, v12_low);
302         if XNN_UNPREDICTABLE(block_width >= 13) {
303           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
304         }
305         vst1_u8(o, v11_low);
306         if XNN_UNPREDICTABLE(block_width > 11) {
307           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
308         }
309         vst1_u8(o, v10_low);
310         if XNN_UNPREDICTABLE(block_width >= 11) {
311           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
312         }
313         vst1_u8(o, v9_low);
314         if XNN_UNPREDICTABLE(block_width > 9) {
315           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
316         }
317         vst1_u8(o, v8_low);
318         if XNN_UNPREDICTABLE(block_width >= 9) {
319           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
320         }
321         vst1_u8(o, v7_low);
322         if XNN_UNPREDICTABLE(block_width > 7) {
323           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
324         }
325         vst1_u8(o, v6_low);
326         if XNN_UNPREDICTABLE(block_width >= 7) {
327           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
328         }
329         vst1_u8(o, v5_low);
330         if XNN_UNPREDICTABLE(block_width > 5) {
331           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
332         }
333         vst1_u8(o, v4_low);
334         if XNN_UNPREDICTABLE(block_width >= 5) {
335           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
336         }
337         vst1_u8(o, v3_low);
338         if XNN_UNPREDICTABLE(block_width > 3) {
339           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
340         }
341         vst1_u8(o, v2_low);
342         if XNN_UNPREDICTABLE(block_width >= 3) {
343           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
344         }
345         vst1_u8(o, v1_low);
346         if XNN_UNPREDICTABLE(block_width > 1) {
347           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
348         }
349         vst1_u8(o, v0_low); o += 8;
350         v0_low = vget_high_u8(v0_0.val[0]);
351         v1_low = vget_high_u8(v0_0.val[1]);
352         v2_low = vget_high_u8(v0_1.val[0]);
353         v3_low = vget_high_u8(v0_1.val[1]);
354         v4_low = vget_high_u8(v0_2.val[0]);
355         v5_low = vget_high_u8(v0_2.val[1]);
356         v6_low = vget_high_u8(v0_3.val[0]);
357         v7_low = vget_high_u8(v0_3.val[1]);
358         v8_low = vget_high_u8(v0_4.val[0]);
359         v9_low = vget_high_u8(v0_4.val[1]);
360         v10_low = vget_high_u8(v0_5.val[0]);
361         v11_low = vget_high_u8(v0_5.val[1]);
362         v12_low = vget_high_u8(v0_6.val[0]);
363         v13_low = vget_high_u8(v0_6.val[1]);
364         v14_low = vget_high_u8(v0_7.val[0]);
365         v15_low = vget_high_u8(v0_7.val[1]);
366       }
367 
368       if (bh & 4) {
369         o = (uint8_t*) ((uintptr_t) o + oN_stride);
370         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v15_low), 0);
371         if XNN_UNPREDICTABLE(block_width > 15) {
372           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
373         }
374         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v14_low), 0);
375         if XNN_UNPREDICTABLE(block_width >= 15) {
376           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
377         }
378         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v13_low), 0);
379         if XNN_UNPREDICTABLE(block_width > 13) {
380           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
381         }
382         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v12_low), 0);
383         if XNN_UNPREDICTABLE(block_width >= 13) {
384           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
385         }
386         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v11_low), 0);
387         if XNN_UNPREDICTABLE(block_width > 11) {
388           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
389         }
390         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v10_low), 0);
391         if XNN_UNPREDICTABLE(block_width >= 11) {
392           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
393         }
394         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v9_low), 0);
395         if XNN_UNPREDICTABLE(block_width > 9) {
396           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
397         }
398         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v8_low), 0);
399         if XNN_UNPREDICTABLE(block_width >= 9) {
400           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
401         }
402         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v7_low), 0);
403         if XNN_UNPREDICTABLE(block_width > 7) {
404           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
405         }
406         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v6_low), 0);
407         if XNN_UNPREDICTABLE(block_width >= 7) {
408           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
409         }
410         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v5_low), 0);
411         if XNN_UNPREDICTABLE(block_width > 5) {
412           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
413         }
414         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v4_low), 0);
415         if XNN_UNPREDICTABLE(block_width >= 5) {
416           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
417         }
418         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v3_low), 0);
419         if XNN_UNPREDICTABLE(block_width > 3) {
420           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
421         }
422         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v2_low), 0);
423         if XNN_UNPREDICTABLE(block_width >= 3) {
424           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
425         }
426         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v1_low), 0);
427         if XNN_UNPREDICTABLE(block_width > 1) {
428           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
429         }
430         vst1_lane_u32((void*) o, vreinterpret_u32_u8(v0_low), 0); o += 4;
431         v0_low = vext_u8(v0_low, v0_low, 4);
432         v1_low = vext_u8(v1_low, v1_low, 4);
433         v2_low = vext_u8(v2_low, v2_low, 4);
434         v3_low = vext_u8(v3_low, v3_low, 4);
435         v4_low = vext_u8(v4_low, v4_low, 4);
436         v5_low = vext_u8(v5_low, v5_low, 4);
437         v6_low = vext_u8(v6_low, v6_low, 4);
438         v7_low = vext_u8(v7_low, v7_low, 4);
439         v8_low = vext_u8(v8_low, v8_low, 4);
440         v9_low = vext_u8(v9_low, v9_low, 4);
441         v10_low = vext_u8(v10_low, v10_low, 4);
442         v11_low = vext_u8(v11_low, v11_low, 4);
443         v12_low = vext_u8(v12_low, v12_low, 4);
444         v13_low = vext_u8(v13_low, v13_low, 4);
445         v14_low = vext_u8(v14_low, v14_low, 4);
446         v15_low = vext_u8(v15_low, v15_low, 4);
447       }
448       if (bh & 2) {
449         o = (uint8_t*) ((uintptr_t) o + oN_stride);
450         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v15_low), 0);
451         if XNN_UNPREDICTABLE(block_width > 15) {
452           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
453         }
454         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v14_low), 0);
455         if XNN_UNPREDICTABLE(block_width >= 15) {
456           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
457         }
458         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v13_low), 0);
459         if XNN_UNPREDICTABLE(block_width > 13) {
460           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
461         }
462         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v12_low), 0);
463         if XNN_UNPREDICTABLE(block_width >= 13) {
464           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
465         }
466         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v11_low), 0);
467         if XNN_UNPREDICTABLE(block_width > 11) {
468           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
469         }
470         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v10_low), 0);
471         if XNN_UNPREDICTABLE(block_width >= 11) {
472           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
473         }
474         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v9_low), 0);
475         if XNN_UNPREDICTABLE(block_width > 9) {
476           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
477         }
478         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v8_low), 0);
479         if XNN_UNPREDICTABLE(block_width >= 9) {
480           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
481         }
482         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v7_low), 0);
483         if XNN_UNPREDICTABLE(block_width > 7) {
484           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
485         }
486         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v6_low), 0);
487         if XNN_UNPREDICTABLE(block_width >= 7) {
488           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
489         }
490         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v5_low), 0);
491         if XNN_UNPREDICTABLE(block_width > 5) {
492           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
493         }
494         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v4_low), 0);
495         if XNN_UNPREDICTABLE(block_width >= 5) {
496           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
497         }
498         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v3_low), 0);
499         if XNN_UNPREDICTABLE(block_width > 3) {
500           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
501         }
502         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v2_low), 0);
503         if XNN_UNPREDICTABLE(block_width >= 3) {
504           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
505         }
506         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v1_low), 0);
507         if XNN_UNPREDICTABLE(block_width > 1) {
508           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
509         }
510         vst1_lane_u16((void*) o, vreinterpret_u16_u8(v0_low), 0); o += 2;
511         v0_low = vext_u8(v0_low, v0_low, 2);
512         v1_low = vext_u8(v1_low, v1_low, 2);
513         v2_low = vext_u8(v2_low, v2_low, 2);
514         v3_low = vext_u8(v3_low, v3_low, 2);
515         v4_low = vext_u8(v4_low, v4_low, 2);
516         v5_low = vext_u8(v5_low, v5_low, 2);
517         v6_low = vext_u8(v6_low, v6_low, 2);
518         v7_low = vext_u8(v7_low, v7_low, 2);
519         v8_low = vext_u8(v8_low, v8_low, 2);
520         v9_low = vext_u8(v9_low, v9_low, 2);
521         v10_low = vext_u8(v10_low, v10_low, 2);
522         v11_low = vext_u8(v11_low, v11_low, 2);
523         v12_low = vext_u8(v12_low, v12_low, 2);
524         v13_low = vext_u8(v13_low, v13_low, 2);
525         v14_low = vext_u8(v14_low, v14_low, 2);
526         v15_low = vext_u8(v15_low, v15_low, 2);
527       }
528       if (bh & 1) {
529         o = (uint8_t*) ((uintptr_t) o + oN_stride);
530         vst1_lane_u8(o, v15_low, 0);
531         if XNN_UNPREDICTABLE(block_width > 15) {
532           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
533         }
534         vst1_lane_u8(o, v14_low, 0);
535         if XNN_UNPREDICTABLE(block_width >= 15) {
536           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
537         }
538         vst1_lane_u8(o, v13_low, 0);
539         if XNN_UNPREDICTABLE(block_width > 13) {
540           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
541         }
542         vst1_lane_u8(o, v12_low, 0);
543         if XNN_UNPREDICTABLE(block_width >= 13) {
544           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
545         }
546         vst1_lane_u8(o, v11_low, 0);
547         if XNN_UNPREDICTABLE(block_width > 11) {
548           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
549         }
550         vst1_lane_u8(o, v10_low, 0);
551         if XNN_UNPREDICTABLE(block_width >= 11) {
552           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
553         }
554         vst1_lane_u8(o, v9_low, 0);
555         if XNN_UNPREDICTABLE(block_width > 9) {
556           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
557         }
558         vst1_lane_u8(o, v8_low, 0);
559         if XNN_UNPREDICTABLE(block_width >= 9) {
560           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
561         }
562         vst1_lane_u8(o, v7_low, 0);
563         if XNN_UNPREDICTABLE(block_width > 7) {
564           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
565         }
566         vst1_lane_u8(o, v6_low, 0);
567         if XNN_UNPREDICTABLE(block_width >= 7) {
568           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
569         }
570         vst1_lane_u8(o, v5_low, 0);
571         if XNN_UNPREDICTABLE(block_width > 5) {
572           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
573         }
574         vst1_lane_u8(o, v4_low, 0);
575         if XNN_UNPREDICTABLE(block_width >= 5) {
576           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
577         }
578         vst1_lane_u8(o, v3_low, 0);
579         if XNN_UNPREDICTABLE(block_width > 3) {
580           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
581         }
582         vst1_lane_u8(o, v2_low, 0);
583         if XNN_UNPREDICTABLE(block_width >= 3) {
584           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
585         }
586         vst1_lane_u8(o, v1_low, 0);
587         if XNN_UNPREDICTABLE(block_width > 1) {
588           o = (uint8_t*) ((uintptr_t) o + minus_output_stride);
589         }
590         vst1_lane_u8(o, v0_low, 0);
591       }
592     }
593 
594     i0 = (const uint8_t*) ((uintptr_t) i0 + input_reset);
595     o = (uint8_t*) ((uintptr_t) o + output_reset);
596     block_width = doz(block_width, tile_width);
597   } while (block_width != 0);
598 }
599