xref: /aosp_15_r20/external/XNNPACK/src/f16-dwconv2d-chw/gen/3x3s2p1-minmax-neonfp16arith-2x4.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker //   Template: src/f16-dwconv2d-chw/3x3s2p1-neonfp16arith.c.in
3*4bdc9457SAndroid Build Coastguard Worker //   Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2020 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker 
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker 
12*4bdc9457SAndroid Build Coastguard Worker #include <arm_neon.h>
13*4bdc9457SAndroid Build Coastguard Worker 
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/dwconv.h>
15*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/math.h>
16*4bdc9457SAndroid Build Coastguard Worker 
17*4bdc9457SAndroid Build Coastguard Worker 
xnn_f16_dwconv2d_chw_ukernel_3x3s2p1__neonfp16arith_2x4(size_t input_height,size_t input_width,const void * input,const void * weights,const void * zero,void * output,uint32_t padding_top,const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS (1)])18*4bdc9457SAndroid Build Coastguard Worker void xnn_f16_dwconv2d_chw_ukernel_3x3s2p1__neonfp16arith_2x4(
19*4bdc9457SAndroid Build Coastguard Worker     size_t input_height,
20*4bdc9457SAndroid Build Coastguard Worker     size_t input_width,
21*4bdc9457SAndroid Build Coastguard Worker     const void* input,
22*4bdc9457SAndroid Build Coastguard Worker     const void* weights,
23*4bdc9457SAndroid Build Coastguard Worker     const void* zero,
24*4bdc9457SAndroid Build Coastguard Worker     void* output,
25*4bdc9457SAndroid Build Coastguard Worker     uint32_t padding_top,
26*4bdc9457SAndroid Build Coastguard Worker     const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker   assert(input_height != 0);
29*4bdc9457SAndroid Build Coastguard Worker   assert(input_width != 0);
30*4bdc9457SAndroid Build Coastguard Worker   assert(input_width % sizeof(__fp16) == 0);
31*4bdc9457SAndroid Build Coastguard Worker   assert(padding_top >= 0);
32*4bdc9457SAndroid Build Coastguard Worker   assert(padding_top <= 1);
33*4bdc9457SAndroid Build Coastguard Worker 
34*4bdc9457SAndroid Build Coastguard Worker   const uint16x4_t vmask_even = vld1_u16(params->neonfp16arith.mask_even);
35*4bdc9457SAndroid Build Coastguard Worker   const uint16x4_t vmask_odd  = vld1_u16(params->neonfp16arith.mask_odd);
36*4bdc9457SAndroid Build Coastguard Worker   const float16x4_t vmax = vld1_dup_f16(&params->neonfp16arith.max);
37*4bdc9457SAndroid Build Coastguard Worker   const float16x4_t vmin = vld1_dup_f16(&params->neonfp16arith.min);
38*4bdc9457SAndroid Build Coastguard Worker 
39*4bdc9457SAndroid Build Coastguard Worker   const __fp16* w0 = (const __fp16*)weights;
40*4bdc9457SAndroid Build Coastguard Worker   const float16x8_t vw01234567 = vld1q_f16(w0);
41*4bdc9457SAndroid Build Coastguard Worker   const float16x4_t vw89 = vreinterpret_f16_u32(vld1_lane_u32((const void*)(w0 + 8), vmov_n_u32(0), 0));
42*4bdc9457SAndroid Build Coastguard Worker 
43*4bdc9457SAndroid Build Coastguard Worker   const size_t input_decrement = round_down_po2(input_width, 4 /* SIMD output width */ * 2 /* subsampling */ * sizeof(__fp16));
44*4bdc9457SAndroid Build Coastguard Worker   const size_t output_width = round_down_po2((input_width + (2 /* padding */ - 3 /* kernel size */ + 2 /* subsampling */) * sizeof(__fp16)) / 2, sizeof(__fp16));
45*4bdc9457SAndroid Build Coastguard Worker 
46*4bdc9457SAndroid Build Coastguard Worker   const __fp16* i0 = (const __fp16*) ((uintptr_t) input - ((-padding_top) & input_width));
47*4bdc9457SAndroid Build Coastguard Worker   const __fp16* i1 = (const __fp16*) ((uintptr_t) i0 + input_width);
48*4bdc9457SAndroid Build Coastguard Worker   if XNN_UNPREDICTABLE(padding_top != 0) {
49*4bdc9457SAndroid Build Coastguard Worker     i0 = zero;
50*4bdc9457SAndroid Build Coastguard Worker   }
51*4bdc9457SAndroid Build Coastguard Worker   const __fp16* i2 = (const __fp16*) ((uintptr_t) i1 + input_width);
52*4bdc9457SAndroid Build Coastguard Worker   const __fp16* i3 = (const __fp16*) ((uintptr_t) i2 + input_width);
53*4bdc9457SAndroid Build Coastguard Worker   const __fp16* i4 = (const __fp16*) ((uintptr_t) i3 + input_width);
54*4bdc9457SAndroid Build Coastguard Worker 
55*4bdc9457SAndroid Build Coastguard Worker   __fp16* o0 = output;
56*4bdc9457SAndroid Build Coastguard Worker   __fp16* o1 = (__fp16*) ((uintptr_t) o0 + output_width);
57*4bdc9457SAndroid Build Coastguard Worker 
58*4bdc9457SAndroid Build Coastguard Worker   size_t padded_input_height = input_height + padding_top + 1 /* padding bottom */;
59*4bdc9457SAndroid Build Coastguard Worker   size_t output_height = (padded_input_height - 3 /* kernel size */ + 2 /* subsampling */) / 2;
60*4bdc9457SAndroid Build Coastguard Worker   do {
61*4bdc9457SAndroid Build Coastguard Worker     if XNN_UNPREDICTABLE(padded_input_height < 4) {
62*4bdc9457SAndroid Build Coastguard Worker       i2 = zero;
63*4bdc9457SAndroid Build Coastguard Worker     }
64*4bdc9457SAndroid Build Coastguard Worker     if XNN_UNPREDICTABLE(padded_input_height < 5) {
65*4bdc9457SAndroid Build Coastguard Worker       i3 = zero;
66*4bdc9457SAndroid Build Coastguard Worker       o1 = o0;
67*4bdc9457SAndroid Build Coastguard Worker     }
68*4bdc9457SAndroid Build Coastguard Worker     if XNN_UNPREDICTABLE(padded_input_height < 6) {
69*4bdc9457SAndroid Build Coastguard Worker       i4 = zero;
70*4bdc9457SAndroid Build Coastguard Worker     }
71*4bdc9457SAndroid Build Coastguard Worker 
72*4bdc9457SAndroid Build Coastguard Worker     float16x4_t vi0x1357 = vmov_n_f16(0);
73*4bdc9457SAndroid Build Coastguard Worker     float16x4_t vi1x1357 = vmov_n_f16(0);
74*4bdc9457SAndroid Build Coastguard Worker     float16x4_t vi2x1357 = vmov_n_f16(0);
75*4bdc9457SAndroid Build Coastguard Worker     float16x4_t vi3x1357 = vmov_n_f16(0);
76*4bdc9457SAndroid Build Coastguard Worker     float16x4_t vi4x1357 = vmov_n_f16(0);
77*4bdc9457SAndroid Build Coastguard Worker 
78*4bdc9457SAndroid Build Coastguard Worker     size_t w = input_width;
79*4bdc9457SAndroid Build Coastguard Worker     for (; w >= 8 * sizeof(__fp16); w -= 8 * sizeof(__fp16)) {
80*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo0p0 = vdup_laneq_f16(vw01234567, 0);
81*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo1p0 = vdup_laneq_f16(vw01234567, 0);
82*4bdc9457SAndroid Build Coastguard Worker 
83*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi0x8ACE9BDF = vld2_f16(i0); i0 += 8;
84*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi1x8ACE9BDF = vld2_f16(i1); i1 += 8;
85*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi2x8ACE9BDF = vld2_f16(i2); i2 += 8;
86*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi3x8ACE9BDF = vld2_f16(i3); i3 += 8;
87*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi4x8ACE9BDF = vld2_f16(i4); i4 += 8;
88*4bdc9457SAndroid Build Coastguard Worker 
89*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x8ACE9BDF.val[0], vw01234567, 2);
90*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x8ACE9BDF.val[0], vw01234567, 2);
91*4bdc9457SAndroid Build Coastguard Worker 
92*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x8ACE9BDF.val[0], vw01234567, 5);
93*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x8ACE9BDF.val[0], vw01234567, 5);
94*4bdc9457SAndroid Build Coastguard Worker 
95*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_lane_f16(vo0p0, vi2x8ACE9BDF.val[0], vw89, 0);
96*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_lane_f16(vo1p0, vi4x8ACE9BDF.val[0], vw89, 0);
97*4bdc9457SAndroid Build Coastguard Worker 
98*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi0x7BDF = vext_f16(vi0x1357, vi0x8ACE9BDF.val[1], 3);
99*4bdc9457SAndroid Build Coastguard Worker       vi0x1357 = vi0x8ACE9BDF.val[1];
100*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi1x7BDF = vext_f16(vi1x1357, vi1x8ACE9BDF.val[1], 3);
101*4bdc9457SAndroid Build Coastguard Worker       vi1x1357 = vi1x8ACE9BDF.val[1];
102*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi2x7BDF = vext_f16(vi2x1357, vi2x8ACE9BDF.val[1], 3);
103*4bdc9457SAndroid Build Coastguard Worker       vi2x1357 = vi2x8ACE9BDF.val[1];
104*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi3x7BDF = vext_f16(vi3x1357, vi3x8ACE9BDF.val[1], 3);
105*4bdc9457SAndroid Build Coastguard Worker       vi3x1357 = vi3x8ACE9BDF.val[1];
106*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi4x7BDF = vext_f16(vi4x1357, vi4x8ACE9BDF.val[1], 3);
107*4bdc9457SAndroid Build Coastguard Worker       vi4x1357 = vi4x8ACE9BDF.val[1];
108*4bdc9457SAndroid Build Coastguard Worker 
109*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x7BDF, vw01234567, 1);
110*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x7BDF, vw01234567, 1);
111*4bdc9457SAndroid Build Coastguard Worker 
112*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x7BDF, vw01234567, 4);
113*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x7BDF, vw01234567, 4);
114*4bdc9457SAndroid Build Coastguard Worker 
115*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi2x7BDF, vw01234567, 7);
116*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi4x7BDF, vw01234567, 7);
117*4bdc9457SAndroid Build Coastguard Worker 
118*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x8ACE9BDF.val[1], vw01234567, 3);
119*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x8ACE9BDF.val[1], vw01234567, 3);
120*4bdc9457SAndroid Build Coastguard Worker 
121*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x8ACE9BDF.val[1], vw01234567, 6);
122*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x8ACE9BDF.val[1], vw01234567, 6);
123*4bdc9457SAndroid Build Coastguard Worker 
124*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_lane_f16(vo0p0, vi2x8ACE9BDF.val[1], vw89, 1);
125*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_lane_f16(vo1p0, vi4x8ACE9BDF.val[1], vw89, 1);
126*4bdc9457SAndroid Build Coastguard Worker 
127*4bdc9457SAndroid Build Coastguard Worker 
128*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo0 = vmax_f16(vo0p0, vmin);
129*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo1 = vmax_f16(vo1p0, vmin);
130*4bdc9457SAndroid Build Coastguard Worker 
131*4bdc9457SAndroid Build Coastguard Worker       vo0 = vmin_f16(vo0, vmax);
132*4bdc9457SAndroid Build Coastguard Worker       vo1 = vmin_f16(vo1, vmax);
133*4bdc9457SAndroid Build Coastguard Worker 
134*4bdc9457SAndroid Build Coastguard Worker       vst1_f16(o1, vo1); o1 += 4;
135*4bdc9457SAndroid Build Coastguard Worker       vst1_f16(o0, vo0); o0 += 4;
136*4bdc9457SAndroid Build Coastguard Worker     }
137*4bdc9457SAndroid Build Coastguard Worker     // Last block has 0-7 pixels to process.
138*4bdc9457SAndroid Build Coastguard Worker     assert(w < 8 * sizeof(__fp16));
139*4bdc9457SAndroid Build Coastguard Worker     if XNN_LIKELY(w != 0) {
140*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo0p0 = vdup_laneq_f16(vw01234567, 0);
141*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo1p0 = vdup_laneq_f16(vw01234567, 0);
142*4bdc9457SAndroid Build Coastguard Worker 
143*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi0x8ACE9BDF = vld2_f16(i0);
144*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi1x8ACE9BDF = vld2_f16(i1);
145*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi2x8ACE9BDF = vld2_f16(i2);
146*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi3x8ACE9BDF = vld2_f16(i3);
147*4bdc9457SAndroid Build Coastguard Worker       const float16x4x2_t vi4x8ACE9BDF = vld2_f16(i4);
148*4bdc9457SAndroid Build Coastguard Worker 
149*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi0x8ACE = vreinterpret_f16_u16(vand_u16(vmask_even, vreinterpret_u16_f16(vi0x8ACE9BDF.val[0])));
150*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi0x9BDF = vreinterpret_f16_u16(vand_u16(vmask_odd,  vreinterpret_u16_f16(vi0x8ACE9BDF.val[1])));
151*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi1x8ACE = vreinterpret_f16_u16(vand_u16(vmask_even, vreinterpret_u16_f16(vi1x8ACE9BDF.val[0])));
152*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi1x9BDF = vreinterpret_f16_u16(vand_u16(vmask_odd,  vreinterpret_u16_f16(vi1x8ACE9BDF.val[1])));
153*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi2x8ACE = vreinterpret_f16_u16(vand_u16(vmask_even, vreinterpret_u16_f16(vi2x8ACE9BDF.val[0])));
154*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi2x9BDF = vreinterpret_f16_u16(vand_u16(vmask_odd,  vreinterpret_u16_f16(vi2x8ACE9BDF.val[1])));
155*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi3x8ACE = vreinterpret_f16_u16(vand_u16(vmask_even, vreinterpret_u16_f16(vi3x8ACE9BDF.val[0])));
156*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi3x9BDF = vreinterpret_f16_u16(vand_u16(vmask_odd,  vreinterpret_u16_f16(vi3x8ACE9BDF.val[1])));
157*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi4x8ACE = vreinterpret_f16_u16(vand_u16(vmask_even, vreinterpret_u16_f16(vi4x8ACE9BDF.val[0])));
158*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi4x9BDF = vreinterpret_f16_u16(vand_u16(vmask_odd,  vreinterpret_u16_f16(vi4x8ACE9BDF.val[1])));
159*4bdc9457SAndroid Build Coastguard Worker 
160*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x8ACE, vw01234567, 2);
161*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x8ACE, vw01234567, 2);
162*4bdc9457SAndroid Build Coastguard Worker 
163*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x8ACE, vw01234567, 5);
164*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x8ACE, vw01234567, 5);
165*4bdc9457SAndroid Build Coastguard Worker 
166*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_lane_f16(vo0p0, vi2x8ACE, vw89, 0);
167*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_lane_f16(vo1p0, vi4x8ACE, vw89, 0);
168*4bdc9457SAndroid Build Coastguard Worker 
169*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi0x7BDF = vext_f16(vi0x1357, vi0x9BDF, 3);
170*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi1x7BDF = vext_f16(vi1x1357, vi1x9BDF, 3);
171*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi2x7BDF = vext_f16(vi2x1357, vi2x9BDF, 3);
172*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi3x7BDF = vext_f16(vi3x1357, vi3x9BDF, 3);
173*4bdc9457SAndroid Build Coastguard Worker       const float16x4_t vi4x7BDF = vext_f16(vi4x1357, vi4x9BDF, 3);
174*4bdc9457SAndroid Build Coastguard Worker 
175*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x7BDF, vw01234567, 1);
176*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x7BDF, vw01234567, 1);
177*4bdc9457SAndroid Build Coastguard Worker 
178*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x7BDF, vw01234567, 4);
179*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x7BDF, vw01234567, 4);
180*4bdc9457SAndroid Build Coastguard Worker 
181*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi2x7BDF, vw01234567, 7);
182*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi4x7BDF, vw01234567, 7);
183*4bdc9457SAndroid Build Coastguard Worker 
184*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi0x9BDF, vw01234567, 3);
185*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi2x9BDF, vw01234567, 3);
186*4bdc9457SAndroid Build Coastguard Worker 
187*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_laneq_f16(vo0p0, vi1x9BDF, vw01234567, 6);
188*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_laneq_f16(vo1p0, vi3x9BDF, vw01234567, 6);
189*4bdc9457SAndroid Build Coastguard Worker 
190*4bdc9457SAndroid Build Coastguard Worker       vo0p0 = vfma_lane_f16(vo0p0, vi2x9BDF, vw89, 1);
191*4bdc9457SAndroid Build Coastguard Worker       vo1p0 = vfma_lane_f16(vo1p0, vi4x9BDF, vw89, 1);
192*4bdc9457SAndroid Build Coastguard Worker 
193*4bdc9457SAndroid Build Coastguard Worker 
194*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo0 = vmax_f16(vo0p0, vmin);
195*4bdc9457SAndroid Build Coastguard Worker       float16x4_t vo1 = vmax_f16(vo1p0, vmin);
196*4bdc9457SAndroid Build Coastguard Worker 
197*4bdc9457SAndroid Build Coastguard Worker       vo0 = vmin_f16(vo0, vmax);
198*4bdc9457SAndroid Build Coastguard Worker       vo1 = vmin_f16(vo1, vmax);
199*4bdc9457SAndroid Build Coastguard Worker 
200*4bdc9457SAndroid Build Coastguard Worker       w += 1 * sizeof(__fp16);
201*4bdc9457SAndroid Build Coastguard Worker 
202*4bdc9457SAndroid Build Coastguard Worker       if XNN_LIKELY(w == 8 * sizeof(__fp16)) {
203*4bdc9457SAndroid Build Coastguard Worker         vst1_f16(o1, vo1); o1 += 4;
204*4bdc9457SAndroid Build Coastguard Worker         vst1_f16(o0, vo0); o0 += 4;
205*4bdc9457SAndroid Build Coastguard Worker       } else {
206*4bdc9457SAndroid Build Coastguard Worker         if (w & (4 * sizeof(__fp16))) {
207*4bdc9457SAndroid Build Coastguard Worker           vst1_lane_u32((void*) o1, vreinterpret_u32_f16(vo1), 0); o1 += 2;
208*4bdc9457SAndroid Build Coastguard Worker           vst1_lane_u32((void*) o0, vreinterpret_u32_f16(vo0), 0); o0 += 2;
209*4bdc9457SAndroid Build Coastguard Worker 
210*4bdc9457SAndroid Build Coastguard Worker           vo0 = vext_f16(vo0, vo0, 2);
211*4bdc9457SAndroid Build Coastguard Worker           vo1 = vext_f16(vo1, vo1, 2);
212*4bdc9457SAndroid Build Coastguard Worker         }
213*4bdc9457SAndroid Build Coastguard Worker         if (w & (2 * sizeof(__fp16))) {
214*4bdc9457SAndroid Build Coastguard Worker           vst1_lane_f16(o1, vo1, 0); o1 += 1;
215*4bdc9457SAndroid Build Coastguard Worker           vst1_lane_f16(o0, vo0, 0); o0 += 1;
216*4bdc9457SAndroid Build Coastguard Worker         }
217*4bdc9457SAndroid Build Coastguard Worker       }
218*4bdc9457SAndroid Build Coastguard Worker 
219*4bdc9457SAndroid Build Coastguard Worker     }
220*4bdc9457SAndroid Build Coastguard Worker 
221*4bdc9457SAndroid Build Coastguard Worker     i0 = (const __fp16*) ((uintptr_t) i4 - input_decrement);
222*4bdc9457SAndroid Build Coastguard Worker     i1 = (const __fp16*) ((uintptr_t) i0 + input_width);
223*4bdc9457SAndroid Build Coastguard Worker     i2 = (const __fp16*) ((uintptr_t) i1 + input_width);
224*4bdc9457SAndroid Build Coastguard Worker     i3 = (const __fp16*) ((uintptr_t) i2 + input_width);
225*4bdc9457SAndroid Build Coastguard Worker     i4 = (const __fp16*) ((uintptr_t) i3 + input_width);
226*4bdc9457SAndroid Build Coastguard Worker 
227*4bdc9457SAndroid Build Coastguard Worker     o0 = o1;
228*4bdc9457SAndroid Build Coastguard Worker     o1 = (__fp16*) ((uintptr_t) o0 + output_width);
229*4bdc9457SAndroid Build Coastguard Worker 
230*4bdc9457SAndroid Build Coastguard Worker     output_height = doz(output_height, 2);
231*4bdc9457SAndroid Build Coastguard Worker     padded_input_height = doz(padded_input_height, 4);
232*4bdc9457SAndroid Build Coastguard Worker   } while (output_height != 0);
233*4bdc9457SAndroid Build Coastguard Worker }
234