1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f16-dwconv2d-chw/5x5p2-neonfp16arith.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2020 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <arm_neon.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/dwconv.h>
15*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/math.h>
16*4bdc9457SAndroid Build Coastguard Worker
17*4bdc9457SAndroid Build Coastguard Worker
xnn_f16_dwconv2d_chw_ukernel_5x5p2__neonfp16arith_2x4(size_t input_height,size_t input_width,const void * input,const void * weights,const void * zero,void * output,uint32_t padding_top,const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS (1)])18*4bdc9457SAndroid Build Coastguard Worker void xnn_f16_dwconv2d_chw_ukernel_5x5p2__neonfp16arith_2x4(
19*4bdc9457SAndroid Build Coastguard Worker size_t input_height,
20*4bdc9457SAndroid Build Coastguard Worker size_t input_width,
21*4bdc9457SAndroid Build Coastguard Worker const void* input,
22*4bdc9457SAndroid Build Coastguard Worker const void* weights,
23*4bdc9457SAndroid Build Coastguard Worker const void* zero,
24*4bdc9457SAndroid Build Coastguard Worker void* output,
25*4bdc9457SAndroid Build Coastguard Worker uint32_t padding_top,
26*4bdc9457SAndroid Build Coastguard Worker const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker assert(input_height != 0);
29*4bdc9457SAndroid Build Coastguard Worker assert(input_width != 0);
30*4bdc9457SAndroid Build Coastguard Worker assert(input_width % sizeof(__fp16) == 0);
31*4bdc9457SAndroid Build Coastguard Worker assert(padding_top == 2);
32*4bdc9457SAndroid Build Coastguard Worker
33*4bdc9457SAndroid Build Coastguard Worker const uint16x4_t vmask = vld1_u16(params->neonfp16arith.mask);
34*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vmax = vld1_dup_f16(¶ms->neonfp16arith.max);
35*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vmin = vld1_dup_f16(¶ms->neonfp16arith.min);
36*4bdc9457SAndroid Build Coastguard Worker
37*4bdc9457SAndroid Build Coastguard Worker const __fp16* w0 = (const __fp16*)weights;
38*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vw01234567 = vld1q_f16(w0);
39*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vw89ABCDEF = vld1q_f16(w0 + 8);
40*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vwGHIJKLMN = vld1q_f16(w0 + 16);
41*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vwOP = vreinterpret_f16_u32(vld1_lane_u32((const void*)(w0 + 24), vmov_n_u32(0), 0));
42*4bdc9457SAndroid Build Coastguard Worker
43*4bdc9457SAndroid Build Coastguard Worker const size_t input_decrement = round_up_po2(input_width, 4 * sizeof(__fp16));
44*4bdc9457SAndroid Build Coastguard Worker
45*4bdc9457SAndroid Build Coastguard Worker const __fp16* i0 = zero;
46*4bdc9457SAndroid Build Coastguard Worker const __fp16* i1 = zero;
47*4bdc9457SAndroid Build Coastguard Worker const __fp16* i2 = input;
48*4bdc9457SAndroid Build Coastguard Worker const __fp16* i3 = (const __fp16*) ((uintptr_t) i2 + input_width);
49*4bdc9457SAndroid Build Coastguard Worker const __fp16* i4 = (const __fp16*) ((uintptr_t) i3 + input_width);
50*4bdc9457SAndroid Build Coastguard Worker const __fp16* i5 = (const __fp16*) ((uintptr_t) i4 + input_width);
51*4bdc9457SAndroid Build Coastguard Worker
52*4bdc9457SAndroid Build Coastguard Worker __fp16* o0 = output;
53*4bdc9457SAndroid Build Coastguard Worker __fp16* o1 = (__fp16*) ((uintptr_t) o0 + input_width);
54*4bdc9457SAndroid Build Coastguard Worker
55*4bdc9457SAndroid Build Coastguard Worker size_t output_height = input_height;
56*4bdc9457SAndroid Build Coastguard Worker do {
57*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(output_height < 2) {
58*4bdc9457SAndroid Build Coastguard Worker i3 = zero;
59*4bdc9457SAndroid Build Coastguard Worker o1 = o0;
60*4bdc9457SAndroid Build Coastguard Worker }
61*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(output_height < 3) {
62*4bdc9457SAndroid Build Coastguard Worker i4 = zero;
63*4bdc9457SAndroid Build Coastguard Worker }
64*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(output_height < 4) {
65*4bdc9457SAndroid Build Coastguard Worker i5 = zero;
66*4bdc9457SAndroid Build Coastguard Worker }
67*4bdc9457SAndroid Build Coastguard Worker
68*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi0x0123 = vmov_n_f16(0);
69*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi1x0123 = vmov_n_f16(0);
70*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi2x0123 = vmov_n_f16(0);
71*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi3x0123 = vmov_n_f16(0);
72*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi4x0123 = vmov_n_f16(0);
73*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi5x0123 = vmov_n_f16(0);
74*4bdc9457SAndroid Build Coastguard Worker
75*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi0x4567 = vld1_f16(i0); i0 += 4;
76*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi1x4567 = vld1_f16(i1); i1 += 4;
77*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi2x4567 = vld1_f16(i2); i2 += 4;
78*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi3x4567 = vld1_f16(i3); i3 += 4;
79*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi4x4567 = vld1_f16(i4); i4 += 4;
80*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi5x4567 = vld1_f16(i5); i5 += 4;
81*4bdc9457SAndroid Build Coastguard Worker
82*4bdc9457SAndroid Build Coastguard Worker size_t w = input_width;
83*4bdc9457SAndroid Build Coastguard Worker for (; w > 8 * sizeof(__fp16); w -= 4 * sizeof(__fp16)) {
84*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0p0 = vdup_laneq_f16(vw01234567, 0);
85*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1p0 = vdup_laneq_f16(vw01234567, 0);
86*4bdc9457SAndroid Build Coastguard Worker
87*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x89AB = vld1_f16(i0); i0 += 4;
88*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x89AB = vld1_f16(i1); i1 += 4;
89*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x89AB = vld1_f16(i2); i2 += 4;
90*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x89AB = vld1_f16(i3); i3 += 4;
91*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x89AB = vld1_f16(i4); i4 += 4;
92*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x89AB = vld1_f16(i5); i5 += 4;
93*4bdc9457SAndroid Build Coastguard Worker
94*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x4567, vw01234567, 3);
95*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x4567, vw01234567, 3);
96*4bdc9457SAndroid Build Coastguard Worker
97*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x4567, vw89ABCDEF, 0);
98*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x4567, vw89ABCDEF, 0);
99*4bdc9457SAndroid Build Coastguard Worker
100*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x4567, vw89ABCDEF, 5);
101*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x4567, vw89ABCDEF, 5);
102*4bdc9457SAndroid Build Coastguard Worker
103*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x4567, vwGHIJKLMN, 2);
104*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x4567, vwGHIJKLMN, 2);
105*4bdc9457SAndroid Build Coastguard Worker
106*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x4567, vwGHIJKLMN, 7);
107*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x4567, vwGHIJKLMN, 7);
108*4bdc9457SAndroid Build Coastguard Worker
109*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x3456 = vext_f16(vi0x0123, vi0x4567, 3);
110*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x3456 = vext_f16(vi1x0123, vi1x4567, 3);
111*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x3456 = vext_f16(vi2x0123, vi2x4567, 3);
112*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x3456 = vext_f16(vi3x0123, vi3x4567, 3);
113*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x3456 = vext_f16(vi4x0123, vi4x4567, 3);
114*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x3456 = vext_f16(vi5x0123, vi5x4567, 3);
115*4bdc9457SAndroid Build Coastguard Worker
116*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x3456, vw01234567, 2);
117*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x3456, vw01234567, 2);
118*4bdc9457SAndroid Build Coastguard Worker
119*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x3456, vw01234567, 7);
120*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x3456, vw01234567, 7);
121*4bdc9457SAndroid Build Coastguard Worker
122*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x3456, vw89ABCDEF, 4);
123*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x3456, vw89ABCDEF, 4);
124*4bdc9457SAndroid Build Coastguard Worker
125*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x3456, vwGHIJKLMN, 1);
126*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x3456, vwGHIJKLMN, 1);
127*4bdc9457SAndroid Build Coastguard Worker
128*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x3456, vwGHIJKLMN, 6);
129*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x3456, vwGHIJKLMN, 6);
130*4bdc9457SAndroid Build Coastguard Worker
131*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x2345 = vext_f16(vi0x0123, vi0x4567, 2);
132*4bdc9457SAndroid Build Coastguard Worker vi0x0123 = vi0x4567;
133*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x2345 = vext_f16(vi1x0123, vi1x4567, 2);
134*4bdc9457SAndroid Build Coastguard Worker vi1x0123 = vi1x4567;
135*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x2345 = vext_f16(vi2x0123, vi2x4567, 2);
136*4bdc9457SAndroid Build Coastguard Worker vi2x0123 = vi2x4567;
137*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x2345 = vext_f16(vi3x0123, vi3x4567, 2);
138*4bdc9457SAndroid Build Coastguard Worker vi3x0123 = vi3x4567;
139*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x2345 = vext_f16(vi4x0123, vi4x4567, 2);
140*4bdc9457SAndroid Build Coastguard Worker vi4x0123 = vi4x4567;
141*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x2345 = vext_f16(vi5x0123, vi5x4567, 2);
142*4bdc9457SAndroid Build Coastguard Worker vi5x0123 = vi5x4567;
143*4bdc9457SAndroid Build Coastguard Worker
144*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x2345, vw01234567, 1);
145*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x2345, vw01234567, 1);
146*4bdc9457SAndroid Build Coastguard Worker
147*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x2345, vw01234567, 6);
148*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x2345, vw01234567, 6);
149*4bdc9457SAndroid Build Coastguard Worker
150*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x2345, vw89ABCDEF, 3);
151*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x2345, vw89ABCDEF, 3);
152*4bdc9457SAndroid Build Coastguard Worker
153*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x2345, vwGHIJKLMN, 0);
154*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x2345, vwGHIJKLMN, 0);
155*4bdc9457SAndroid Build Coastguard Worker
156*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x2345, vwGHIJKLMN, 5);
157*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x2345, vwGHIJKLMN, 5);
158*4bdc9457SAndroid Build Coastguard Worker
159*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x5678 = vext_f16(vi0x4567, vi0x89AB, 1);
160*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x5678 = vext_f16(vi1x4567, vi1x89AB, 1);
161*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x5678 = vext_f16(vi2x4567, vi2x89AB, 1);
162*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x5678 = vext_f16(vi3x4567, vi3x89AB, 1);
163*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x5678 = vext_f16(vi4x4567, vi4x89AB, 1);
164*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x5678 = vext_f16(vi5x4567, vi5x89AB, 1);
165*4bdc9457SAndroid Build Coastguard Worker
166*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x5678, vw01234567, 4);
167*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x5678, vw01234567, 4);
168*4bdc9457SAndroid Build Coastguard Worker
169*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x5678, vw89ABCDEF, 1);
170*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x5678, vw89ABCDEF, 1);
171*4bdc9457SAndroid Build Coastguard Worker
172*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x5678, vw89ABCDEF, 6);
173*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x5678, vw89ABCDEF, 6);
174*4bdc9457SAndroid Build Coastguard Worker
175*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x5678, vwGHIJKLMN, 3);
176*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x5678, vwGHIJKLMN, 3);
177*4bdc9457SAndroid Build Coastguard Worker
178*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x5678, vwOP, 0);
179*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x5678, vwOP, 0);
180*4bdc9457SAndroid Build Coastguard Worker
181*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x6789 = vext_f16(vi0x4567, vi0x89AB, 2);
182*4bdc9457SAndroid Build Coastguard Worker vi0x4567 = vi0x89AB;
183*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x6789 = vext_f16(vi1x4567, vi1x89AB, 2);
184*4bdc9457SAndroid Build Coastguard Worker vi1x4567 = vi1x89AB;
185*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x6789 = vext_f16(vi2x4567, vi2x89AB, 2);
186*4bdc9457SAndroid Build Coastguard Worker vi2x4567 = vi2x89AB;
187*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x6789 = vext_f16(vi3x4567, vi3x89AB, 2);
188*4bdc9457SAndroid Build Coastguard Worker vi3x4567 = vi3x89AB;
189*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x6789 = vext_f16(vi4x4567, vi4x89AB, 2);
190*4bdc9457SAndroid Build Coastguard Worker vi4x4567 = vi4x89AB;
191*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x6789 = vext_f16(vi5x4567, vi5x89AB, 2);
192*4bdc9457SAndroid Build Coastguard Worker vi5x4567 = vi5x89AB;
193*4bdc9457SAndroid Build Coastguard Worker
194*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x6789, vw01234567, 5);
195*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x6789, vw01234567, 5);
196*4bdc9457SAndroid Build Coastguard Worker
197*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x6789, vw89ABCDEF, 2);
198*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x6789, vw89ABCDEF, 2);
199*4bdc9457SAndroid Build Coastguard Worker
200*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x6789, vw89ABCDEF, 7);
201*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x6789, vw89ABCDEF, 7);
202*4bdc9457SAndroid Build Coastguard Worker
203*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x6789, vwGHIJKLMN, 4);
204*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x6789, vwGHIJKLMN, 4);
205*4bdc9457SAndroid Build Coastguard Worker
206*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x6789, vwOP, 1);
207*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x6789, vwOP, 1);
208*4bdc9457SAndroid Build Coastguard Worker
209*4bdc9457SAndroid Build Coastguard Worker
210*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0 = vmax_f16(vo0p0, vmin);
211*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1 = vmax_f16(vo1p0, vmin);
212*4bdc9457SAndroid Build Coastguard Worker
213*4bdc9457SAndroid Build Coastguard Worker vo0 = vmin_f16(vo0, vmax);
214*4bdc9457SAndroid Build Coastguard Worker vo1 = vmin_f16(vo1, vmax);
215*4bdc9457SAndroid Build Coastguard Worker
216*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o1, vo1); o1 += 4;
217*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o0, vo0); o0 += 4;
218*4bdc9457SAndroid Build Coastguard Worker }
219*4bdc9457SAndroid Build Coastguard Worker // Always process the last block of 5..8 pixels.
220*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(w > 4 * sizeof(__fp16)) {
221*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0p0 = vdup_laneq_f16(vw01234567, 0);
222*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1p0 = vdup_laneq_f16(vw01234567, 0);
223*4bdc9457SAndroid Build Coastguard Worker
224*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi0x89AB = vld1_f16(i0); i0 += 4;
225*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi1x89AB = vld1_f16(i1); i1 += 4;
226*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi2x89AB = vld1_f16(i2); i2 += 4;
227*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi3x89AB = vld1_f16(i3); i3 += 4;
228*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi4x89AB = vld1_f16(i4); i4 += 4;
229*4bdc9457SAndroid Build Coastguard Worker float16x4_t vi5x89AB = vld1_f16(i5); i5 += 4;
230*4bdc9457SAndroid Build Coastguard Worker
231*4bdc9457SAndroid Build Coastguard Worker vi0x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi0x89AB)));
232*4bdc9457SAndroid Build Coastguard Worker vi1x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi1x89AB)));
233*4bdc9457SAndroid Build Coastguard Worker vi2x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi2x89AB)));
234*4bdc9457SAndroid Build Coastguard Worker vi3x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi3x89AB)));
235*4bdc9457SAndroid Build Coastguard Worker vi4x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi4x89AB)));
236*4bdc9457SAndroid Build Coastguard Worker vi5x89AB = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi5x89AB)));
237*4bdc9457SAndroid Build Coastguard Worker
238*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x4567, vw01234567, 3);
239*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x4567, vw01234567, 3);
240*4bdc9457SAndroid Build Coastguard Worker
241*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x4567, vw89ABCDEF, 0);
242*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x4567, vw89ABCDEF, 0);
243*4bdc9457SAndroid Build Coastguard Worker
244*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x4567, vw89ABCDEF, 5);
245*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x4567, vw89ABCDEF, 5);
246*4bdc9457SAndroid Build Coastguard Worker
247*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x4567, vwGHIJKLMN, 2);
248*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x4567, vwGHIJKLMN, 2);
249*4bdc9457SAndroid Build Coastguard Worker
250*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x4567, vwGHIJKLMN, 7);
251*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x4567, vwGHIJKLMN, 7);
252*4bdc9457SAndroid Build Coastguard Worker
253*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x3456 = vext_f16(vi0x0123, vi0x4567, 3);
254*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x3456 = vext_f16(vi1x0123, vi1x4567, 3);
255*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x3456 = vext_f16(vi2x0123, vi2x4567, 3);
256*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x3456 = vext_f16(vi3x0123, vi3x4567, 3);
257*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x3456 = vext_f16(vi4x0123, vi4x4567, 3);
258*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x3456 = vext_f16(vi5x0123, vi5x4567, 3);
259*4bdc9457SAndroid Build Coastguard Worker
260*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x3456, vw01234567, 2);
261*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x3456, vw01234567, 2);
262*4bdc9457SAndroid Build Coastguard Worker
263*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x3456, vw01234567, 7);
264*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x3456, vw01234567, 7);
265*4bdc9457SAndroid Build Coastguard Worker
266*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x3456, vw89ABCDEF, 4);
267*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x3456, vw89ABCDEF, 4);
268*4bdc9457SAndroid Build Coastguard Worker
269*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x3456, vwGHIJKLMN, 1);
270*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x3456, vwGHIJKLMN, 1);
271*4bdc9457SAndroid Build Coastguard Worker
272*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x3456, vwGHIJKLMN, 6);
273*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x3456, vwGHIJKLMN, 6);
274*4bdc9457SAndroid Build Coastguard Worker
275*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x2345 = vext_f16(vi0x0123, vi0x4567, 2);
276*4bdc9457SAndroid Build Coastguard Worker vi0x0123 = vi0x4567;
277*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x2345 = vext_f16(vi1x0123, vi1x4567, 2);
278*4bdc9457SAndroid Build Coastguard Worker vi1x0123 = vi1x4567;
279*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x2345 = vext_f16(vi2x0123, vi2x4567, 2);
280*4bdc9457SAndroid Build Coastguard Worker vi2x0123 = vi2x4567;
281*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x2345 = vext_f16(vi3x0123, vi3x4567, 2);
282*4bdc9457SAndroid Build Coastguard Worker vi3x0123 = vi3x4567;
283*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x2345 = vext_f16(vi4x0123, vi4x4567, 2);
284*4bdc9457SAndroid Build Coastguard Worker vi4x0123 = vi4x4567;
285*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x2345 = vext_f16(vi5x0123, vi5x4567, 2);
286*4bdc9457SAndroid Build Coastguard Worker vi5x0123 = vi5x4567;
287*4bdc9457SAndroid Build Coastguard Worker
288*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x2345, vw01234567, 1);
289*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x2345, vw01234567, 1);
290*4bdc9457SAndroid Build Coastguard Worker
291*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x2345, vw01234567, 6);
292*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x2345, vw01234567, 6);
293*4bdc9457SAndroid Build Coastguard Worker
294*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x2345, vw89ABCDEF, 3);
295*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x2345, vw89ABCDEF, 3);
296*4bdc9457SAndroid Build Coastguard Worker
297*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x2345, vwGHIJKLMN, 0);
298*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x2345, vwGHIJKLMN, 0);
299*4bdc9457SAndroid Build Coastguard Worker
300*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x2345, vwGHIJKLMN, 5);
301*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x2345, vwGHIJKLMN, 5);
302*4bdc9457SAndroid Build Coastguard Worker
303*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x5678 = vext_f16(vi0x4567, vi0x89AB, 1);
304*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x5678 = vext_f16(vi1x4567, vi1x89AB, 1);
305*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x5678 = vext_f16(vi2x4567, vi2x89AB, 1);
306*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x5678 = vext_f16(vi3x4567, vi3x89AB, 1);
307*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x5678 = vext_f16(vi4x4567, vi4x89AB, 1);
308*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x5678 = vext_f16(vi5x4567, vi5x89AB, 1);
309*4bdc9457SAndroid Build Coastguard Worker
310*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x5678, vw01234567, 4);
311*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x5678, vw01234567, 4);
312*4bdc9457SAndroid Build Coastguard Worker
313*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x5678, vw89ABCDEF, 1);
314*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x5678, vw89ABCDEF, 1);
315*4bdc9457SAndroid Build Coastguard Worker
316*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x5678, vw89ABCDEF, 6);
317*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x5678, vw89ABCDEF, 6);
318*4bdc9457SAndroid Build Coastguard Worker
319*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x5678, vwGHIJKLMN, 3);
320*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x5678, vwGHIJKLMN, 3);
321*4bdc9457SAndroid Build Coastguard Worker
322*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x5678, vwOP, 0);
323*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x5678, vwOP, 0);
324*4bdc9457SAndroid Build Coastguard Worker
325*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x6789 = vext_f16(vi0x4567, vi0x89AB, 2);
326*4bdc9457SAndroid Build Coastguard Worker vi0x4567 = vi0x89AB;
327*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x6789 = vext_f16(vi1x4567, vi1x89AB, 2);
328*4bdc9457SAndroid Build Coastguard Worker vi1x4567 = vi1x89AB;
329*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x6789 = vext_f16(vi2x4567, vi2x89AB, 2);
330*4bdc9457SAndroid Build Coastguard Worker vi2x4567 = vi2x89AB;
331*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x6789 = vext_f16(vi3x4567, vi3x89AB, 2);
332*4bdc9457SAndroid Build Coastguard Worker vi3x4567 = vi3x89AB;
333*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x6789 = vext_f16(vi4x4567, vi4x89AB, 2);
334*4bdc9457SAndroid Build Coastguard Worker vi4x4567 = vi4x89AB;
335*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x6789 = vext_f16(vi5x4567, vi5x89AB, 2);
336*4bdc9457SAndroid Build Coastguard Worker vi5x4567 = vi5x89AB;
337*4bdc9457SAndroid Build Coastguard Worker
338*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x6789, vw01234567, 5);
339*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x6789, vw01234567, 5);
340*4bdc9457SAndroid Build Coastguard Worker
341*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x6789, vw89ABCDEF, 2);
342*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x6789, vw89ABCDEF, 2);
343*4bdc9457SAndroid Build Coastguard Worker
344*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x6789, vw89ABCDEF, 7);
345*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x6789, vw89ABCDEF, 7);
346*4bdc9457SAndroid Build Coastguard Worker
347*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x6789, vwGHIJKLMN, 4);
348*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x6789, vwGHIJKLMN, 4);
349*4bdc9457SAndroid Build Coastguard Worker
350*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x6789, vwOP, 1);
351*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x6789, vwOP, 1);
352*4bdc9457SAndroid Build Coastguard Worker
353*4bdc9457SAndroid Build Coastguard Worker
354*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0 = vmax_f16(vo0p0, vmin);
355*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1 = vmax_f16(vo1p0, vmin);
356*4bdc9457SAndroid Build Coastguard Worker
357*4bdc9457SAndroid Build Coastguard Worker vo0 = vmin_f16(vo0, vmax);
358*4bdc9457SAndroid Build Coastguard Worker vo1 = vmin_f16(vo1, vmax);
359*4bdc9457SAndroid Build Coastguard Worker
360*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o1, vo1); o1 += 4;
361*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o0, vo0); o0 += 4;
362*4bdc9457SAndroid Build Coastguard Worker
363*4bdc9457SAndroid Build Coastguard Worker w -= 4 * sizeof(__fp16);
364*4bdc9457SAndroid Build Coastguard Worker }
365*4bdc9457SAndroid Build Coastguard Worker assert(w >= 1 * sizeof(__fp16));
366*4bdc9457SAndroid Build Coastguard Worker assert(w <= 4 * sizeof(__fp16));
367*4bdc9457SAndroid Build Coastguard Worker {
368*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0p0 = vdup_laneq_f16(vw01234567, 0);
369*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1p0 = vdup_laneq_f16(vw01234567, 0);
370*4bdc9457SAndroid Build Coastguard Worker
371*4bdc9457SAndroid Build Coastguard Worker vi0x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi0x4567)));
372*4bdc9457SAndroid Build Coastguard Worker vi1x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi1x4567)));
373*4bdc9457SAndroid Build Coastguard Worker vi2x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi2x4567)));
374*4bdc9457SAndroid Build Coastguard Worker vi3x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi3x4567)));
375*4bdc9457SAndroid Build Coastguard Worker vi4x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi4x4567)));
376*4bdc9457SAndroid Build Coastguard Worker vi5x4567 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi5x4567)));
377*4bdc9457SAndroid Build Coastguard Worker
378*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x4567, vw01234567, 3);
379*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x4567, vw01234567, 3);
380*4bdc9457SAndroid Build Coastguard Worker
381*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x4567, vw89ABCDEF, 0);
382*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x4567, vw89ABCDEF, 0);
383*4bdc9457SAndroid Build Coastguard Worker
384*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x4567, vw89ABCDEF, 5);
385*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x4567, vw89ABCDEF, 5);
386*4bdc9457SAndroid Build Coastguard Worker
387*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x4567, vwGHIJKLMN, 2);
388*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x4567, vwGHIJKLMN, 2);
389*4bdc9457SAndroid Build Coastguard Worker
390*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x4567, vwGHIJKLMN, 7);
391*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x4567, vwGHIJKLMN, 7);
392*4bdc9457SAndroid Build Coastguard Worker
393*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x3456 = vext_f16(vi0x0123, vi0x4567, 3);
394*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x3456 = vext_f16(vi1x0123, vi1x4567, 3);
395*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x3456 = vext_f16(vi2x0123, vi2x4567, 3);
396*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x3456 = vext_f16(vi3x0123, vi3x4567, 3);
397*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x3456 = vext_f16(vi4x0123, vi4x4567, 3);
398*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x3456 = vext_f16(vi5x0123, vi5x4567, 3);
399*4bdc9457SAndroid Build Coastguard Worker
400*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x3456, vw01234567, 2);
401*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x3456, vw01234567, 2);
402*4bdc9457SAndroid Build Coastguard Worker
403*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x3456, vw01234567, 7);
404*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x3456, vw01234567, 7);
405*4bdc9457SAndroid Build Coastguard Worker
406*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x3456, vw89ABCDEF, 4);
407*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x3456, vw89ABCDEF, 4);
408*4bdc9457SAndroid Build Coastguard Worker
409*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x3456, vwGHIJKLMN, 1);
410*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x3456, vwGHIJKLMN, 1);
411*4bdc9457SAndroid Build Coastguard Worker
412*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x3456, vwGHIJKLMN, 6);
413*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x3456, vwGHIJKLMN, 6);
414*4bdc9457SAndroid Build Coastguard Worker
415*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x2345 = vext_f16(vi0x0123, vi0x4567, 2);
416*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x2345 = vext_f16(vi1x0123, vi1x4567, 2);
417*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x2345 = vext_f16(vi2x0123, vi2x4567, 2);
418*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x2345 = vext_f16(vi3x0123, vi3x4567, 2);
419*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x2345 = vext_f16(vi4x0123, vi4x4567, 2);
420*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x2345 = vext_f16(vi5x0123, vi5x4567, 2);
421*4bdc9457SAndroid Build Coastguard Worker
422*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x2345, vw01234567, 1);
423*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x2345, vw01234567, 1);
424*4bdc9457SAndroid Build Coastguard Worker
425*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x2345, vw01234567, 6);
426*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x2345, vw01234567, 6);
427*4bdc9457SAndroid Build Coastguard Worker
428*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x2345, vw89ABCDEF, 3);
429*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x2345, vw89ABCDEF, 3);
430*4bdc9457SAndroid Build Coastguard Worker
431*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x2345, vwGHIJKLMN, 0);
432*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x2345, vwGHIJKLMN, 0);
433*4bdc9457SAndroid Build Coastguard Worker
434*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi4x2345, vwGHIJKLMN, 5);
435*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi5x2345, vwGHIJKLMN, 5);
436*4bdc9457SAndroid Build Coastguard Worker
437*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vzero = vmov_n_f16(0);
438*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x5678 = vext_f16(vi0x4567, vzero, 1);
439*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x5678 = vext_f16(vi1x4567, vzero, 1);
440*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x5678 = vext_f16(vi2x4567, vzero, 1);
441*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x5678 = vext_f16(vi3x4567, vzero, 1);
442*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x5678 = vext_f16(vi4x4567, vzero, 1);
443*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x5678 = vext_f16(vi5x4567, vzero, 1);
444*4bdc9457SAndroid Build Coastguard Worker
445*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x5678, vw01234567, 4);
446*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x5678, vw01234567, 4);
447*4bdc9457SAndroid Build Coastguard Worker
448*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x5678, vw89ABCDEF, 1);
449*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x5678, vw89ABCDEF, 1);
450*4bdc9457SAndroid Build Coastguard Worker
451*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x5678, vw89ABCDEF, 6);
452*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x5678, vw89ABCDEF, 6);
453*4bdc9457SAndroid Build Coastguard Worker
454*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x5678, vwGHIJKLMN, 3);
455*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x5678, vwGHIJKLMN, 3);
456*4bdc9457SAndroid Build Coastguard Worker
457*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x5678, vwOP, 0);
458*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x5678, vwOP, 0);
459*4bdc9457SAndroid Build Coastguard Worker
460*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi0x6789 = vext_f16(vi0x5678, vzero, 1);
461*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi1x6789 = vext_f16(vi1x5678, vzero, 1);
462*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi2x6789 = vext_f16(vi2x5678, vzero, 1);
463*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi3x6789 = vext_f16(vi3x5678, vzero, 1);
464*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi4x6789 = vext_f16(vi4x5678, vzero, 1);
465*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vi5x6789 = vext_f16(vi5x5678, vzero, 1);
466*4bdc9457SAndroid Build Coastguard Worker
467*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi0x6789, vw01234567, 5);
468*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi1x6789, vw01234567, 5);
469*4bdc9457SAndroid Build Coastguard Worker
470*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi1x6789, vw89ABCDEF, 2);
471*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi2x6789, vw89ABCDEF, 2);
472*4bdc9457SAndroid Build Coastguard Worker
473*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi2x6789, vw89ABCDEF, 7);
474*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi3x6789, vw89ABCDEF, 7);
475*4bdc9457SAndroid Build Coastguard Worker
476*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_laneq_f16(vo0p0, vi3x6789, vwGHIJKLMN, 4);
477*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_laneq_f16(vo1p0, vi4x6789, vwGHIJKLMN, 4);
478*4bdc9457SAndroid Build Coastguard Worker
479*4bdc9457SAndroid Build Coastguard Worker vo0p0 = vfma_lane_f16(vo0p0, vi4x6789, vwOP, 1);
480*4bdc9457SAndroid Build Coastguard Worker vo1p0 = vfma_lane_f16(vo1p0, vi5x6789, vwOP, 1);
481*4bdc9457SAndroid Build Coastguard Worker
482*4bdc9457SAndroid Build Coastguard Worker
483*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo0 = vmax_f16(vo0p0, vmin);
484*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo1 = vmax_f16(vo1p0, vmin);
485*4bdc9457SAndroid Build Coastguard Worker
486*4bdc9457SAndroid Build Coastguard Worker vo0 = vmin_f16(vo0, vmax);
487*4bdc9457SAndroid Build Coastguard Worker vo1 = vmin_f16(vo1, vmax);
488*4bdc9457SAndroid Build Coastguard Worker
489*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(w & (4 * sizeof(__fp16))) {
490*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o1, vo1); o1 += 4;
491*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o0, vo0); o0 += 4;
492*4bdc9457SAndroid Build Coastguard Worker } else {
493*4bdc9457SAndroid Build Coastguard Worker if (w & (2 * sizeof(__fp16))) {
494*4bdc9457SAndroid Build Coastguard Worker vst1_lane_u32((void*) o1, vreinterpret_u32_f16(vo1), 0); o1 += 2;
495*4bdc9457SAndroid Build Coastguard Worker vst1_lane_u32((void*) o0, vreinterpret_u32_f16(vo0), 0); o0 += 2;
496*4bdc9457SAndroid Build Coastguard Worker
497*4bdc9457SAndroid Build Coastguard Worker vo0 = vext_f16(vo0, vo0, 2);
498*4bdc9457SAndroid Build Coastguard Worker vo1 = vext_f16(vo1, vo1, 2);
499*4bdc9457SAndroid Build Coastguard Worker }
500*4bdc9457SAndroid Build Coastguard Worker if (w & (1 * sizeof(__fp16))) {
501*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f16(o1, vo1, 0); o1 += 1;
502*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f16(o0, vo0, 0); o0 += 1;
503*4bdc9457SAndroid Build Coastguard Worker }
504*4bdc9457SAndroid Build Coastguard Worker }
505*4bdc9457SAndroid Build Coastguard Worker }
506*4bdc9457SAndroid Build Coastguard Worker
507*4bdc9457SAndroid Build Coastguard Worker i0 = (const __fp16*) ((uintptr_t) i2 - input_decrement);
508*4bdc9457SAndroid Build Coastguard Worker i1 = (const __fp16*) ((uintptr_t) i3 - input_decrement);
509*4bdc9457SAndroid Build Coastguard Worker i2 = (const __fp16*) ((uintptr_t) i1 + input_width);
510*4bdc9457SAndroid Build Coastguard Worker i3 = (const __fp16*) ((uintptr_t) i2 + input_width);
511*4bdc9457SAndroid Build Coastguard Worker i4 = (const __fp16*) ((uintptr_t) i3 + input_width);
512*4bdc9457SAndroid Build Coastguard Worker i5 = (const __fp16*) ((uintptr_t) i4 + input_width);
513*4bdc9457SAndroid Build Coastguard Worker
514*4bdc9457SAndroid Build Coastguard Worker o0 = o1;
515*4bdc9457SAndroid Build Coastguard Worker o1 = (__fp16*) ((uintptr_t) o0 + input_width);
516*4bdc9457SAndroid Build Coastguard Worker
517*4bdc9457SAndroid Build Coastguard Worker output_height = doz(output_height, 2);
518*4bdc9457SAndroid Build Coastguard Worker } while (output_height != 0);
519*4bdc9457SAndroid Build Coastguard Worker }
520