xref: /aosp_15_r20/external/XNNPACK/src/f16-dwconv/gen/up16x25-minmax-neonfp16arith.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-dwconv/up-neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/dwconv.h>
15 
16 
xnn_f16_dwconv_minmax_ukernel_up16x25__neonfp16arith(size_t channels,size_t output_width,const void ** input,const void * weights,void * output_ptr,size_t input_stride,size_t output_increment,size_t input_offset,const void * zero,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_dwconv_minmax_ukernel_up16x25__neonfp16arith(
18     size_t channels,
19     size_t output_width,
20     const void** input,
21     const void* weights,
22     void* output_ptr,
23     size_t input_stride,
24     size_t output_increment,
25     size_t input_offset,
26     const void* zero,
27     const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
28 {
29   assert(channels != 0);
30   assert(output_width != 0);
31 
32   __fp16* output = (__fp16*) output_ptr;
33   const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.max));
34   const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.min));
35   do {
36     const __fp16* i0 = (const __fp16*) input[0];
37     assert(i0 != NULL);
38     if XNN_UNPREDICTABLE(i0 != (const __fp16*) zero) {
39       i0 = (const __fp16*) ((uintptr_t) i0 + input_offset);
40     }
41     const __fp16* i1 = (const __fp16*) input[1];
42     assert(i1 != NULL);
43     if XNN_UNPREDICTABLE(i1 != (const __fp16*) zero) {
44       i1 = (const __fp16*) ((uintptr_t) i1 + input_offset);
45     }
46     const __fp16* i2 = (const __fp16*) input[2];
47     assert(i2 != NULL);
48     if XNN_UNPREDICTABLE(i2 != (const __fp16*) zero) {
49       i2 = (const __fp16*) ((uintptr_t) i2 + input_offset);
50     }
51     const __fp16* i3 = (const __fp16*) input[3];
52     assert(i3 != NULL);
53     if XNN_UNPREDICTABLE(i3 != (const __fp16*) zero) {
54       i3 = (const __fp16*) ((uintptr_t) i3 + input_offset);
55     }
56     const __fp16* i4 = (const __fp16*) input[4];
57     assert(i4 != NULL);
58     if XNN_UNPREDICTABLE(i4 != (const __fp16*) zero) {
59       i4 = (const __fp16*) ((uintptr_t) i4 + input_offset);
60     }
61     const __fp16* i5 = (const __fp16*) input[5];
62     assert(i5 != NULL);
63     if XNN_UNPREDICTABLE(i5 != (const __fp16*) zero) {
64       i5 = (const __fp16*) ((uintptr_t) i5 + input_offset);
65     }
66     const __fp16* i6 = (const __fp16*) input[6];
67     assert(i6 != NULL);
68     if XNN_UNPREDICTABLE(i6 != (const __fp16*) zero) {
69       i6 = (const __fp16*) ((uintptr_t) i6 + input_offset);
70     }
71     const __fp16* i7 = (const __fp16*) input[7];
72     assert(i7 != NULL);
73     if XNN_UNPREDICTABLE(i7 != (const __fp16*) zero) {
74       i7 = (const __fp16*) ((uintptr_t) i7 + input_offset);
75     }
76     const __fp16* i8 = (const __fp16*) input[8];
77     assert(i8 != NULL);
78     if XNN_UNPREDICTABLE(i8 != (const __fp16*) zero) {
79       i8 = (const __fp16*) ((uintptr_t) i8 + input_offset);
80     }
81     const __fp16* i9 = (const __fp16*) input[9];
82     assert(i9 != NULL);
83     if XNN_UNPREDICTABLE(i9 != (const __fp16*) zero) {
84       i9 = (const __fp16*) ((uintptr_t) i9 + input_offset);
85     }
86     const __fp16* i10 = (const __fp16*) input[10];
87     assert(i10 != NULL);
88     if XNN_UNPREDICTABLE(i10 != (const __fp16*) zero) {
89       i10 = (const __fp16*) ((uintptr_t) i10 + input_offset);
90     }
91     const __fp16* i11 = (const __fp16*) input[11];
92     assert(i11 != NULL);
93     if XNN_UNPREDICTABLE(i11 != (const __fp16*) zero) {
94       i11 = (const __fp16*) ((uintptr_t) i11 + input_offset);
95     }
96     const __fp16* i12 = (const __fp16*) input[12];
97     assert(i12 != NULL);
98     if XNN_UNPREDICTABLE(i12 != (const __fp16*) zero) {
99       i12 = (const __fp16*) ((uintptr_t) i12 + input_offset);
100     }
101     const __fp16* i13 = (const __fp16*) input[13];
102     assert(i13 != NULL);
103     if XNN_UNPREDICTABLE(i13 != (const __fp16*) zero) {
104       i13 = (const __fp16*) ((uintptr_t) i13 + input_offset);
105     }
106     const __fp16* i14 = (const __fp16*) input[14];
107     assert(i14 != NULL);
108     if XNN_UNPREDICTABLE(i14 != (const __fp16*) zero) {
109       i14 = (const __fp16*) ((uintptr_t) i14 + input_offset);
110     }
111     const __fp16* i15 = (const __fp16*) input[15];
112     assert(i15 != NULL);
113     if XNN_UNPREDICTABLE(i15 != (const __fp16*) zero) {
114       i15 = (const __fp16*) ((uintptr_t) i15 + input_offset);
115     }
116     const __fp16* i16 = (const __fp16*) input[16];
117     assert(i16 != NULL);
118     if XNN_UNPREDICTABLE(i16 != (const __fp16*) zero) {
119       i16 = (const __fp16*) ((uintptr_t) i16 + input_offset);
120     }
121     const __fp16* i17 = (const __fp16*) input[17];
122     assert(i17 != NULL);
123     if XNN_UNPREDICTABLE(i17 != (const __fp16*) zero) {
124       i17 = (const __fp16*) ((uintptr_t) i17 + input_offset);
125     }
126     const __fp16* i18 = (const __fp16*) input[18];
127     assert(i18 != NULL);
128     if XNN_UNPREDICTABLE(i18 != (const __fp16*) zero) {
129       i18 = (const __fp16*) ((uintptr_t) i18 + input_offset);
130     }
131     const __fp16* i19 = (const __fp16*) input[19];
132     assert(i19 != NULL);
133     if XNN_UNPREDICTABLE(i19 != (const __fp16*) zero) {
134       i19 = (const __fp16*) ((uintptr_t) i19 + input_offset);
135     }
136     const __fp16* i20 = (const __fp16*) input[20];
137     assert(i20 != NULL);
138     if XNN_UNPREDICTABLE(i20 != (const __fp16*) zero) {
139       i20 = (const __fp16*) ((uintptr_t) i20 + input_offset);
140     }
141     const __fp16* i21 = (const __fp16*) input[21];
142     assert(i21 != NULL);
143     if XNN_UNPREDICTABLE(i21 != (const __fp16*) zero) {
144       i21 = (const __fp16*) ((uintptr_t) i21 + input_offset);
145     }
146     const __fp16* i22 = (const __fp16*) input[22];
147     assert(i22 != NULL);
148     if XNN_UNPREDICTABLE(i22 != (const __fp16*) zero) {
149       i22 = (const __fp16*) ((uintptr_t) i22 + input_offset);
150     }
151     const __fp16* i23 = (const __fp16*) input[23];
152     assert(i23 != NULL);
153     if XNN_UNPREDICTABLE(i23 != (const __fp16*) zero) {
154       i23 = (const __fp16*) ((uintptr_t) i23 + input_offset);
155     }
156     const __fp16* i24 = (const __fp16*) input[24];
157     assert(i24 != NULL);
158     if XNN_UNPREDICTABLE(i24 != (const __fp16*) zero) {
159       i24 = (const __fp16*) ((uintptr_t) i24 + input_offset);
160     }
161 
162     input = (const void**) ((uintptr_t) input + input_stride);
163 
164     size_t c = channels;
165     const __fp16* w = (const __fp16*) weights;
166     for (; c >= 16; c -= 16) {
167       float16x8_t vacc01234567p0 = vld1q_f16(w); w += 8;
168       float16x8_t vacc89ABCDEFp0 = vld1q_f16(w); w += 8;
169 
170 
171       const float16x8_t vi0x01234567 = vld1q_f16(i0); i0 += 8;
172       const float16x8_t vi0x89ABCDEF = vld1q_f16(i0); i0 += 8;
173       const float16x8_t vk0x01234567 = vld1q_f16(w); w += 8;
174       const float16x8_t vk0x89ABCDEF = vld1q_f16(w); w += 8;
175       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
176       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi0x89ABCDEF, vk0x89ABCDEF);
177 
178       const float16x8_t vi1x01234567 = vld1q_f16(i1); i1 += 8;
179       const float16x8_t vi1x89ABCDEF = vld1q_f16(i1); i1 += 8;
180       const float16x8_t vk1x01234567 = vld1q_f16(w); w += 8;
181       const float16x8_t vk1x89ABCDEF = vld1q_f16(w); w += 8;
182       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
183       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi1x89ABCDEF, vk1x89ABCDEF);
184 
185       const float16x8_t vi2x01234567 = vld1q_f16(i2); i2 += 8;
186       const float16x8_t vi2x89ABCDEF = vld1q_f16(i2); i2 += 8;
187       const float16x8_t vk2x01234567 = vld1q_f16(w); w += 8;
188       const float16x8_t vk2x89ABCDEF = vld1q_f16(w); w += 8;
189       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
190       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi2x89ABCDEF, vk2x89ABCDEF);
191 
192       const float16x8_t vi3x01234567 = vld1q_f16(i3); i3 += 8;
193       const float16x8_t vi3x89ABCDEF = vld1q_f16(i3); i3 += 8;
194       const float16x8_t vk3x01234567 = vld1q_f16(w); w += 8;
195       const float16x8_t vk3x89ABCDEF = vld1q_f16(w); w += 8;
196       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
197       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi3x89ABCDEF, vk3x89ABCDEF);
198 
199       const float16x8_t vi4x01234567 = vld1q_f16(i4); i4 += 8;
200       const float16x8_t vi4x89ABCDEF = vld1q_f16(i4); i4 += 8;
201       const float16x8_t vk4x01234567 = vld1q_f16(w); w += 8;
202       const float16x8_t vk4x89ABCDEF = vld1q_f16(w); w += 8;
203       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
204       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi4x89ABCDEF, vk4x89ABCDEF);
205 
206       const float16x8_t vi5x01234567 = vld1q_f16(i5); i5 += 8;
207       const float16x8_t vi5x89ABCDEF = vld1q_f16(i5); i5 += 8;
208       const float16x8_t vk5x01234567 = vld1q_f16(w); w += 8;
209       const float16x8_t vk5x89ABCDEF = vld1q_f16(w); w += 8;
210       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
211       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi5x89ABCDEF, vk5x89ABCDEF);
212 
213       const float16x8_t vi6x01234567 = vld1q_f16(i6); i6 += 8;
214       const float16x8_t vi6x89ABCDEF = vld1q_f16(i6); i6 += 8;
215       const float16x8_t vk6x01234567 = vld1q_f16(w); w += 8;
216       const float16x8_t vk6x89ABCDEF = vld1q_f16(w); w += 8;
217       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
218       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi6x89ABCDEF, vk6x89ABCDEF);
219 
220       const float16x8_t vi7x01234567 = vld1q_f16(i7); i7 += 8;
221       const float16x8_t vi7x89ABCDEF = vld1q_f16(i7); i7 += 8;
222       const float16x8_t vk7x01234567 = vld1q_f16(w); w += 8;
223       const float16x8_t vk7x89ABCDEF = vld1q_f16(w); w += 8;
224       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
225       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi7x89ABCDEF, vk7x89ABCDEF);
226 
227       const float16x8_t vi8x01234567 = vld1q_f16(i8); i8 += 8;
228       const float16x8_t vi8x89ABCDEF = vld1q_f16(i8); i8 += 8;
229       const float16x8_t vk8x01234567 = vld1q_f16(w); w += 8;
230       const float16x8_t vk8x89ABCDEF = vld1q_f16(w); w += 8;
231       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
232       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi8x89ABCDEF, vk8x89ABCDEF);
233 
234       const float16x8_t vi9x01234567 = vld1q_f16(i9); i9 += 8;
235       const float16x8_t vi9x89ABCDEF = vld1q_f16(i9); i9 += 8;
236       const float16x8_t vk9x01234567 = vld1q_f16(w); w += 8;
237       const float16x8_t vk9x89ABCDEF = vld1q_f16(w); w += 8;
238       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
239       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi9x89ABCDEF, vk9x89ABCDEF);
240 
241       const float16x8_t vi10x01234567 = vld1q_f16(i10); i10 += 8;
242       const float16x8_t vi10x89ABCDEF = vld1q_f16(i10); i10 += 8;
243       const float16x8_t vk10x01234567 = vld1q_f16(w); w += 8;
244       const float16x8_t vk10x89ABCDEF = vld1q_f16(w); w += 8;
245       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
246       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi10x89ABCDEF, vk10x89ABCDEF);
247 
248       const float16x8_t vi11x01234567 = vld1q_f16(i11); i11 += 8;
249       const float16x8_t vi11x89ABCDEF = vld1q_f16(i11); i11 += 8;
250       const float16x8_t vk11x01234567 = vld1q_f16(w); w += 8;
251       const float16x8_t vk11x89ABCDEF = vld1q_f16(w); w += 8;
252       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
253       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi11x89ABCDEF, vk11x89ABCDEF);
254 
255       const float16x8_t vi12x01234567 = vld1q_f16(i12); i12 += 8;
256       const float16x8_t vi12x89ABCDEF = vld1q_f16(i12); i12 += 8;
257       const float16x8_t vk12x01234567 = vld1q_f16(w); w += 8;
258       const float16x8_t vk12x89ABCDEF = vld1q_f16(w); w += 8;
259       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
260       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi12x89ABCDEF, vk12x89ABCDEF);
261 
262       const float16x8_t vi13x01234567 = vld1q_f16(i13); i13 += 8;
263       const float16x8_t vi13x89ABCDEF = vld1q_f16(i13); i13 += 8;
264       const float16x8_t vk13x01234567 = vld1q_f16(w); w += 8;
265       const float16x8_t vk13x89ABCDEF = vld1q_f16(w); w += 8;
266       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
267       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi13x89ABCDEF, vk13x89ABCDEF);
268 
269       const float16x8_t vi14x01234567 = vld1q_f16(i14); i14 += 8;
270       const float16x8_t vi14x89ABCDEF = vld1q_f16(i14); i14 += 8;
271       const float16x8_t vk14x01234567 = vld1q_f16(w); w += 8;
272       const float16x8_t vk14x89ABCDEF = vld1q_f16(w); w += 8;
273       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
274       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi14x89ABCDEF, vk14x89ABCDEF);
275 
276       const float16x8_t vi15x01234567 = vld1q_f16(i15); i15 += 8;
277       const float16x8_t vi15x89ABCDEF = vld1q_f16(i15); i15 += 8;
278       const float16x8_t vk15x01234567 = vld1q_f16(w); w += 8;
279       const float16x8_t vk15x89ABCDEF = vld1q_f16(w); w += 8;
280       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
281       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi15x89ABCDEF, vk15x89ABCDEF);
282 
283       const float16x8_t vi16x01234567 = vld1q_f16(i16); i16 += 8;
284       const float16x8_t vi16x89ABCDEF = vld1q_f16(i16); i16 += 8;
285       const float16x8_t vk16x01234567 = vld1q_f16(w); w += 8;
286       const float16x8_t vk16x89ABCDEF = vld1q_f16(w); w += 8;
287       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
288       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi16x89ABCDEF, vk16x89ABCDEF);
289 
290       const float16x8_t vi17x01234567 = vld1q_f16(i17); i17 += 8;
291       const float16x8_t vi17x89ABCDEF = vld1q_f16(i17); i17 += 8;
292       const float16x8_t vk17x01234567 = vld1q_f16(w); w += 8;
293       const float16x8_t vk17x89ABCDEF = vld1q_f16(w); w += 8;
294       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
295       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi17x89ABCDEF, vk17x89ABCDEF);
296 
297       const float16x8_t vi18x01234567 = vld1q_f16(i18); i18 += 8;
298       const float16x8_t vi18x89ABCDEF = vld1q_f16(i18); i18 += 8;
299       const float16x8_t vk18x01234567 = vld1q_f16(w); w += 8;
300       const float16x8_t vk18x89ABCDEF = vld1q_f16(w); w += 8;
301       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
302       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi18x89ABCDEF, vk18x89ABCDEF);
303 
304       const float16x8_t vi19x01234567 = vld1q_f16(i19); i19 += 8;
305       const float16x8_t vi19x89ABCDEF = vld1q_f16(i19); i19 += 8;
306       const float16x8_t vk19x01234567 = vld1q_f16(w); w += 8;
307       const float16x8_t vk19x89ABCDEF = vld1q_f16(w); w += 8;
308       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
309       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi19x89ABCDEF, vk19x89ABCDEF);
310 
311       const float16x8_t vi20x01234567 = vld1q_f16(i20); i20 += 8;
312       const float16x8_t vi20x89ABCDEF = vld1q_f16(i20); i20 += 8;
313       const float16x8_t vk20x01234567 = vld1q_f16(w); w += 8;
314       const float16x8_t vk20x89ABCDEF = vld1q_f16(w); w += 8;
315       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
316       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi20x89ABCDEF, vk20x89ABCDEF);
317 
318       const float16x8_t vi21x01234567 = vld1q_f16(i21); i21 += 8;
319       const float16x8_t vi21x89ABCDEF = vld1q_f16(i21); i21 += 8;
320       const float16x8_t vk21x01234567 = vld1q_f16(w); w += 8;
321       const float16x8_t vk21x89ABCDEF = vld1q_f16(w); w += 8;
322       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
323       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi21x89ABCDEF, vk21x89ABCDEF);
324 
325       const float16x8_t vi22x01234567 = vld1q_f16(i22); i22 += 8;
326       const float16x8_t vi22x89ABCDEF = vld1q_f16(i22); i22 += 8;
327       const float16x8_t vk22x01234567 = vld1q_f16(w); w += 8;
328       const float16x8_t vk22x89ABCDEF = vld1q_f16(w); w += 8;
329       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
330       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi22x89ABCDEF, vk22x89ABCDEF);
331 
332       const float16x8_t vi23x01234567 = vld1q_f16(i23); i23 += 8;
333       const float16x8_t vi23x89ABCDEF = vld1q_f16(i23); i23 += 8;
334       const float16x8_t vk23x01234567 = vld1q_f16(w); w += 8;
335       const float16x8_t vk23x89ABCDEF = vld1q_f16(w); w += 8;
336       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
337       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi23x89ABCDEF, vk23x89ABCDEF);
338 
339       const float16x8_t vi24x01234567 = vld1q_f16(i24); i24 += 8;
340       const float16x8_t vi24x89ABCDEF = vld1q_f16(i24); i24 += 8;
341       const float16x8_t vk24x01234567 = vld1q_f16(w); w += 8;
342       const float16x8_t vk24x89ABCDEF = vld1q_f16(w); w += 8;
343       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
344       vacc89ABCDEFp0 = vfmaq_f16(vacc89ABCDEFp0, vi24x89ABCDEF, vk24x89ABCDEF);
345 
346 
347       float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
348       float16x8_t vacc89ABCDEF = vmaxq_f16(vacc89ABCDEFp0, vmin);
349       vacc01234567 = vminq_f16(vacc01234567, vmax);
350       vacc89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
351 
352       vst1q_f16(output, vacc01234567); output += 8;
353       vst1q_f16(output, vacc89ABCDEF); output += 8;
354     }
355     for (; c >= 8; c -= 8) {
356       float16x8_t vacc01234567p0 = vld1q_f16(w); w += 8;
357 
358 
359       const float16x8_t vi0x01234567 = vld1q_f16(i0); i0 += 8;
360       const float16x8_t vk0x01234567 = vld1q_f16(w + 8);
361       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
362 
363       const float16x8_t vi1x01234567 = vld1q_f16(i1); i1 += 8;
364       const float16x8_t vk1x01234567 = vld1q_f16(w + 24);
365       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
366 
367       const float16x8_t vi2x01234567 = vld1q_f16(i2); i2 += 8;
368       const float16x8_t vk2x01234567 = vld1q_f16(w + 40);
369       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
370 
371       const float16x8_t vi3x01234567 = vld1q_f16(i3); i3 += 8;
372       const float16x8_t vk3x01234567 = vld1q_f16(w + 56);
373       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
374 
375       const float16x8_t vi4x01234567 = vld1q_f16(i4); i4 += 8;
376       const float16x8_t vk4x01234567 = vld1q_f16(w + 72);
377       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
378 
379       const float16x8_t vi5x01234567 = vld1q_f16(i5); i5 += 8;
380       const float16x8_t vk5x01234567 = vld1q_f16(w + 88);
381       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
382 
383       const float16x8_t vi6x01234567 = vld1q_f16(i6); i6 += 8;
384       const float16x8_t vk6x01234567 = vld1q_f16(w + 104);
385       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
386 
387       const float16x8_t vi7x01234567 = vld1q_f16(i7); i7 += 8;
388       const float16x8_t vk7x01234567 = vld1q_f16(w + 120);
389       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
390 
391       const float16x8_t vi8x01234567 = vld1q_f16(i8); i8 += 8;
392       const float16x8_t vk8x01234567 = vld1q_f16(w + 136);
393       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
394 
395       const float16x8_t vi9x01234567 = vld1q_f16(i9); i9 += 8;
396       const float16x8_t vk9x01234567 = vld1q_f16(w + 152);
397       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
398 
399       const float16x8_t vi10x01234567 = vld1q_f16(i10); i10 += 8;
400       const float16x8_t vk10x01234567 = vld1q_f16(w + 168);
401       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
402 
403       const float16x8_t vi11x01234567 = vld1q_f16(i11); i11 += 8;
404       const float16x8_t vk11x01234567 = vld1q_f16(w + 184);
405       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
406 
407       const float16x8_t vi12x01234567 = vld1q_f16(i12); i12 += 8;
408       const float16x8_t vk12x01234567 = vld1q_f16(w + 200);
409       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
410 
411       const float16x8_t vi13x01234567 = vld1q_f16(i13); i13 += 8;
412       const float16x8_t vk13x01234567 = vld1q_f16(w + 216);
413       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
414 
415       const float16x8_t vi14x01234567 = vld1q_f16(i14); i14 += 8;
416       const float16x8_t vk14x01234567 = vld1q_f16(w + 232);
417       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
418 
419       const float16x8_t vi15x01234567 = vld1q_f16(i15); i15 += 8;
420       const float16x8_t vk15x01234567 = vld1q_f16(w + 248);
421       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
422 
423       const float16x8_t vi16x01234567 = vld1q_f16(i16); i16 += 8;
424       const float16x8_t vk16x01234567 = vld1q_f16(w + 264);
425       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
426 
427       const float16x8_t vi17x01234567 = vld1q_f16(i17); i17 += 8;
428       const float16x8_t vk17x01234567 = vld1q_f16(w + 280);
429       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
430 
431       const float16x8_t vi18x01234567 = vld1q_f16(i18); i18 += 8;
432       const float16x8_t vk18x01234567 = vld1q_f16(w + 296);
433       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
434 
435       const float16x8_t vi19x01234567 = vld1q_f16(i19); i19 += 8;
436       const float16x8_t vk19x01234567 = vld1q_f16(w + 312);
437       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
438 
439       const float16x8_t vi20x01234567 = vld1q_f16(i20); i20 += 8;
440       const float16x8_t vk20x01234567 = vld1q_f16(w + 328);
441       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
442 
443       const float16x8_t vi21x01234567 = vld1q_f16(i21); i21 += 8;
444       const float16x8_t vk21x01234567 = vld1q_f16(w + 344);
445       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
446 
447       const float16x8_t vi22x01234567 = vld1q_f16(i22); i22 += 8;
448       const float16x8_t vk22x01234567 = vld1q_f16(w + 360);
449       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
450 
451       const float16x8_t vi23x01234567 = vld1q_f16(i23); i23 += 8;
452       const float16x8_t vk23x01234567 = vld1q_f16(w + 376);
453       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
454 
455       const float16x8_t vi24x01234567 = vld1q_f16(i24); i24 += 8;
456       const float16x8_t vk24x01234567 = vld1q_f16(w + 392);
457       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
458 
459 
460       float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
461       vacc01234567 = vminq_f16(vacc01234567, vmax);
462 
463       vst1q_f16(output, vacc01234567); output += 8;
464     }
465     if XNN_UNLIKELY(c != 0) {
466       float16x8_t vacc01234567p0 = vld1q_f16(w);
467 
468 
469       const float16x8_t vi0x01234567 = vld1q_f16(i0);
470       const float16x8_t vk0x01234567 = vld1q_f16(w + 16);
471       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi0x01234567, vk0x01234567);
472 
473       const float16x8_t vi1x01234567 = vld1q_f16(i1);
474       const float16x8_t vk1x01234567 = vld1q_f16(w + 32);
475       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi1x01234567, vk1x01234567);
476 
477       const float16x8_t vi2x01234567 = vld1q_f16(i2);
478       const float16x8_t vk2x01234567 = vld1q_f16(w + 48);
479       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi2x01234567, vk2x01234567);
480 
481       const float16x8_t vi3x01234567 = vld1q_f16(i3);
482       const float16x8_t vk3x01234567 = vld1q_f16(w + 64);
483       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi3x01234567, vk3x01234567);
484 
485       const float16x8_t vi4x01234567 = vld1q_f16(i4);
486       const float16x8_t vk4x01234567 = vld1q_f16(w + 80);
487       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi4x01234567, vk4x01234567);
488 
489       const float16x8_t vi5x01234567 = vld1q_f16(i5);
490       const float16x8_t vk5x01234567 = vld1q_f16(w + 96);
491       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi5x01234567, vk5x01234567);
492 
493       const float16x8_t vi6x01234567 = vld1q_f16(i6);
494       const float16x8_t vk6x01234567 = vld1q_f16(w + 112);
495       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi6x01234567, vk6x01234567);
496 
497       const float16x8_t vi7x01234567 = vld1q_f16(i7);
498       const float16x8_t vk7x01234567 = vld1q_f16(w + 128);
499       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi7x01234567, vk7x01234567);
500 
501       const float16x8_t vi8x01234567 = vld1q_f16(i8);
502       const float16x8_t vk8x01234567 = vld1q_f16(w + 144);
503       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi8x01234567, vk8x01234567);
504 
505       const float16x8_t vi9x01234567 = vld1q_f16(i9);
506       const float16x8_t vk9x01234567 = vld1q_f16(w + 160);
507       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi9x01234567, vk9x01234567);
508 
509       const float16x8_t vi10x01234567 = vld1q_f16(i10);
510       const float16x8_t vk10x01234567 = vld1q_f16(w + 176);
511       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi10x01234567, vk10x01234567);
512 
513       const float16x8_t vi11x01234567 = vld1q_f16(i11);
514       const float16x8_t vk11x01234567 = vld1q_f16(w + 192);
515       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi11x01234567, vk11x01234567);
516 
517       const float16x8_t vi12x01234567 = vld1q_f16(i12);
518       const float16x8_t vk12x01234567 = vld1q_f16(w + 208);
519       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi12x01234567, vk12x01234567);
520 
521       const float16x8_t vi13x01234567 = vld1q_f16(i13);
522       const float16x8_t vk13x01234567 = vld1q_f16(w + 224);
523       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi13x01234567, vk13x01234567);
524 
525       const float16x8_t vi14x01234567 = vld1q_f16(i14);
526       const float16x8_t vk14x01234567 = vld1q_f16(w + 240);
527       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi14x01234567, vk14x01234567);
528 
529       const float16x8_t vi15x01234567 = vld1q_f16(i15);
530       const float16x8_t vk15x01234567 = vld1q_f16(w + 256);
531       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi15x01234567, vk15x01234567);
532 
533       const float16x8_t vi16x01234567 = vld1q_f16(i16);
534       const float16x8_t vk16x01234567 = vld1q_f16(w + 272);
535       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi16x01234567, vk16x01234567);
536 
537       const float16x8_t vi17x01234567 = vld1q_f16(i17);
538       const float16x8_t vk17x01234567 = vld1q_f16(w + 288);
539       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi17x01234567, vk17x01234567);
540 
541       const float16x8_t vi18x01234567 = vld1q_f16(i18);
542       const float16x8_t vk18x01234567 = vld1q_f16(w + 304);
543       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi18x01234567, vk18x01234567);
544 
545       const float16x8_t vi19x01234567 = vld1q_f16(i19);
546       const float16x8_t vk19x01234567 = vld1q_f16(w + 320);
547       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi19x01234567, vk19x01234567);
548 
549       const float16x8_t vi20x01234567 = vld1q_f16(i20);
550       const float16x8_t vk20x01234567 = vld1q_f16(w + 336);
551       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi20x01234567, vk20x01234567);
552 
553       const float16x8_t vi21x01234567 = vld1q_f16(i21);
554       const float16x8_t vk21x01234567 = vld1q_f16(w + 352);
555       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi21x01234567, vk21x01234567);
556 
557       const float16x8_t vi22x01234567 = vld1q_f16(i22);
558       const float16x8_t vk22x01234567 = vld1q_f16(w + 368);
559       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi22x01234567, vk22x01234567);
560 
561       const float16x8_t vi23x01234567 = vld1q_f16(i23);
562       const float16x8_t vk23x01234567 = vld1q_f16(w + 384);
563       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi23x01234567, vk23x01234567);
564 
565       const float16x8_t vi24x01234567 = vld1q_f16(i24);
566       const float16x8_t vk24x01234567 = vld1q_f16(w + 400);
567       vacc01234567p0 = vfmaq_f16(vacc01234567p0, vi24x01234567, vk24x01234567);
568 
569 
570       float16x8_t vacc01234567 = vmaxq_f16(vacc01234567p0, vmin);
571       vacc01234567 = vminq_f16(vacc01234567, vmax);
572 
573       float16x4_t vacc0123 = vget_low_f16(vacc01234567);
574       if (c & 4) {
575         vst1_f16(output, vacc0123); output += 4;
576         vacc0123 = vget_high_f16(vacc01234567);
577       }
578       if (c & 2) {
579         vst1_lane_u32((void*) output, vreinterpret_u32_f16(vacc0123), 0); output += 2;
580         vacc0123 = vext_f16(vacc0123, vacc0123, 2);
581       }
582       if (c & 1) {
583         vst1_lane_f16(output, vacc0123, 0); output += 1;
584       }
585     }
586 
587     output = (__fp16*) ((uintptr_t) output + output_increment);
588   } while (--output_width != 0);
589 }
590