xref: /aosp_15_r20/external/XNNPACK/src/f16-gemm/gen/8x16-minmax-neonfp16arith-ld64.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-gemm/neonfp16arith-ld64.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2020 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 
11 #include <assert.h>
12 
13 #include <arm_neon.h>
14 
15 #include <xnnpack/common.h>
16 
17 #include <xnnpack/gemm.h>
18 
19 
xnn_f16_gemm_minmax_ukernel_8x16__neonfp16arith_ld64(size_t mr,size_t nc,size_t kc,const void * restrict a,size_t a_stride,const void * restrict w,void * restrict c,size_t cm_stride,size_t cn_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])20 void xnn_f16_gemm_minmax_ukernel_8x16__neonfp16arith_ld64(
21     size_t mr,
22     size_t nc,
23     size_t kc,
24     const void* restrict a,
25     size_t a_stride,
26     const void* restrict w,
27     void* restrict c,
28     size_t cm_stride,
29     size_t cn_stride,
30     const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
31 {
32   assert(mr != 0);
33   assert(mr <= 8);
34   assert(nc != 0);
35   assert(kc != 0);
36   assert(kc % sizeof(__fp16) == 0);
37   assert(a != NULL);
38   assert(w != NULL);
39   assert(c != NULL);
40 
41   const __fp16* a0 = (const __fp16*) a;
42   __fp16* c0 = (__fp16*) c;
43   const __fp16* a1 = (const __fp16*) ((uintptr_t) a0 + a_stride);
44   __fp16* c1 = (__fp16*) ((uintptr_t) c0 + cm_stride);
45   if XNN_UNPREDICTABLE(mr < 2) {
46     a1 = a0;
47     c1 = c0;
48   }
49   const __fp16* a2 = (const __fp16*) ((uintptr_t) a1 + a_stride);
50   __fp16* c2 = (__fp16*) ((uintptr_t) c1 + cm_stride);
51   if XNN_UNPREDICTABLE(mr <= 2) {
52     a2 = a1;
53     c2 = c1;
54   }
55   const __fp16* a3 = (const __fp16*) ((uintptr_t) a2 + a_stride);
56   __fp16* c3 = (__fp16*) ((uintptr_t) c2 + cm_stride);
57   if XNN_UNPREDICTABLE(mr < 4) {
58     a3 = a2;
59     c3 = c2;
60   }
61   const __fp16* a4 = (const __fp16*) ((uintptr_t) a3 + a_stride);
62   __fp16* c4 = (__fp16*) ((uintptr_t) c3 + cm_stride);
63   if XNN_UNPREDICTABLE(mr <= 4) {
64     a4 = a3;
65     c4 = c3;
66   }
67   const __fp16* a5 = (const __fp16*) ((uintptr_t) a4 + a_stride);
68   __fp16* c5 = (__fp16*) ((uintptr_t) c4 + cm_stride);
69   if XNN_UNPREDICTABLE(mr < 6) {
70     a5 = a4;
71     c5 = c4;
72   }
73   const __fp16* a6 = (const __fp16*) ((uintptr_t) a5 + a_stride);
74   __fp16* c6 = (__fp16*) ((uintptr_t) c5 + cm_stride);
75   if XNN_UNPREDICTABLE(mr <= 6) {
76     a6 = a5;
77     c6 = c5;
78   }
79   const __fp16* a7 = (const __fp16*) ((uintptr_t) a6 + a_stride);
80   __fp16* c7 = (__fp16*) ((uintptr_t) c6 + cm_stride);
81   if XNN_UNPREDICTABLE(mr != 8) {
82     a7 = a6;
83     c7 = c6;
84   }
85 
86   do {
87     float16x8_t vacc0x01234567 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
88     float16x8_t vacc0x89ABCDEF = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
89     float16x8_t vacc1x01234567 = vacc0x01234567;
90     float16x8_t vacc1x89ABCDEF = vacc0x89ABCDEF;
91     float16x8_t vacc2x01234567 = vacc0x01234567;
92     float16x8_t vacc2x89ABCDEF = vacc0x89ABCDEF;
93     float16x8_t vacc3x01234567 = vacc0x01234567;
94     float16x8_t vacc3x89ABCDEF = vacc0x89ABCDEF;
95     float16x8_t vacc4x01234567 = vacc0x01234567;
96     float16x8_t vacc4x89ABCDEF = vacc0x89ABCDEF;
97     float16x8_t vacc5x01234567 = vacc0x01234567;
98     float16x8_t vacc5x89ABCDEF = vacc0x89ABCDEF;
99     float16x8_t vacc6x01234567 = vacc0x01234567;
100     float16x8_t vacc6x89ABCDEF = vacc0x89ABCDEF;
101     float16x8_t vacc7x01234567 = vacc0x01234567;
102     float16x8_t vacc7x89ABCDEF = vacc0x89ABCDEF;
103 
104     size_t k = kc;
105     while (k >= 4 * sizeof(__fp16)) {
106       const float16x4_t va0 = vld1_f16(a0); a0 += 4;
107       const float16x4_t va1 = vld1_f16(a1); a1 += 4;
108       const float16x4_t va2 = vld1_f16(a2); a2 += 4;
109       const float16x4_t va3 = vld1_f16(a3); a3 += 4;
110       const float16x4_t va4 = vld1_f16(a4); a4 += 4;
111       const float16x4_t va5 = vld1_f16(a5); a5 += 4;
112       const float16x4_t va6 = vld1_f16(a6); a6 += 4;
113       const float16x4_t va7 = vld1_f16(a7); a7 += 4;
114 
115       const float16x8_t vb01234567c0 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
116       const float16x8_t vb89ABCDEFc0 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
117 
118       #if XNN_ARCH_ARM64
119         vacc0x01234567 = vfmaq_lane_f16(vacc0x01234567, vb01234567c0, va0, 0);
120         vacc1x01234567 = vfmaq_lane_f16(vacc1x01234567, vb01234567c0, va1, 0);
121         vacc2x01234567 = vfmaq_lane_f16(vacc2x01234567, vb01234567c0, va2, 0);
122         vacc3x01234567 = vfmaq_lane_f16(vacc3x01234567, vb01234567c0, va3, 0);
123         vacc4x01234567 = vfmaq_lane_f16(vacc4x01234567, vb01234567c0, va4, 0);
124         vacc5x01234567 = vfmaq_lane_f16(vacc5x01234567, vb01234567c0, va5, 0);
125         vacc6x01234567 = vfmaq_lane_f16(vacc6x01234567, vb01234567c0, va6, 0);
126         vacc7x01234567 = vfmaq_lane_f16(vacc7x01234567, vb01234567c0, va7, 0);
127         vacc0x89ABCDEF = vfmaq_lane_f16(vacc0x89ABCDEF, vb89ABCDEFc0, va0, 0);
128         vacc1x89ABCDEF = vfmaq_lane_f16(vacc1x89ABCDEF, vb89ABCDEFc0, va1, 0);
129         vacc2x89ABCDEF = vfmaq_lane_f16(vacc2x89ABCDEF, vb89ABCDEFc0, va2, 0);
130         vacc3x89ABCDEF = vfmaq_lane_f16(vacc3x89ABCDEF, vb89ABCDEFc0, va3, 0);
131         vacc4x89ABCDEF = vfmaq_lane_f16(vacc4x89ABCDEF, vb89ABCDEFc0, va4, 0);
132         vacc5x89ABCDEF = vfmaq_lane_f16(vacc5x89ABCDEF, vb89ABCDEFc0, va5, 0);
133         vacc6x89ABCDEF = vfmaq_lane_f16(vacc6x89ABCDEF, vb89ABCDEFc0, va6, 0);
134         vacc7x89ABCDEF = vfmaq_lane_f16(vacc7x89ABCDEF, vb89ABCDEFc0, va7, 0);
135       #else
136         const float16x8_t va0c0 = vdupq_lane_f16(va0, 0);
137         const float16x8_t va1c0 = vdupq_lane_f16(va1, 0);
138         const float16x8_t va2c0 = vdupq_lane_f16(va2, 0);
139         const float16x8_t va3c0 = vdupq_lane_f16(va3, 0);
140         const float16x8_t va4c0 = vdupq_lane_f16(va4, 0);
141         const float16x8_t va5c0 = vdupq_lane_f16(va5, 0);
142         const float16x8_t va6c0 = vdupq_lane_f16(va6, 0);
143         const float16x8_t va7c0 = vdupq_lane_f16(va7, 0);
144 
145         vacc0x01234567 = vfmaq_f16(vacc0x01234567, va0c0, vb01234567c0);
146         vacc1x01234567 = vfmaq_f16(vacc1x01234567, va1c0, vb01234567c0);
147         vacc2x01234567 = vfmaq_f16(vacc2x01234567, va2c0, vb01234567c0);
148         vacc3x01234567 = vfmaq_f16(vacc3x01234567, va3c0, vb01234567c0);
149         vacc4x01234567 = vfmaq_f16(vacc4x01234567, va4c0, vb01234567c0);
150         vacc5x01234567 = vfmaq_f16(vacc5x01234567, va5c0, vb01234567c0);
151         vacc6x01234567 = vfmaq_f16(vacc6x01234567, va6c0, vb01234567c0);
152         vacc7x01234567 = vfmaq_f16(vacc7x01234567, va7c0, vb01234567c0);
153         vacc0x89ABCDEF = vfmaq_f16(vacc0x89ABCDEF, va0c0, vb89ABCDEFc0);
154         vacc1x89ABCDEF = vfmaq_f16(vacc1x89ABCDEF, va1c0, vb89ABCDEFc0);
155         vacc2x89ABCDEF = vfmaq_f16(vacc2x89ABCDEF, va2c0, vb89ABCDEFc0);
156         vacc3x89ABCDEF = vfmaq_f16(vacc3x89ABCDEF, va3c0, vb89ABCDEFc0);
157         vacc4x89ABCDEF = vfmaq_f16(vacc4x89ABCDEF, va4c0, vb89ABCDEFc0);
158         vacc5x89ABCDEF = vfmaq_f16(vacc5x89ABCDEF, va5c0, vb89ABCDEFc0);
159         vacc6x89ABCDEF = vfmaq_f16(vacc6x89ABCDEF, va6c0, vb89ABCDEFc0);
160         vacc7x89ABCDEF = vfmaq_f16(vacc7x89ABCDEF, va7c0, vb89ABCDEFc0);
161       #endif
162       const float16x8_t vb01234567c1 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
163       const float16x8_t vb89ABCDEFc1 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
164 
165       #if XNN_ARCH_ARM64
166         vacc0x01234567 = vfmaq_lane_f16(vacc0x01234567, vb01234567c1, va0, 1);
167         vacc1x01234567 = vfmaq_lane_f16(vacc1x01234567, vb01234567c1, va1, 1);
168         vacc2x01234567 = vfmaq_lane_f16(vacc2x01234567, vb01234567c1, va2, 1);
169         vacc3x01234567 = vfmaq_lane_f16(vacc3x01234567, vb01234567c1, va3, 1);
170         vacc4x01234567 = vfmaq_lane_f16(vacc4x01234567, vb01234567c1, va4, 1);
171         vacc5x01234567 = vfmaq_lane_f16(vacc5x01234567, vb01234567c1, va5, 1);
172         vacc6x01234567 = vfmaq_lane_f16(vacc6x01234567, vb01234567c1, va6, 1);
173         vacc7x01234567 = vfmaq_lane_f16(vacc7x01234567, vb01234567c1, va7, 1);
174         vacc0x89ABCDEF = vfmaq_lane_f16(vacc0x89ABCDEF, vb89ABCDEFc1, va0, 1);
175         vacc1x89ABCDEF = vfmaq_lane_f16(vacc1x89ABCDEF, vb89ABCDEFc1, va1, 1);
176         vacc2x89ABCDEF = vfmaq_lane_f16(vacc2x89ABCDEF, vb89ABCDEFc1, va2, 1);
177         vacc3x89ABCDEF = vfmaq_lane_f16(vacc3x89ABCDEF, vb89ABCDEFc1, va3, 1);
178         vacc4x89ABCDEF = vfmaq_lane_f16(vacc4x89ABCDEF, vb89ABCDEFc1, va4, 1);
179         vacc5x89ABCDEF = vfmaq_lane_f16(vacc5x89ABCDEF, vb89ABCDEFc1, va5, 1);
180         vacc6x89ABCDEF = vfmaq_lane_f16(vacc6x89ABCDEF, vb89ABCDEFc1, va6, 1);
181         vacc7x89ABCDEF = vfmaq_lane_f16(vacc7x89ABCDEF, vb89ABCDEFc1, va7, 1);
182       #else
183         const float16x8_t va0c1 = vdupq_lane_f16(va0, 1);
184         const float16x8_t va1c1 = vdupq_lane_f16(va1, 1);
185         const float16x8_t va2c1 = vdupq_lane_f16(va2, 1);
186         const float16x8_t va3c1 = vdupq_lane_f16(va3, 1);
187         const float16x8_t va4c1 = vdupq_lane_f16(va4, 1);
188         const float16x8_t va5c1 = vdupq_lane_f16(va5, 1);
189         const float16x8_t va6c1 = vdupq_lane_f16(va6, 1);
190         const float16x8_t va7c1 = vdupq_lane_f16(va7, 1);
191 
192         vacc0x01234567 = vfmaq_f16(vacc0x01234567, va0c1, vb01234567c1);
193         vacc1x01234567 = vfmaq_f16(vacc1x01234567, va1c1, vb01234567c1);
194         vacc2x01234567 = vfmaq_f16(vacc2x01234567, va2c1, vb01234567c1);
195         vacc3x01234567 = vfmaq_f16(vacc3x01234567, va3c1, vb01234567c1);
196         vacc4x01234567 = vfmaq_f16(vacc4x01234567, va4c1, vb01234567c1);
197         vacc5x01234567 = vfmaq_f16(vacc5x01234567, va5c1, vb01234567c1);
198         vacc6x01234567 = vfmaq_f16(vacc6x01234567, va6c1, vb01234567c1);
199         vacc7x01234567 = vfmaq_f16(vacc7x01234567, va7c1, vb01234567c1);
200         vacc0x89ABCDEF = vfmaq_f16(vacc0x89ABCDEF, va0c1, vb89ABCDEFc1);
201         vacc1x89ABCDEF = vfmaq_f16(vacc1x89ABCDEF, va1c1, vb89ABCDEFc1);
202         vacc2x89ABCDEF = vfmaq_f16(vacc2x89ABCDEF, va2c1, vb89ABCDEFc1);
203         vacc3x89ABCDEF = vfmaq_f16(vacc3x89ABCDEF, va3c1, vb89ABCDEFc1);
204         vacc4x89ABCDEF = vfmaq_f16(vacc4x89ABCDEF, va4c1, vb89ABCDEFc1);
205         vacc5x89ABCDEF = vfmaq_f16(vacc5x89ABCDEF, va5c1, vb89ABCDEFc1);
206         vacc6x89ABCDEF = vfmaq_f16(vacc6x89ABCDEF, va6c1, vb89ABCDEFc1);
207         vacc7x89ABCDEF = vfmaq_f16(vacc7x89ABCDEF, va7c1, vb89ABCDEFc1);
208       #endif
209       const float16x8_t vb01234567c2 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
210       const float16x8_t vb89ABCDEFc2 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
211 
212       #if XNN_ARCH_ARM64
213         vacc0x01234567 = vfmaq_lane_f16(vacc0x01234567, vb01234567c2, va0, 2);
214         vacc1x01234567 = vfmaq_lane_f16(vacc1x01234567, vb01234567c2, va1, 2);
215         vacc2x01234567 = vfmaq_lane_f16(vacc2x01234567, vb01234567c2, va2, 2);
216         vacc3x01234567 = vfmaq_lane_f16(vacc3x01234567, vb01234567c2, va3, 2);
217         vacc4x01234567 = vfmaq_lane_f16(vacc4x01234567, vb01234567c2, va4, 2);
218         vacc5x01234567 = vfmaq_lane_f16(vacc5x01234567, vb01234567c2, va5, 2);
219         vacc6x01234567 = vfmaq_lane_f16(vacc6x01234567, vb01234567c2, va6, 2);
220         vacc7x01234567 = vfmaq_lane_f16(vacc7x01234567, vb01234567c2, va7, 2);
221         vacc0x89ABCDEF = vfmaq_lane_f16(vacc0x89ABCDEF, vb89ABCDEFc2, va0, 2);
222         vacc1x89ABCDEF = vfmaq_lane_f16(vacc1x89ABCDEF, vb89ABCDEFc2, va1, 2);
223         vacc2x89ABCDEF = vfmaq_lane_f16(vacc2x89ABCDEF, vb89ABCDEFc2, va2, 2);
224         vacc3x89ABCDEF = vfmaq_lane_f16(vacc3x89ABCDEF, vb89ABCDEFc2, va3, 2);
225         vacc4x89ABCDEF = vfmaq_lane_f16(vacc4x89ABCDEF, vb89ABCDEFc2, va4, 2);
226         vacc5x89ABCDEF = vfmaq_lane_f16(vacc5x89ABCDEF, vb89ABCDEFc2, va5, 2);
227         vacc6x89ABCDEF = vfmaq_lane_f16(vacc6x89ABCDEF, vb89ABCDEFc2, va6, 2);
228         vacc7x89ABCDEF = vfmaq_lane_f16(vacc7x89ABCDEF, vb89ABCDEFc2, va7, 2);
229       #else
230         const float16x8_t va0c2 = vdupq_lane_f16(va0, 2);
231         const float16x8_t va1c2 = vdupq_lane_f16(va1, 2);
232         const float16x8_t va2c2 = vdupq_lane_f16(va2, 2);
233         const float16x8_t va3c2 = vdupq_lane_f16(va3, 2);
234         const float16x8_t va4c2 = vdupq_lane_f16(va4, 2);
235         const float16x8_t va5c2 = vdupq_lane_f16(va5, 2);
236         const float16x8_t va6c2 = vdupq_lane_f16(va6, 2);
237         const float16x8_t va7c2 = vdupq_lane_f16(va7, 2);
238 
239         vacc0x01234567 = vfmaq_f16(vacc0x01234567, va0c2, vb01234567c2);
240         vacc1x01234567 = vfmaq_f16(vacc1x01234567, va1c2, vb01234567c2);
241         vacc2x01234567 = vfmaq_f16(vacc2x01234567, va2c2, vb01234567c2);
242         vacc3x01234567 = vfmaq_f16(vacc3x01234567, va3c2, vb01234567c2);
243         vacc4x01234567 = vfmaq_f16(vacc4x01234567, va4c2, vb01234567c2);
244         vacc5x01234567 = vfmaq_f16(vacc5x01234567, va5c2, vb01234567c2);
245         vacc6x01234567 = vfmaq_f16(vacc6x01234567, va6c2, vb01234567c2);
246         vacc7x01234567 = vfmaq_f16(vacc7x01234567, va7c2, vb01234567c2);
247         vacc0x89ABCDEF = vfmaq_f16(vacc0x89ABCDEF, va0c2, vb89ABCDEFc2);
248         vacc1x89ABCDEF = vfmaq_f16(vacc1x89ABCDEF, va1c2, vb89ABCDEFc2);
249         vacc2x89ABCDEF = vfmaq_f16(vacc2x89ABCDEF, va2c2, vb89ABCDEFc2);
250         vacc3x89ABCDEF = vfmaq_f16(vacc3x89ABCDEF, va3c2, vb89ABCDEFc2);
251         vacc4x89ABCDEF = vfmaq_f16(vacc4x89ABCDEF, va4c2, vb89ABCDEFc2);
252         vacc5x89ABCDEF = vfmaq_f16(vacc5x89ABCDEF, va5c2, vb89ABCDEFc2);
253         vacc6x89ABCDEF = vfmaq_f16(vacc6x89ABCDEF, va6c2, vb89ABCDEFc2);
254         vacc7x89ABCDEF = vfmaq_f16(vacc7x89ABCDEF, va7c2, vb89ABCDEFc2);
255       #endif
256       const float16x8_t vb01234567c3 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
257       const float16x8_t vb89ABCDEFc3 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
258 
259       #if XNN_ARCH_ARM64
260         vacc0x01234567 = vfmaq_lane_f16(vacc0x01234567, vb01234567c3, va0, 3);
261         vacc1x01234567 = vfmaq_lane_f16(vacc1x01234567, vb01234567c3, va1, 3);
262         vacc2x01234567 = vfmaq_lane_f16(vacc2x01234567, vb01234567c3, va2, 3);
263         vacc3x01234567 = vfmaq_lane_f16(vacc3x01234567, vb01234567c3, va3, 3);
264         vacc4x01234567 = vfmaq_lane_f16(vacc4x01234567, vb01234567c3, va4, 3);
265         vacc5x01234567 = vfmaq_lane_f16(vacc5x01234567, vb01234567c3, va5, 3);
266         vacc6x01234567 = vfmaq_lane_f16(vacc6x01234567, vb01234567c3, va6, 3);
267         vacc7x01234567 = vfmaq_lane_f16(vacc7x01234567, vb01234567c3, va7, 3);
268         vacc0x89ABCDEF = vfmaq_lane_f16(vacc0x89ABCDEF, vb89ABCDEFc3, va0, 3);
269         vacc1x89ABCDEF = vfmaq_lane_f16(vacc1x89ABCDEF, vb89ABCDEFc3, va1, 3);
270         vacc2x89ABCDEF = vfmaq_lane_f16(vacc2x89ABCDEF, vb89ABCDEFc3, va2, 3);
271         vacc3x89ABCDEF = vfmaq_lane_f16(vacc3x89ABCDEF, vb89ABCDEFc3, va3, 3);
272         vacc4x89ABCDEF = vfmaq_lane_f16(vacc4x89ABCDEF, vb89ABCDEFc3, va4, 3);
273         vacc5x89ABCDEF = vfmaq_lane_f16(vacc5x89ABCDEF, vb89ABCDEFc3, va5, 3);
274         vacc6x89ABCDEF = vfmaq_lane_f16(vacc6x89ABCDEF, vb89ABCDEFc3, va6, 3);
275         vacc7x89ABCDEF = vfmaq_lane_f16(vacc7x89ABCDEF, vb89ABCDEFc3, va7, 3);
276       #else
277         const float16x8_t va0c3 = vdupq_lane_f16(va0, 3);
278         const float16x8_t va1c3 = vdupq_lane_f16(va1, 3);
279         const float16x8_t va2c3 = vdupq_lane_f16(va2, 3);
280         const float16x8_t va3c3 = vdupq_lane_f16(va3, 3);
281         const float16x8_t va4c3 = vdupq_lane_f16(va4, 3);
282         const float16x8_t va5c3 = vdupq_lane_f16(va5, 3);
283         const float16x8_t va6c3 = vdupq_lane_f16(va6, 3);
284         const float16x8_t va7c3 = vdupq_lane_f16(va7, 3);
285 
286         vacc0x01234567 = vfmaq_f16(vacc0x01234567, va0c3, vb01234567c3);
287         vacc1x01234567 = vfmaq_f16(vacc1x01234567, va1c3, vb01234567c3);
288         vacc2x01234567 = vfmaq_f16(vacc2x01234567, va2c3, vb01234567c3);
289         vacc3x01234567 = vfmaq_f16(vacc3x01234567, va3c3, vb01234567c3);
290         vacc4x01234567 = vfmaq_f16(vacc4x01234567, va4c3, vb01234567c3);
291         vacc5x01234567 = vfmaq_f16(vacc5x01234567, va5c3, vb01234567c3);
292         vacc6x01234567 = vfmaq_f16(vacc6x01234567, va6c3, vb01234567c3);
293         vacc7x01234567 = vfmaq_f16(vacc7x01234567, va7c3, vb01234567c3);
294         vacc0x89ABCDEF = vfmaq_f16(vacc0x89ABCDEF, va0c3, vb89ABCDEFc3);
295         vacc1x89ABCDEF = vfmaq_f16(vacc1x89ABCDEF, va1c3, vb89ABCDEFc3);
296         vacc2x89ABCDEF = vfmaq_f16(vacc2x89ABCDEF, va2c3, vb89ABCDEFc3);
297         vacc3x89ABCDEF = vfmaq_f16(vacc3x89ABCDEF, va3c3, vb89ABCDEFc3);
298         vacc4x89ABCDEF = vfmaq_f16(vacc4x89ABCDEF, va4c3, vb89ABCDEFc3);
299         vacc5x89ABCDEF = vfmaq_f16(vacc5x89ABCDEF, va5c3, vb89ABCDEFc3);
300         vacc6x89ABCDEF = vfmaq_f16(vacc6x89ABCDEF, va6c3, vb89ABCDEFc3);
301         vacc7x89ABCDEF = vfmaq_f16(vacc7x89ABCDEF, va7c3, vb89ABCDEFc3);
302       #endif
303 
304       k -= 4 * sizeof(__fp16);
305     }
306     if XNN_UNLIKELY(k != 0) {
307       do {
308         const float16x8_t va0 = vld1q_dup_f16(a0); a0 += 1;
309         const float16x8_t va1 = vld1q_dup_f16(a1); a1 += 1;
310         const float16x8_t va2 = vld1q_dup_f16(a2); a2 += 1;
311         const float16x8_t va3 = vld1q_dup_f16(a3); a3 += 1;
312         const float16x8_t va4 = vld1q_dup_f16(a4); a4 += 1;
313         const float16x8_t va5 = vld1q_dup_f16(a5); a5 += 1;
314         const float16x8_t va6 = vld1q_dup_f16(a6); a6 += 1;
315         const float16x8_t va7 = vld1q_dup_f16(a7); a7 += 1;
316 
317         const float16x8_t vb01234567 = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
318         const float16x8_t vb89ABCDEF = vld1q_f16(w); w = (const void*) ((uintptr_t) w + sizeof(float16x8_t));
319 
320         vacc0x01234567 = vfmaq_f16(vacc0x01234567, va0, vb01234567);
321         vacc1x01234567 = vfmaq_f16(vacc1x01234567, va1, vb01234567);
322         vacc2x01234567 = vfmaq_f16(vacc2x01234567, va2, vb01234567);
323         vacc3x01234567 = vfmaq_f16(vacc3x01234567, va3, vb01234567);
324         vacc4x01234567 = vfmaq_f16(vacc4x01234567, va4, vb01234567);
325         vacc5x01234567 = vfmaq_f16(vacc5x01234567, va5, vb01234567);
326         vacc6x01234567 = vfmaq_f16(vacc6x01234567, va6, vb01234567);
327         vacc7x01234567 = vfmaq_f16(vacc7x01234567, va7, vb01234567);
328         vacc0x89ABCDEF = vfmaq_f16(vacc0x89ABCDEF, va0, vb89ABCDEF);
329         vacc1x89ABCDEF = vfmaq_f16(vacc1x89ABCDEF, va1, vb89ABCDEF);
330         vacc2x89ABCDEF = vfmaq_f16(vacc2x89ABCDEF, va2, vb89ABCDEF);
331         vacc3x89ABCDEF = vfmaq_f16(vacc3x89ABCDEF, va3, vb89ABCDEF);
332         vacc4x89ABCDEF = vfmaq_f16(vacc4x89ABCDEF, va4, vb89ABCDEF);
333         vacc5x89ABCDEF = vfmaq_f16(vacc5x89ABCDEF, va5, vb89ABCDEF);
334         vacc6x89ABCDEF = vfmaq_f16(vacc6x89ABCDEF, va6, vb89ABCDEF);
335         vacc7x89ABCDEF = vfmaq_f16(vacc7x89ABCDEF, va7, vb89ABCDEF);
336 
337         k -= sizeof(__fp16);
338       } while (k != 0);
339     }
340 
341 
342     const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.max));
343     vacc0x01234567 = vminq_f16(vacc0x01234567, vmax);
344     vacc1x01234567 = vminq_f16(vacc1x01234567, vmax);
345     vacc2x01234567 = vminq_f16(vacc2x01234567, vmax);
346     vacc3x01234567 = vminq_f16(vacc3x01234567, vmax);
347     vacc4x01234567 = vminq_f16(vacc4x01234567, vmax);
348     vacc5x01234567 = vminq_f16(vacc5x01234567, vmax);
349     vacc6x01234567 = vminq_f16(vacc6x01234567, vmax);
350     vacc7x01234567 = vminq_f16(vacc7x01234567, vmax);
351     vacc0x89ABCDEF = vminq_f16(vacc0x89ABCDEF, vmax);
352     vacc1x89ABCDEF = vminq_f16(vacc1x89ABCDEF, vmax);
353     vacc2x89ABCDEF = vminq_f16(vacc2x89ABCDEF, vmax);
354     vacc3x89ABCDEF = vminq_f16(vacc3x89ABCDEF, vmax);
355     vacc4x89ABCDEF = vminq_f16(vacc4x89ABCDEF, vmax);
356     vacc5x89ABCDEF = vminq_f16(vacc5x89ABCDEF, vmax);
357     vacc6x89ABCDEF = vminq_f16(vacc6x89ABCDEF, vmax);
358     vacc7x89ABCDEF = vminq_f16(vacc7x89ABCDEF, vmax);
359 
360     const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.min));
361     vacc0x01234567 = vmaxq_f16(vacc0x01234567, vmin);
362     vacc1x01234567 = vmaxq_f16(vacc1x01234567, vmin);
363     vacc2x01234567 = vmaxq_f16(vacc2x01234567, vmin);
364     vacc3x01234567 = vmaxq_f16(vacc3x01234567, vmin);
365     vacc4x01234567 = vmaxq_f16(vacc4x01234567, vmin);
366     vacc5x01234567 = vmaxq_f16(vacc5x01234567, vmin);
367     vacc6x01234567 = vmaxq_f16(vacc6x01234567, vmin);
368     vacc7x01234567 = vmaxq_f16(vacc7x01234567, vmin);
369     vacc0x89ABCDEF = vmaxq_f16(vacc0x89ABCDEF, vmin);
370     vacc1x89ABCDEF = vmaxq_f16(vacc1x89ABCDEF, vmin);
371     vacc2x89ABCDEF = vmaxq_f16(vacc2x89ABCDEF, vmin);
372     vacc3x89ABCDEF = vmaxq_f16(vacc3x89ABCDEF, vmin);
373     vacc4x89ABCDEF = vmaxq_f16(vacc4x89ABCDEF, vmin);
374     vacc5x89ABCDEF = vmaxq_f16(vacc5x89ABCDEF, vmin);
375     vacc6x89ABCDEF = vmaxq_f16(vacc6x89ABCDEF, vmin);
376     vacc7x89ABCDEF = vmaxq_f16(vacc7x89ABCDEF, vmin);
377 
378     if XNN_LIKELY(nc >= 16) {
379       vst1q_f16(c0, vacc0x01234567);
380       vst1q_f16(c0 + 8, vacc0x89ABCDEF);
381       c0 = (__fp16*) ((uintptr_t) c0 + cn_stride);
382       vst1q_f16(c1, vacc1x01234567);
383       vst1q_f16(c1 + 8, vacc1x89ABCDEF);
384       c1 = (__fp16*) ((uintptr_t) c1 + cn_stride);
385       vst1q_f16(c2, vacc2x01234567);
386       vst1q_f16(c2 + 8, vacc2x89ABCDEF);
387       c2 = (__fp16*) ((uintptr_t) c2 + cn_stride);
388       vst1q_f16(c3, vacc3x01234567);
389       vst1q_f16(c3 + 8, vacc3x89ABCDEF);
390       c3 = (__fp16*) ((uintptr_t) c3 + cn_stride);
391       vst1q_f16(c4, vacc4x01234567);
392       vst1q_f16(c4 + 8, vacc4x89ABCDEF);
393       c4 = (__fp16*) ((uintptr_t) c4 + cn_stride);
394       vst1q_f16(c5, vacc5x01234567);
395       vst1q_f16(c5 + 8, vacc5x89ABCDEF);
396       c5 = (__fp16*) ((uintptr_t) c5 + cn_stride);
397       vst1q_f16(c6, vacc6x01234567);
398       vst1q_f16(c6 + 8, vacc6x89ABCDEF);
399       c6 = (__fp16*) ((uintptr_t) c6 + cn_stride);
400       vst1q_f16(c7, vacc7x01234567);
401       vst1q_f16(c7 + 8, vacc7x89ABCDEF);
402       c7 = (__fp16*) ((uintptr_t) c7 + cn_stride);
403 
404       a0 = (const __fp16*) ((uintptr_t) a0 - kc);
405       a1 = (const __fp16*) ((uintptr_t) a1 - kc);
406       a2 = (const __fp16*) ((uintptr_t) a2 - kc);
407       a3 = (const __fp16*) ((uintptr_t) a3 - kc);
408       a4 = (const __fp16*) ((uintptr_t) a4 - kc);
409       a5 = (const __fp16*) ((uintptr_t) a5 - kc);
410       a6 = (const __fp16*) ((uintptr_t) a6 - kc);
411       a7 = (const __fp16*) ((uintptr_t) a7 - kc);
412 
413       nc -= 16;
414     } else {
415       if (nc & 8) {
416         vst1q_f16(c0, vacc0x01234567); c0 += 8;
417         vst1q_f16(c1, vacc1x01234567); c1 += 8;
418         vst1q_f16(c2, vacc2x01234567); c2 += 8;
419         vst1q_f16(c3, vacc3x01234567); c3 += 8;
420         vst1q_f16(c4, vacc4x01234567); c4 += 8;
421         vst1q_f16(c5, vacc5x01234567); c5 += 8;
422         vst1q_f16(c6, vacc6x01234567); c6 += 8;
423         vst1q_f16(c7, vacc7x01234567); c7 += 8;
424 
425         vacc0x01234567 = vacc0x89ABCDEF;
426         vacc1x01234567 = vacc1x89ABCDEF;
427         vacc2x01234567 = vacc2x89ABCDEF;
428         vacc3x01234567 = vacc3x89ABCDEF;
429         vacc4x01234567 = vacc4x89ABCDEF;
430         vacc5x01234567 = vacc5x89ABCDEF;
431         vacc6x01234567 = vacc6x89ABCDEF;
432         vacc7x01234567 = vacc7x89ABCDEF;
433       }
434       float16x4_t vacc0x0123 = vget_low_f16(vacc0x01234567);
435       float16x4_t vacc1x0123 = vget_low_f16(vacc1x01234567);
436       float16x4_t vacc2x0123 = vget_low_f16(vacc2x01234567);
437       float16x4_t vacc3x0123 = vget_low_f16(vacc3x01234567);
438       float16x4_t vacc4x0123 = vget_low_f16(vacc4x01234567);
439       float16x4_t vacc5x0123 = vget_low_f16(vacc5x01234567);
440       float16x4_t vacc6x0123 = vget_low_f16(vacc6x01234567);
441       float16x4_t vacc7x0123 = vget_low_f16(vacc7x01234567);
442       if (nc & 4) {
443         vst1_f16(c0, vacc0x0123); c0 += 4;
444         vst1_f16(c1, vacc1x0123); c1 += 4;
445         vst1_f16(c2, vacc2x0123); c2 += 4;
446         vst1_f16(c3, vacc3x0123); c3 += 4;
447         vst1_f16(c4, vacc4x0123); c4 += 4;
448         vst1_f16(c5, vacc5x0123); c5 += 4;
449         vst1_f16(c6, vacc6x0123); c6 += 4;
450         vst1_f16(c7, vacc7x0123); c7 += 4;
451 
452         vacc0x0123 = vget_high_f16(vacc0x01234567);
453         vacc1x0123 = vget_high_f16(vacc1x01234567);
454         vacc2x0123 = vget_high_f16(vacc2x01234567);
455         vacc3x0123 = vget_high_f16(vacc3x01234567);
456         vacc4x0123 = vget_high_f16(vacc4x01234567);
457         vacc5x0123 = vget_high_f16(vacc5x01234567);
458         vacc6x0123 = vget_high_f16(vacc6x01234567);
459         vacc7x0123 = vget_high_f16(vacc7x01234567);
460       }
461       if (nc & 2) {
462         vst1_lane_u32((void*) c0, vreinterpret_u32_f16(vacc0x0123), 0); c0 += 2;
463         vst1_lane_u32((void*) c1, vreinterpret_u32_f16(vacc1x0123), 0); c1 += 2;
464         vst1_lane_u32((void*) c2, vreinterpret_u32_f16(vacc2x0123), 0); c2 += 2;
465         vst1_lane_u32((void*) c3, vreinterpret_u32_f16(vacc3x0123), 0); c3 += 2;
466         vst1_lane_u32((void*) c4, vreinterpret_u32_f16(vacc4x0123), 0); c4 += 2;
467         vst1_lane_u32((void*) c5, vreinterpret_u32_f16(vacc5x0123), 0); c5 += 2;
468         vst1_lane_u32((void*) c6, vreinterpret_u32_f16(vacc6x0123), 0); c6 += 2;
469         vst1_lane_u32((void*) c7, vreinterpret_u32_f16(vacc7x0123), 0); c7 += 2;
470 
471         vacc0x0123 = vext_f16(vacc0x0123, vacc0x0123, 2);
472         vacc1x0123 = vext_f16(vacc1x0123, vacc1x0123, 2);
473         vacc2x0123 = vext_f16(vacc2x0123, vacc2x0123, 2);
474         vacc3x0123 = vext_f16(vacc3x0123, vacc3x0123, 2);
475         vacc4x0123 = vext_f16(vacc4x0123, vacc4x0123, 2);
476         vacc5x0123 = vext_f16(vacc5x0123, vacc5x0123, 2);
477         vacc6x0123 = vext_f16(vacc6x0123, vacc6x0123, 2);
478         vacc7x0123 = vext_f16(vacc7x0123, vacc7x0123, 2);
479       }
480       if (nc & 1) {
481         vst1_lane_f16(c0, vacc0x0123, 0);
482         vst1_lane_f16(c1, vacc1x0123, 0);
483         vst1_lane_f16(c2, vacc2x0123, 0);
484         vst1_lane_f16(c3, vacc3x0123, 0);
485         vst1_lane_f16(c4, vacc4x0123, 0);
486         vst1_lane_f16(c5, vacc5x0123, 0);
487         vst1_lane_f16(c6, vacc6x0123, 0);
488         vst1_lane_f16(c7, vacc7x0123, 0);
489       }
490 
491       nc = 0;
492     }
493   } while (nc != 0);
494 }
495