xref: /aosp_15_r20/external/XNNPACK/src/f16-spmm/gen/32x1-minmax-neonfp16arith-x2.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-spmm/neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(
18     size_t mc,
19     size_t nc,
20     const void*restrict input,
21     const void*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     void*restrict output,
25     size_t output_stride,
26     const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(__fp16) == 0);
30   assert(nc != 0);
31 
32   const __fp16*restrict i = (const __fp16*) input;
33   __fp16*restrict o = (__fp16*) output;
34 
35   const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.max));
36   const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.min));
37 
38   size_t output_decrement = output_stride * nc - 32 * sizeof(__fp16);
39   while XNN_LIKELY(mc >= 32 * sizeof(__fp16)) {
40     const __fp16*restrict w = (const __fp16*) weights;
41     const int32_t* dmap = widx_dmap;
42     const uint32_t* nnzmap = nidx_nnzmap;
43     size_t n = nc;
44     do {
45       uint32_t nnz = *nnzmap++;
46       float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
47       float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
48       float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
49       float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
50       float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
51       float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
52       float16x8_t vaccOPQRSTUVx0 = vacc01234567x0;
53       float16x8_t vaccOPQRSTUVx1 = vmovq_n_f16(0.0f);
54       for (; nnz >= 2; nnz -= 2) {
55         const intptr_t diff0 = dmap[0];
56         const intptr_t diff1 = dmap[1];
57         dmap += 2;
58         const float16x8_t va01234567x0 = vld1q_f16(i);
59         const float16x8_t va89ABCDEFx0 = vld1q_f16(i + 8);
60         const float16x8_t vaGHIJKLMNx0 = vld1q_f16(i + 16);
61         const float16x8_t vaOPQRSTUVx0 = vld1q_f16(i + 24);
62         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
63         const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
64         vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
65         vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
66         vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
67         vaccOPQRSTUVx0 = vfmaq_f16(vaccOPQRSTUVx0, vaOPQRSTUVx0, vb0);
68         const float16x8_t va01234567x1 = vld1q_f16(i);
69         const float16x8_t va89ABCDEFx1 = vld1q_f16(i + 8);
70         const float16x8_t vaGHIJKLMNx1 = vld1q_f16(i + 16);
71         const float16x8_t vaOPQRSTUVx1 = vld1q_f16(i + 24);
72         i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
73         const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
74         vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
75         vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
76         vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
77         vaccOPQRSTUVx1 = vfmaq_f16(vaccOPQRSTUVx1, vaOPQRSTUVx1, vb1);
78       }
79       float16x8_t vacc01234567 = vacc01234567x0;
80       float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
81       float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
82       float16x8_t vaccOPQRSTUV = vaccOPQRSTUVx0;
83       vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
84       vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
85       vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
86       vaccOPQRSTUV = vaddq_f16(vaccOPQRSTUV, vaccOPQRSTUVx1);
87       if XNN_LIKELY(nnz != 0) {
88         do {
89           const intptr_t diff = *dmap++;
90           const float16x8_t va01234567 = vld1q_f16(i);
91           const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
92           const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
93           const float16x8_t vaOPQRSTUV = vld1q_f16(i + 24);
94           i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
95           const float16x8_t vb = vld1q_dup_f16(w); w += 1;
96           vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
97           vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
98           vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
99           vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
100         } while (--nnz != 0);
101       }
102       float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
103       float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
104       float16x8_t voutGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
105       float16x8_t voutOPQRSTUV = vminq_f16(vaccOPQRSTUV, vmax);
106       vout01234567 = vmaxq_f16(vout01234567, vmin);
107       vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
108       voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
109       voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
110       vst1q_f16(o, vout01234567);
111       vst1q_f16(o + 8, vout89ABCDEF);
112       vst1q_f16(o + 16, voutGHIJKLMN);
113       vst1q_f16(o + 24, voutOPQRSTUV);
114       o = (__fp16*restrict) ((uintptr_t) o + output_stride);
115     } while (--n != 0);
116     o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
117     i += 32;
118     mc -= 32 * sizeof(__fp16);
119   }
120   if XNN_UNLIKELY(mc != 0) {
121     output_decrement += 16 * sizeof(__fp16);
122     if (mc & (16 * sizeof(__fp16))) {
123       const __fp16*restrict w = (const __fp16*) weights;
124       const int32_t* dmap = widx_dmap;
125       const uint32_t* nnzmap = nidx_nnzmap;
126       size_t n = nc;
127       do {
128         uint32_t nnz = *nnzmap++;
129         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
130         float16x8_t vacc89ABCDEF = vacc01234567;
131         if XNN_LIKELY(nnz != 0) {
132           do {
133             const intptr_t diff = *dmap++;
134             const float16x8_t va01234567 = vld1q_f16(i);
135             const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
136             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
137             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
138             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
139             vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
140           } while (--nnz != 0);
141         }
142         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
143         float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
144         vout01234567 = vmaxq_f16(vout01234567, vmin);
145         vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
146         vst1q_f16(o, vout01234567);
147         vst1q_f16(o + 8, vout89ABCDEF);
148         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
149       } while (--n != 0);
150       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
151       i += 16;
152     }
153     output_decrement += 8 * sizeof(__fp16);
154     if (mc & (8 * sizeof(__fp16))) {
155       const __fp16*restrict w = (const __fp16*) weights;
156       const int32_t* dmap = widx_dmap;
157       const uint32_t* nnzmap = nidx_nnzmap;
158       size_t n = nc;
159       do {
160         uint32_t nnz = *nnzmap++;
161         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
162         if XNN_LIKELY(nnz != 0) {
163           do {
164             const intptr_t diff = *dmap++;
165             const float16x8_t va01234567 = vld1q_f16(i);
166             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
167             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
168             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
169           } while (--nnz != 0);
170         }
171         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
172         vout01234567 = vmaxq_f16(vout01234567, vmin);
173         vst1q_f16(o, vout01234567);
174         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
175       } while (--n != 0);
176       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
177       i += 8;
178     }
179     output_decrement += 4 * sizeof(__fp16);
180     if (mc & (4 * sizeof(__fp16))) {
181       const __fp16*restrict w = (const __fp16*) weights;
182       const int32_t* dmap = widx_dmap;
183       const uint32_t* nnzmap = nidx_nnzmap;
184       size_t n = nc;
185       do {
186         uint32_t nnz = *nnzmap++;
187         float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
188         if XNN_LIKELY(nnz != 0) {
189           do {
190             const intptr_t diff = *dmap++;
191             const float16x4_t va0123 = vld1_f16(i);
192             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
193             const float16x4_t vb = vld1_dup_f16(w); w += 1;
194             vacc0123 = vfma_f16(vacc0123, va0123, vb);
195           } while (--nnz != 0);
196         }
197         float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
198         vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
199         vst1_f16(o, vout0123);
200         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
201       } while (--n != 0);
202       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
203       i += 4;
204     }
205     output_decrement += 2 * sizeof(__fp16);
206     if (mc & (2 * sizeof(__fp16))) {
207       const __fp16*restrict w = (const __fp16*) weights;
208       const int32_t* dmap = widx_dmap;
209       const uint32_t* nnzmap = nidx_nnzmap;
210       size_t n = nc;
211       do {
212         uint32_t nnz = *nnzmap++;
213         float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
214         if XNN_LIKELY(nnz != 0) {
215           do {
216             const intptr_t diff = *dmap++;
217             const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32((const void*) i));
218             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
219             const float16x4_t vb = vld1_dup_f16(w); w += 1;
220             vacc01 = vfma_f16(vacc01, va01, vb);
221           } while (--nnz != 0);
222         }
223         float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
224         vout01 = vmax_f16(vout01, vget_low_f16(vmin));
225         vst1_lane_f32((void*) o, vreinterpret_f32_f16(vout01), 0);
226         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
227       } while (--n != 0);
228       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
229       i += 2;
230     }
231     output_decrement += 1 * sizeof(__fp16);
232     if (mc & (1 * sizeof(__fp16))) {
233       const __fp16*restrict w = (const __fp16*) weights;
234       const int32_t* dmap = widx_dmap;
235       const uint32_t* nnzmap = nidx_nnzmap;
236       size_t n = nc;
237       do {
238         uint32_t nnz = *nnzmap++;
239         float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
240         if XNN_LIKELY(nnz != 0) {
241           do {
242             const intptr_t diff = *dmap++;
243             const float16x4_t va0 = vld1_dup_f16(i);
244             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
245             const float16x4_t vb = vld1_dup_f16(w); w += 1;
246             vacc0 = vfma_f16(vacc0, va0, vb);
247           } while (--nnz != 0);
248         }
249         float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
250         vout0 = vmax_f16(vout0, vget_low_f16(vmin));
251         vst1_lane_f16(o, vout0, 0);
252         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
253       } while (--n != 0);
254       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
255       i += 1;
256     }
257   }
258 }
259