xref: /aosp_15_r20/external/XNNPACK/src/f16-spmm/gen/32x1-minmax-neonfp16arith.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36) 
1 // Auto-generated file. Do not edit!
2 //   Template: src/f16-spmm/neonfp16arith.c.in
3 //   Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9 
10 #include <assert.h>
11 
12 #include <arm_neon.h>
13 
14 #include <xnnpack/spmm.h>
15 
16 
xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith(
18     size_t mc,
19     size_t nc,
20     const void*restrict input,
21     const void*restrict weights,
22     const int32_t*restrict widx_dmap,
23     const uint32_t*restrict nidx_nnzmap,
24     void*restrict output,
25     size_t output_stride,
26     const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28   assert(mc != 0);
29   assert(mc % sizeof(__fp16) == 0);
30   assert(nc != 0);
31 
32   const __fp16*restrict i = (const __fp16*) input;
33   __fp16*restrict o = (__fp16*) output;
34 
35   const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.max));
36   const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(&params->neon.min));
37 
38   size_t output_decrement = output_stride * nc - 32 * sizeof(__fp16);
39   while XNN_LIKELY(mc >= 32 * sizeof(__fp16)) {
40     const __fp16*restrict w = (const __fp16*) weights;
41     const int32_t* dmap = widx_dmap;
42     const uint32_t* nnzmap = nidx_nnzmap;
43     size_t n = nc;
44     do {
45       uint32_t nnz = *nnzmap++;
46       float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
47       float16x8_t vacc89ABCDEF = vacc01234567;
48       float16x8_t vaccGHIJKLMN = vacc01234567;
49       float16x8_t vaccOPQRSTUV = vacc01234567;
50       if XNN_LIKELY(nnz != 0) {
51         do {
52           const intptr_t diff = *dmap++;
53           const float16x8_t va01234567 = vld1q_f16(i);
54           const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
55           const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
56           const float16x8_t vaOPQRSTUV = vld1q_f16(i + 24);
57           i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
58           const float16x8_t vb = vld1q_dup_f16(w); w += 1;
59           vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
60           vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
61           vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
62           vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
63         } while (--nnz != 0);
64       }
65       float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
66       float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
67       float16x8_t voutGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
68       float16x8_t voutOPQRSTUV = vminq_f16(vaccOPQRSTUV, vmax);
69       vout01234567 = vmaxq_f16(vout01234567, vmin);
70       vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
71       voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
72       voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
73       vst1q_f16(o, vout01234567);
74       vst1q_f16(o + 8, vout89ABCDEF);
75       vst1q_f16(o + 16, voutGHIJKLMN);
76       vst1q_f16(o + 24, voutOPQRSTUV);
77       o = (__fp16*restrict) ((uintptr_t) o + output_stride);
78     } while (--n != 0);
79     o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
80     i += 32;
81     mc -= 32 * sizeof(__fp16);
82   }
83   if XNN_UNLIKELY(mc != 0) {
84     output_decrement += 16 * sizeof(__fp16);
85     if (mc & (16 * sizeof(__fp16))) {
86       const __fp16*restrict w = (const __fp16*) weights;
87       const int32_t* dmap = widx_dmap;
88       const uint32_t* nnzmap = nidx_nnzmap;
89       size_t n = nc;
90       do {
91         uint32_t nnz = *nnzmap++;
92         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
93         float16x8_t vacc89ABCDEF = vacc01234567;
94         if XNN_LIKELY(nnz != 0) {
95           do {
96             const intptr_t diff = *dmap++;
97             const float16x8_t va01234567 = vld1q_f16(i);
98             const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
99             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
100             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
101             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
102             vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
103           } while (--nnz != 0);
104         }
105         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
106         float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
107         vout01234567 = vmaxq_f16(vout01234567, vmin);
108         vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
109         vst1q_f16(o, vout01234567);
110         vst1q_f16(o + 8, vout89ABCDEF);
111         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
112       } while (--n != 0);
113       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
114       i += 16;
115     }
116     output_decrement += 8 * sizeof(__fp16);
117     if (mc & (8 * sizeof(__fp16))) {
118       const __fp16*restrict w = (const __fp16*) weights;
119       const int32_t* dmap = widx_dmap;
120       const uint32_t* nnzmap = nidx_nnzmap;
121       size_t n = nc;
122       do {
123         uint32_t nnz = *nnzmap++;
124         float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
125         if XNN_LIKELY(nnz != 0) {
126           do {
127             const intptr_t diff = *dmap++;
128             const float16x8_t va01234567 = vld1q_f16(i);
129             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
130             const float16x8_t vb = vld1q_dup_f16(w); w += 1;
131             vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
132           } while (--nnz != 0);
133         }
134         float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
135         vout01234567 = vmaxq_f16(vout01234567, vmin);
136         vst1q_f16(o, vout01234567);
137         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
138       } while (--n != 0);
139       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
140       i += 8;
141     }
142     output_decrement += 4 * sizeof(__fp16);
143     if (mc & (4 * sizeof(__fp16))) {
144       const __fp16*restrict w = (const __fp16*) weights;
145       const int32_t* dmap = widx_dmap;
146       const uint32_t* nnzmap = nidx_nnzmap;
147       size_t n = nc;
148       do {
149         uint32_t nnz = *nnzmap++;
150         float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
151         if XNN_LIKELY(nnz != 0) {
152           do {
153             const intptr_t diff = *dmap++;
154             const float16x4_t va0123 = vld1_f16(i);
155             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
156             const float16x4_t vb = vld1_dup_f16(w); w += 1;
157             vacc0123 = vfma_f16(vacc0123, va0123, vb);
158           } while (--nnz != 0);
159         }
160         float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
161         vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
162         vst1_f16(o, vout0123);
163         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
164       } while (--n != 0);
165       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
166       i += 4;
167     }
168     output_decrement += 2 * sizeof(__fp16);
169     if (mc & (2 * sizeof(__fp16))) {
170       const __fp16*restrict w = (const __fp16*) weights;
171       const int32_t* dmap = widx_dmap;
172       const uint32_t* nnzmap = nidx_nnzmap;
173       size_t n = nc;
174       do {
175         uint32_t nnz = *nnzmap++;
176         float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
177         if XNN_LIKELY(nnz != 0) {
178           do {
179             const intptr_t diff = *dmap++;
180             const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32((const void*) i));
181             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
182             const float16x4_t vb = vld1_dup_f16(w); w += 1;
183             vacc01 = vfma_f16(vacc01, va01, vb);
184           } while (--nnz != 0);
185         }
186         float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
187         vout01 = vmax_f16(vout01, vget_low_f16(vmin));
188         vst1_lane_f32((void*) o, vreinterpret_f32_f16(vout01), 0);
189         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
190       } while (--n != 0);
191       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
192       i += 2;
193     }
194     output_decrement += 1 * sizeof(__fp16);
195     if (mc & (1 * sizeof(__fp16))) {
196       const __fp16*restrict w = (const __fp16*) weights;
197       const int32_t* dmap = widx_dmap;
198       const uint32_t* nnzmap = nidx_nnzmap;
199       size_t n = nc;
200       do {
201         uint32_t nnz = *nnzmap++;
202         float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
203         if XNN_LIKELY(nnz != 0) {
204           do {
205             const intptr_t diff = *dmap++;
206             const float16x4_t va0 = vld1_dup_f16(i);
207             i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
208             const float16x4_t vb = vld1_dup_f16(w); w += 1;
209             vacc0 = vfma_f16(vacc0, va0, vb);
210           } while (--nnz != 0);
211         }
212         float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
213         vout0 = vmax_f16(vout0, vget_low_f16(vmin));
214         vst1_lane_f16(o, vout0, 0);
215         o = (__fp16*restrict) ((uintptr_t) o + output_stride);
216       } while (--n != 0);
217       o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
218       i += 1;
219     }
220   }
221 }
222