1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_32x1__neonfp16arith_x2(
18 size_t mc,
19 size_t nc,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 size_t output_stride,
26 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mc != 0);
29 assert(mc % sizeof(__fp16) == 0);
30 assert(nc != 0);
31
32 const __fp16*restrict i = (const __fp16*) input;
33 __fp16*restrict o = (__fp16*) output;
34
35 const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.max));
36 const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.min));
37
38 size_t output_decrement = output_stride * nc - 32 * sizeof(__fp16);
39 while XNN_LIKELY(mc >= 32 * sizeof(__fp16)) {
40 const __fp16*restrict w = (const __fp16*) weights;
41 const int32_t* dmap = widx_dmap;
42 const uint32_t* nnzmap = nidx_nnzmap;
43 size_t n = nc;
44 do {
45 uint32_t nnz = *nnzmap++;
46 float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
47 float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
48 float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
49 float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
50 float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
51 float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
52 float16x8_t vaccOPQRSTUVx0 = vacc01234567x0;
53 float16x8_t vaccOPQRSTUVx1 = vmovq_n_f16(0.0f);
54 for (; nnz >= 2; nnz -= 2) {
55 const intptr_t diff0 = dmap[0];
56 const intptr_t diff1 = dmap[1];
57 dmap += 2;
58 const float16x8_t va01234567x0 = vld1q_f16(i);
59 const float16x8_t va89ABCDEFx0 = vld1q_f16(i + 8);
60 const float16x8_t vaGHIJKLMNx0 = vld1q_f16(i + 16);
61 const float16x8_t vaOPQRSTUVx0 = vld1q_f16(i + 24);
62 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
63 const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
64 vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
65 vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
66 vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
67 vaccOPQRSTUVx0 = vfmaq_f16(vaccOPQRSTUVx0, vaOPQRSTUVx0, vb0);
68 const float16x8_t va01234567x1 = vld1q_f16(i);
69 const float16x8_t va89ABCDEFx1 = vld1q_f16(i + 8);
70 const float16x8_t vaGHIJKLMNx1 = vld1q_f16(i + 16);
71 const float16x8_t vaOPQRSTUVx1 = vld1q_f16(i + 24);
72 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
73 const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
74 vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
75 vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
76 vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
77 vaccOPQRSTUVx1 = vfmaq_f16(vaccOPQRSTUVx1, vaOPQRSTUVx1, vb1);
78 }
79 float16x8_t vacc01234567 = vacc01234567x0;
80 float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
81 float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
82 float16x8_t vaccOPQRSTUV = vaccOPQRSTUVx0;
83 vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
84 vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
85 vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
86 vaccOPQRSTUV = vaddq_f16(vaccOPQRSTUV, vaccOPQRSTUVx1);
87 if XNN_LIKELY(nnz != 0) {
88 do {
89 const intptr_t diff = *dmap++;
90 const float16x8_t va01234567 = vld1q_f16(i);
91 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
92 const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
93 const float16x8_t vaOPQRSTUV = vld1q_f16(i + 24);
94 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
95 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
96 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
97 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
98 vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
99 vaccOPQRSTUV = vfmaq_f16(vaccOPQRSTUV, vaOPQRSTUV, vb);
100 } while (--nnz != 0);
101 }
102 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
103 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
104 float16x8_t voutGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
105 float16x8_t voutOPQRSTUV = vminq_f16(vaccOPQRSTUV, vmax);
106 vout01234567 = vmaxq_f16(vout01234567, vmin);
107 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
108 voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
109 voutOPQRSTUV = vmaxq_f16(voutOPQRSTUV, vmin);
110 vst1q_f16(o, vout01234567);
111 vst1q_f16(o + 8, vout89ABCDEF);
112 vst1q_f16(o + 16, voutGHIJKLMN);
113 vst1q_f16(o + 24, voutOPQRSTUV);
114 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
115 } while (--n != 0);
116 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
117 i += 32;
118 mc -= 32 * sizeof(__fp16);
119 }
120 if XNN_UNLIKELY(mc != 0) {
121 output_decrement += 16 * sizeof(__fp16);
122 if (mc & (16 * sizeof(__fp16))) {
123 const __fp16*restrict w = (const __fp16*) weights;
124 const int32_t* dmap = widx_dmap;
125 const uint32_t* nnzmap = nidx_nnzmap;
126 size_t n = nc;
127 do {
128 uint32_t nnz = *nnzmap++;
129 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
130 float16x8_t vacc89ABCDEF = vacc01234567;
131 if XNN_LIKELY(nnz != 0) {
132 do {
133 const intptr_t diff = *dmap++;
134 const float16x8_t va01234567 = vld1q_f16(i);
135 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
136 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
137 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
138 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
139 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
140 } while (--nnz != 0);
141 }
142 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
143 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
144 vout01234567 = vmaxq_f16(vout01234567, vmin);
145 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
146 vst1q_f16(o, vout01234567);
147 vst1q_f16(o + 8, vout89ABCDEF);
148 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
149 } while (--n != 0);
150 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
151 i += 16;
152 }
153 output_decrement += 8 * sizeof(__fp16);
154 if (mc & (8 * sizeof(__fp16))) {
155 const __fp16*restrict w = (const __fp16*) weights;
156 const int32_t* dmap = widx_dmap;
157 const uint32_t* nnzmap = nidx_nnzmap;
158 size_t n = nc;
159 do {
160 uint32_t nnz = *nnzmap++;
161 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
162 if XNN_LIKELY(nnz != 0) {
163 do {
164 const intptr_t diff = *dmap++;
165 const float16x8_t va01234567 = vld1q_f16(i);
166 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
167 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
168 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
169 } while (--nnz != 0);
170 }
171 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
172 vout01234567 = vmaxq_f16(vout01234567, vmin);
173 vst1q_f16(o, vout01234567);
174 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
175 } while (--n != 0);
176 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
177 i += 8;
178 }
179 output_decrement += 4 * sizeof(__fp16);
180 if (mc & (4 * sizeof(__fp16))) {
181 const __fp16*restrict w = (const __fp16*) weights;
182 const int32_t* dmap = widx_dmap;
183 const uint32_t* nnzmap = nidx_nnzmap;
184 size_t n = nc;
185 do {
186 uint32_t nnz = *nnzmap++;
187 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
188 if XNN_LIKELY(nnz != 0) {
189 do {
190 const intptr_t diff = *dmap++;
191 const float16x4_t va0123 = vld1_f16(i);
192 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
193 const float16x4_t vb = vld1_dup_f16(w); w += 1;
194 vacc0123 = vfma_f16(vacc0123, va0123, vb);
195 } while (--nnz != 0);
196 }
197 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
198 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
199 vst1_f16(o, vout0123);
200 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
201 } while (--n != 0);
202 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
203 i += 4;
204 }
205 output_decrement += 2 * sizeof(__fp16);
206 if (mc & (2 * sizeof(__fp16))) {
207 const __fp16*restrict w = (const __fp16*) weights;
208 const int32_t* dmap = widx_dmap;
209 const uint32_t* nnzmap = nidx_nnzmap;
210 size_t n = nc;
211 do {
212 uint32_t nnz = *nnzmap++;
213 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
214 if XNN_LIKELY(nnz != 0) {
215 do {
216 const intptr_t diff = *dmap++;
217 const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32((const void*) i));
218 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
219 const float16x4_t vb = vld1_dup_f16(w); w += 1;
220 vacc01 = vfma_f16(vacc01, va01, vb);
221 } while (--nnz != 0);
222 }
223 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
224 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
225 vst1_lane_f32((void*) o, vreinterpret_f32_f16(vout01), 0);
226 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
227 } while (--n != 0);
228 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
229 i += 2;
230 }
231 output_decrement += 1 * sizeof(__fp16);
232 if (mc & (1 * sizeof(__fp16))) {
233 const __fp16*restrict w = (const __fp16*) weights;
234 const int32_t* dmap = widx_dmap;
235 const uint32_t* nnzmap = nidx_nnzmap;
236 size_t n = nc;
237 do {
238 uint32_t nnz = *nnzmap++;
239 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
240 if XNN_LIKELY(nnz != 0) {
241 do {
242 const intptr_t diff = *dmap++;
243 const float16x4_t va0 = vld1_dup_f16(i);
244 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
245 const float16x4_t vb = vld1_dup_f16(w); w += 1;
246 vacc0 = vfma_f16(vacc0, va0, vb);
247 } while (--nnz != 0);
248 }
249 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
250 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
251 vst1_lane_f16(o, vout0, 0);
252 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
253 } while (--n != 0);
254 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
255 i += 1;
256 }
257 }
258 }
259