1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith_x2(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith_x2(
18 size_t mc,
19 size_t nc,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 size_t output_stride,
26 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mc != 0);
29 assert(mc % sizeof(__fp16) == 0);
30 assert(nc != 0);
31
32 const __fp16*restrict i = (const __fp16*) input;
33 __fp16*restrict o = (__fp16*) output;
34
35 const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.max));
36 const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.min));
37
38 size_t output_decrement = output_stride * nc - 24 * sizeof(__fp16);
39 while XNN_LIKELY(mc >= 24 * sizeof(__fp16)) {
40 const __fp16*restrict w = (const __fp16*) weights;
41 const int32_t* dmap = widx_dmap;
42 const uint32_t* nnzmap = nidx_nnzmap;
43 size_t n = nc;
44 do {
45 uint32_t nnz = *nnzmap++;
46 float16x8_t vacc01234567x0 = vld1q_dup_f16(w); w += 1;
47 float16x8_t vacc01234567x1 = vmovq_n_f16(0.0f);
48 float16x8_t vacc89ABCDEFx0 = vacc01234567x0;
49 float16x8_t vacc89ABCDEFx1 = vmovq_n_f16(0.0f);
50 float16x8_t vaccGHIJKLMNx0 = vacc01234567x0;
51 float16x8_t vaccGHIJKLMNx1 = vmovq_n_f16(0.0f);
52 for (; nnz >= 2; nnz -= 2) {
53 const intptr_t diff0 = dmap[0];
54 const intptr_t diff1 = dmap[1];
55 dmap += 2;
56 const float16x8_t va01234567x0 = vld1q_f16(i);
57 const float16x8_t va89ABCDEFx0 = vld1q_f16(i + 8);
58 const float16x8_t vaGHIJKLMNx0 = vld1q_f16(i + 16);
59 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff0);
60 const float16x8_t vb0 = vld1q_dup_f16(w); w += 1;
61 vacc01234567x0 = vfmaq_f16(vacc01234567x0, va01234567x0, vb0);
62 vacc89ABCDEFx0 = vfmaq_f16(vacc89ABCDEFx0, va89ABCDEFx0, vb0);
63 vaccGHIJKLMNx0 = vfmaq_f16(vaccGHIJKLMNx0, vaGHIJKLMNx0, vb0);
64 const float16x8_t va01234567x1 = vld1q_f16(i);
65 const float16x8_t va89ABCDEFx1 = vld1q_f16(i + 8);
66 const float16x8_t vaGHIJKLMNx1 = vld1q_f16(i + 16);
67 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff1);
68 const float16x8_t vb1 = vld1q_dup_f16(w); w += 1;
69 vacc01234567x1 = vfmaq_f16(vacc01234567x1, va01234567x1, vb1);
70 vacc89ABCDEFx1 = vfmaq_f16(vacc89ABCDEFx1, va89ABCDEFx1, vb1);
71 vaccGHIJKLMNx1 = vfmaq_f16(vaccGHIJKLMNx1, vaGHIJKLMNx1, vb1);
72 }
73 float16x8_t vacc01234567 = vacc01234567x0;
74 float16x8_t vacc89ABCDEF = vacc89ABCDEFx0;
75 float16x8_t vaccGHIJKLMN = vaccGHIJKLMNx0;
76 vacc01234567 = vaddq_f16(vacc01234567, vacc01234567x1);
77 vacc89ABCDEF = vaddq_f16(vacc89ABCDEF, vacc89ABCDEFx1);
78 vaccGHIJKLMN = vaddq_f16(vaccGHIJKLMN, vaccGHIJKLMNx1);
79 if XNN_LIKELY(nnz != 0) {
80 do {
81 const intptr_t diff = *dmap++;
82 const float16x8_t va01234567 = vld1q_f16(i);
83 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
84 const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
85 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
86 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
87 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
88 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
89 vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
90 } while (--nnz != 0);
91 }
92 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
93 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
94 float16x8_t voutGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
95 vout01234567 = vmaxq_f16(vout01234567, vmin);
96 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
97 voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
98 vst1q_f16(o, vout01234567);
99 vst1q_f16(o + 8, vout89ABCDEF);
100 vst1q_f16(o + 16, voutGHIJKLMN);
101 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
102 } while (--n != 0);
103 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
104 i += 24;
105 mc -= 24 * sizeof(__fp16);
106 }
107 if XNN_UNLIKELY(mc != 0) {
108 output_decrement += 8 * sizeof(__fp16);
109 if (mc & (16 * sizeof(__fp16))) {
110 const __fp16*restrict w = (const __fp16*) weights;
111 const int32_t* dmap = widx_dmap;
112 const uint32_t* nnzmap = nidx_nnzmap;
113 size_t n = nc;
114 do {
115 uint32_t nnz = *nnzmap++;
116 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
117 float16x8_t vacc89ABCDEF = vacc01234567;
118 if XNN_LIKELY(nnz != 0) {
119 do {
120 const intptr_t diff = *dmap++;
121 const float16x8_t va01234567 = vld1q_f16(i);
122 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
123 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
124 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
125 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
126 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
127 } while (--nnz != 0);
128 }
129 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
130 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
131 vout01234567 = vmaxq_f16(vout01234567, vmin);
132 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
133 vst1q_f16(o, vout01234567);
134 vst1q_f16(o + 8, vout89ABCDEF);
135 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
136 } while (--n != 0);
137 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
138 i += 16;
139 }
140 output_decrement += 8 * sizeof(__fp16);
141 if (mc & (8 * sizeof(__fp16))) {
142 const __fp16*restrict w = (const __fp16*) weights;
143 const int32_t* dmap = widx_dmap;
144 const uint32_t* nnzmap = nidx_nnzmap;
145 size_t n = nc;
146 do {
147 uint32_t nnz = *nnzmap++;
148 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
149 if XNN_LIKELY(nnz != 0) {
150 do {
151 const intptr_t diff = *dmap++;
152 const float16x8_t va01234567 = vld1q_f16(i);
153 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
154 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
155 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
156 } while (--nnz != 0);
157 }
158 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
159 vout01234567 = vmaxq_f16(vout01234567, vmin);
160 vst1q_f16(o, vout01234567);
161 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
162 } while (--n != 0);
163 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
164 i += 8;
165 }
166 output_decrement += 4 * sizeof(__fp16);
167 if (mc & (4 * sizeof(__fp16))) {
168 const __fp16*restrict w = (const __fp16*) weights;
169 const int32_t* dmap = widx_dmap;
170 const uint32_t* nnzmap = nidx_nnzmap;
171 size_t n = nc;
172 do {
173 uint32_t nnz = *nnzmap++;
174 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
175 if XNN_LIKELY(nnz != 0) {
176 do {
177 const intptr_t diff = *dmap++;
178 const float16x4_t va0123 = vld1_f16(i);
179 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
180 const float16x4_t vb = vld1_dup_f16(w); w += 1;
181 vacc0123 = vfma_f16(vacc0123, va0123, vb);
182 } while (--nnz != 0);
183 }
184 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
185 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
186 vst1_f16(o, vout0123);
187 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
188 } while (--n != 0);
189 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
190 i += 4;
191 }
192 output_decrement += 2 * sizeof(__fp16);
193 if (mc & (2 * sizeof(__fp16))) {
194 const __fp16*restrict w = (const __fp16*) weights;
195 const int32_t* dmap = widx_dmap;
196 const uint32_t* nnzmap = nidx_nnzmap;
197 size_t n = nc;
198 do {
199 uint32_t nnz = *nnzmap++;
200 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
201 if XNN_LIKELY(nnz != 0) {
202 do {
203 const intptr_t diff = *dmap++;
204 const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32((const void*) i));
205 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
206 const float16x4_t vb = vld1_dup_f16(w); w += 1;
207 vacc01 = vfma_f16(vacc01, va01, vb);
208 } while (--nnz != 0);
209 }
210 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
211 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
212 vst1_lane_f32((void*) o, vreinterpret_f32_f16(vout01), 0);
213 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
214 } while (--n != 0);
215 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
216 i += 2;
217 }
218 output_decrement += 1 * sizeof(__fp16);
219 if (mc & (1 * sizeof(__fp16))) {
220 const __fp16*restrict w = (const __fp16*) weights;
221 const int32_t* dmap = widx_dmap;
222 const uint32_t* nnzmap = nidx_nnzmap;
223 size_t n = nc;
224 do {
225 uint32_t nnz = *nnzmap++;
226 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
227 if XNN_LIKELY(nnz != 0) {
228 do {
229 const intptr_t diff = *dmap++;
230 const float16x4_t va0 = vld1_dup_f16(i);
231 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
232 const float16x4_t vb = vld1_dup_f16(w); w += 1;
233 vacc0 = vfma_f16(vacc0, va0, vb);
234 } while (--nnz != 0);
235 }
236 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
237 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
238 vst1_lane_f16(o, vout0, 0);
239 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
240 } while (--n != 0);
241 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
242 i += 1;
243 }
244 }
245 }
246