1 // Auto-generated file. Do not edit!
2 // Template: src/f16-spmm/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/spmm.h>
15
16
xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith(size_t mc,size_t nc,const void * restrict input,const void * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,void * restrict output,size_t output_stride,const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_f16_spmm_minmax_ukernel_24x1__neonfp16arith(
18 size_t mc,
19 size_t nc,
20 const void*restrict input,
21 const void*restrict weights,
22 const int32_t*restrict widx_dmap,
23 const uint32_t*restrict nidx_nnzmap,
24 void*restrict output,
25 size_t output_stride,
26 const union xnn_f16_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27 {
28 assert(mc != 0);
29 assert(mc % sizeof(__fp16) == 0);
30 assert(nc != 0);
31
32 const __fp16*restrict i = (const __fp16*) input;
33 __fp16*restrict o = (__fp16*) output;
34
35 const float16x8_t vmax = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.max));
36 const float16x8_t vmin = vreinterpretq_f16_u16(vld1q_dup_u16(¶ms->neon.min));
37
38 size_t output_decrement = output_stride * nc - 24 * sizeof(__fp16);
39 while XNN_LIKELY(mc >= 24 * sizeof(__fp16)) {
40 const __fp16*restrict w = (const __fp16*) weights;
41 const int32_t* dmap = widx_dmap;
42 const uint32_t* nnzmap = nidx_nnzmap;
43 size_t n = nc;
44 do {
45 uint32_t nnz = *nnzmap++;
46 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
47 float16x8_t vacc89ABCDEF = vacc01234567;
48 float16x8_t vaccGHIJKLMN = vacc01234567;
49 if XNN_LIKELY(nnz != 0) {
50 do {
51 const intptr_t diff = *dmap++;
52 const float16x8_t va01234567 = vld1q_f16(i);
53 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
54 const float16x8_t vaGHIJKLMN = vld1q_f16(i + 16);
55 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
56 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
57 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
58 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
59 vaccGHIJKLMN = vfmaq_f16(vaccGHIJKLMN, vaGHIJKLMN, vb);
60 } while (--nnz != 0);
61 }
62 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
63 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
64 float16x8_t voutGHIJKLMN = vminq_f16(vaccGHIJKLMN, vmax);
65 vout01234567 = vmaxq_f16(vout01234567, vmin);
66 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
67 voutGHIJKLMN = vmaxq_f16(voutGHIJKLMN, vmin);
68 vst1q_f16(o, vout01234567);
69 vst1q_f16(o + 8, vout89ABCDEF);
70 vst1q_f16(o + 16, voutGHIJKLMN);
71 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
72 } while (--n != 0);
73 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
74 i += 24;
75 mc -= 24 * sizeof(__fp16);
76 }
77 if XNN_UNLIKELY(mc != 0) {
78 output_decrement += 8 * sizeof(__fp16);
79 if (mc & (16 * sizeof(__fp16))) {
80 const __fp16*restrict w = (const __fp16*) weights;
81 const int32_t* dmap = widx_dmap;
82 const uint32_t* nnzmap = nidx_nnzmap;
83 size_t n = nc;
84 do {
85 uint32_t nnz = *nnzmap++;
86 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
87 float16x8_t vacc89ABCDEF = vacc01234567;
88 if XNN_LIKELY(nnz != 0) {
89 do {
90 const intptr_t diff = *dmap++;
91 const float16x8_t va01234567 = vld1q_f16(i);
92 const float16x8_t va89ABCDEF = vld1q_f16(i + 8);
93 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
94 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
95 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
96 vacc89ABCDEF = vfmaq_f16(vacc89ABCDEF, va89ABCDEF, vb);
97 } while (--nnz != 0);
98 }
99 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
100 float16x8_t vout89ABCDEF = vminq_f16(vacc89ABCDEF, vmax);
101 vout01234567 = vmaxq_f16(vout01234567, vmin);
102 vout89ABCDEF = vmaxq_f16(vout89ABCDEF, vmin);
103 vst1q_f16(o, vout01234567);
104 vst1q_f16(o + 8, vout89ABCDEF);
105 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
106 } while (--n != 0);
107 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
108 i += 16;
109 }
110 output_decrement += 8 * sizeof(__fp16);
111 if (mc & (8 * sizeof(__fp16))) {
112 const __fp16*restrict w = (const __fp16*) weights;
113 const int32_t* dmap = widx_dmap;
114 const uint32_t* nnzmap = nidx_nnzmap;
115 size_t n = nc;
116 do {
117 uint32_t nnz = *nnzmap++;
118 float16x8_t vacc01234567 = vld1q_dup_f16(w); w += 1;
119 if XNN_LIKELY(nnz != 0) {
120 do {
121 const intptr_t diff = *dmap++;
122 const float16x8_t va01234567 = vld1q_f16(i);
123 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
124 const float16x8_t vb = vld1q_dup_f16(w); w += 1;
125 vacc01234567 = vfmaq_f16(vacc01234567, va01234567, vb);
126 } while (--nnz != 0);
127 }
128 float16x8_t vout01234567 = vminq_f16(vacc01234567, vmax);
129 vout01234567 = vmaxq_f16(vout01234567, vmin);
130 vst1q_f16(o, vout01234567);
131 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
132 } while (--n != 0);
133 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
134 i += 8;
135 }
136 output_decrement += 4 * sizeof(__fp16);
137 if (mc & (4 * sizeof(__fp16))) {
138 const __fp16*restrict w = (const __fp16*) weights;
139 const int32_t* dmap = widx_dmap;
140 const uint32_t* nnzmap = nidx_nnzmap;
141 size_t n = nc;
142 do {
143 uint32_t nnz = *nnzmap++;
144 float16x4_t vacc0123 = vld1_dup_f16(w); w += 1;
145 if XNN_LIKELY(nnz != 0) {
146 do {
147 const intptr_t diff = *dmap++;
148 const float16x4_t va0123 = vld1_f16(i);
149 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
150 const float16x4_t vb = vld1_dup_f16(w); w += 1;
151 vacc0123 = vfma_f16(vacc0123, va0123, vb);
152 } while (--nnz != 0);
153 }
154 float16x4_t vout0123 = vmin_f16(vacc0123, vget_low_f16(vmax));
155 vout0123 = vmax_f16(vout0123, vget_low_f16(vmin));
156 vst1_f16(o, vout0123);
157 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
158 } while (--n != 0);
159 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
160 i += 4;
161 }
162 output_decrement += 2 * sizeof(__fp16);
163 if (mc & (2 * sizeof(__fp16))) {
164 const __fp16*restrict w = (const __fp16*) weights;
165 const int32_t* dmap = widx_dmap;
166 const uint32_t* nnzmap = nidx_nnzmap;
167 size_t n = nc;
168 do {
169 uint32_t nnz = *nnzmap++;
170 float16x4_t vacc01 = vld1_dup_f16(w); w += 1;
171 if XNN_LIKELY(nnz != 0) {
172 do {
173 const intptr_t diff = *dmap++;
174 const float16x4_t va01 = vreinterpret_f16_f32(vld1_dup_f32((const void*) i));
175 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
176 const float16x4_t vb = vld1_dup_f16(w); w += 1;
177 vacc01 = vfma_f16(vacc01, va01, vb);
178 } while (--nnz != 0);
179 }
180 float16x4_t vout01 = vmin_f16(vacc01, vget_low_f16(vmax));
181 vout01 = vmax_f16(vout01, vget_low_f16(vmin));
182 vst1_lane_f32((void*) o, vreinterpret_f32_f16(vout01), 0);
183 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
184 } while (--n != 0);
185 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
186 i += 2;
187 }
188 output_decrement += 1 * sizeof(__fp16);
189 if (mc & (1 * sizeof(__fp16))) {
190 const __fp16*restrict w = (const __fp16*) weights;
191 const int32_t* dmap = widx_dmap;
192 const uint32_t* nnzmap = nidx_nnzmap;
193 size_t n = nc;
194 do {
195 uint32_t nnz = *nnzmap++;
196 float16x4_t vacc0 = vld1_dup_f16(w); w += 1;
197 if XNN_LIKELY(nnz != 0) {
198 do {
199 const intptr_t diff = *dmap++;
200 const float16x4_t va0 = vld1_dup_f16(i);
201 i = (const __fp16*restrict) ((uintptr_t) i + (uintptr_t) diff);
202 const float16x4_t vb = vld1_dup_f16(w); w += 1;
203 vacc0 = vfma_f16(vacc0, va0, vb);
204 } while (--nnz != 0);
205 }
206 float16x4_t vout0 = vmin_f16(vacc0, vget_low_f16(vmax));
207 vout0 = vmax_f16(vout0, vget_low_f16(vmin));
208 vst1_lane_f16(o, vout0, 0);
209 o = (__fp16*restrict) ((uintptr_t) o + output_stride);
210 } while (--n != 0);
211 o = (__fp16*restrict) ((uintptr_t) o - output_decrement);
212 i += 1;
213 }
214 }
215 }
216