1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f32-spmm/neon-blocked.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2019 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <arm_neon.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/spmm.h>
15*4bdc9457SAndroid Build Coastguard Worker
16*4bdc9457SAndroid Build Coastguard Worker
xnn_f32_spmm_minmax_ukernel_12x2__neonfma(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_spmm_minmax_ukernel_12x2__neonfma(
18*4bdc9457SAndroid Build Coastguard Worker size_t mc,
19*4bdc9457SAndroid Build Coastguard Worker size_t nc,
20*4bdc9457SAndroid Build Coastguard Worker const float*restrict input,
21*4bdc9457SAndroid Build Coastguard Worker const float*restrict weights,
22*4bdc9457SAndroid Build Coastguard Worker const int32_t*restrict widx_dmap,
23*4bdc9457SAndroid Build Coastguard Worker const uint32_t*restrict nidx_nnzmap,
24*4bdc9457SAndroid Build Coastguard Worker float*restrict output,
25*4bdc9457SAndroid Build Coastguard Worker size_t output_stride,
26*4bdc9457SAndroid Build Coastguard Worker const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker assert(mc != 0);
29*4bdc9457SAndroid Build Coastguard Worker assert(mc % sizeof(float) == 0);
30*4bdc9457SAndroid Build Coastguard Worker assert(nc != 0);
31*4bdc9457SAndroid Build Coastguard Worker
32*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min);
33*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max);
34*4bdc9457SAndroid Build Coastguard Worker size_t output_decrement = output_stride * nc - 12 * sizeof(float);
35*4bdc9457SAndroid Build Coastguard Worker while XNN_LIKELY(mc >= 12 * sizeof(float)) {
36*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
37*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
38*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
39*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
40*4bdc9457SAndroid Build Coastguard Worker while (n >= 2) {
41*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
42*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n0 = vld1q_dup_f32(w); w += 1;
43*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n0 = vacc0123n0;
44*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc89ABn0 = vacc0123n0;
45*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n1 = vld1q_dup_f32(w); w += 1;
46*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n1 = vacc0123n1;
47*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc89ABn1 = vacc0123n1;
48*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
49*4bdc9457SAndroid Build Coastguard Worker do {
50*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
51*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
52*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
53*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi89AB = vld1q_f32(input + 8);
54*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
55*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(input + 16);
56*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_f32(w); w += 2;
57*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(w + 32);
58*4bdc9457SAndroid Build Coastguard Worker vacc0123n0 = vfmaq_lane_f32(vacc0123n0, vi0123, vw, 0);
59*4bdc9457SAndroid Build Coastguard Worker vacc4567n0 = vfmaq_lane_f32(vacc4567n0, vi4567, vw, 0);
60*4bdc9457SAndroid Build Coastguard Worker vacc89ABn0 = vfmaq_lane_f32(vacc89ABn0, vi89AB, vw, 0);
61*4bdc9457SAndroid Build Coastguard Worker vacc0123n1 = vfmaq_lane_f32(vacc0123n1, vi0123, vw, 1);
62*4bdc9457SAndroid Build Coastguard Worker vacc4567n1 = vfmaq_lane_f32(vacc4567n1, vi4567, vw, 1);
63*4bdc9457SAndroid Build Coastguard Worker vacc89ABn1 = vfmaq_lane_f32(vacc89ABn1, vi89AB, vw, 1);
64*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
65*4bdc9457SAndroid Build Coastguard Worker }
66*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n0 = vminq_f32(vacc0123n0, vmax);
67*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n0 = vminq_f32(vacc4567n0, vmax);
68*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout89ABn0 = vminq_f32(vacc89ABn0, vmax);
69*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n1 = vminq_f32(vacc0123n1, vmax);
70*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n1 = vminq_f32(vacc4567n1, vmax);
71*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout89ABn1 = vminq_f32(vacc89ABn1, vmax);
72*4bdc9457SAndroid Build Coastguard Worker
73*4bdc9457SAndroid Build Coastguard Worker vout0123n0 = vmaxq_f32(vout0123n0, vmin);
74*4bdc9457SAndroid Build Coastguard Worker vout4567n0 = vmaxq_f32(vout4567n0, vmin);
75*4bdc9457SAndroid Build Coastguard Worker vout89ABn0 = vmaxq_f32(vout89ABn0, vmin);
76*4bdc9457SAndroid Build Coastguard Worker vout0123n1 = vmaxq_f32(vout0123n1, vmin);
77*4bdc9457SAndroid Build Coastguard Worker vout4567n1 = vmaxq_f32(vout4567n1, vmin);
78*4bdc9457SAndroid Build Coastguard Worker vout89ABn1 = vmaxq_f32(vout89ABn1, vmin);
79*4bdc9457SAndroid Build Coastguard Worker
80*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n0);
81*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n0);
82*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 8, vout89ABn0);
83*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
84*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n1);
85*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n1);
86*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 8, vout89ABn1);
87*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
88*4bdc9457SAndroid Build Coastguard Worker n -= 2;
89*4bdc9457SAndroid Build Coastguard Worker }
90*4bdc9457SAndroid Build Coastguard Worker
91*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
92*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
93*4bdc9457SAndroid Build Coastguard Worker do {
94*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
95*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
96*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567 = vacc0123;
97*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc89AB = vacc0123;
98*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
99*4bdc9457SAndroid Build Coastguard Worker do {
100*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
101*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
102*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
103*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi89AB = vld1q_f32(input + 8);
104*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
105*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(input + 16);
106*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_dup_f32(w); w += 1;
107*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(w + 32);
108*4bdc9457SAndroid Build Coastguard Worker vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
109*4bdc9457SAndroid Build Coastguard Worker vacc4567 = vfmaq_f32(vacc4567, vi4567, vw);
110*4bdc9457SAndroid Build Coastguard Worker vacc89AB = vfmaq_f32(vacc89AB, vi89AB, vw);
111*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
112*4bdc9457SAndroid Build Coastguard Worker }
113*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
114*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
115*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
116*4bdc9457SAndroid Build Coastguard Worker
117*4bdc9457SAndroid Build Coastguard Worker vout0123 = vmaxq_f32(vout0123, vmin);
118*4bdc9457SAndroid Build Coastguard Worker vout4567 = vmaxq_f32(vout4567, vmin);
119*4bdc9457SAndroid Build Coastguard Worker vout89AB = vmaxq_f32(vout89AB, vmin);
120*4bdc9457SAndroid Build Coastguard Worker
121*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123);
122*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567);
123*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 8, vout89AB);
124*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
125*4bdc9457SAndroid Build Coastguard Worker n -= 1;
126*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
127*4bdc9457SAndroid Build Coastguard Worker }
128*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
129*4bdc9457SAndroid Build Coastguard Worker input += 12;
130*4bdc9457SAndroid Build Coastguard Worker mc -= 12 * sizeof(float);
131*4bdc9457SAndroid Build Coastguard Worker }
132*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(mc != 0) {
133*4bdc9457SAndroid Build Coastguard Worker output_decrement += 4 * sizeof(float);
134*4bdc9457SAndroid Build Coastguard Worker if (mc & (8 * sizeof(float))) {
135*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
136*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
137*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
138*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
139*4bdc9457SAndroid Build Coastguard Worker while (n >= 2) {
140*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
141*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n0 = vld1q_dup_f32(w); w += 1;
142*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n0 = vacc0123n0;
143*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n1 = vld1q_dup_f32(w); w += 1;
144*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n1 = vacc0123n1;
145*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
146*4bdc9457SAndroid Build Coastguard Worker do {
147*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
148*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
149*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
150*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
151*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_f32(w); w += 2;
152*4bdc9457SAndroid Build Coastguard Worker
153*4bdc9457SAndroid Build Coastguard Worker vacc0123n0 = vfmaq_lane_f32(vacc0123n0, vi0123, vw, 0);
154*4bdc9457SAndroid Build Coastguard Worker vacc4567n0 = vfmaq_lane_f32(vacc4567n0, vi4567, vw, 0);
155*4bdc9457SAndroid Build Coastguard Worker vacc0123n1 = vfmaq_lane_f32(vacc0123n1, vi0123, vw, 1);
156*4bdc9457SAndroid Build Coastguard Worker vacc4567n1 = vfmaq_lane_f32(vacc4567n1, vi4567, vw, 1);
157*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
158*4bdc9457SAndroid Build Coastguard Worker }
159*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n0 = vminq_f32(vacc0123n0, vmax);
160*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n0 = vminq_f32(vacc4567n0, vmax);
161*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n1 = vminq_f32(vacc0123n1, vmax);
162*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n1 = vminq_f32(vacc4567n1, vmax);
163*4bdc9457SAndroid Build Coastguard Worker
164*4bdc9457SAndroid Build Coastguard Worker vout0123n0 = vmaxq_f32(vout0123n0, vmin);
165*4bdc9457SAndroid Build Coastguard Worker vout4567n0 = vmaxq_f32(vout4567n0, vmin);
166*4bdc9457SAndroid Build Coastguard Worker vout0123n1 = vmaxq_f32(vout0123n1, vmin);
167*4bdc9457SAndroid Build Coastguard Worker vout4567n1 = vmaxq_f32(vout4567n1, vmin);
168*4bdc9457SAndroid Build Coastguard Worker
169*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n0);
170*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n0);
171*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
172*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n1);
173*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n1);
174*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
175*4bdc9457SAndroid Build Coastguard Worker n -= 2;
176*4bdc9457SAndroid Build Coastguard Worker }
177*4bdc9457SAndroid Build Coastguard Worker
178*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
179*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
180*4bdc9457SAndroid Build Coastguard Worker do {
181*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
182*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
183*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567 = vacc0123;
184*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
185*4bdc9457SAndroid Build Coastguard Worker do {
186*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
187*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
188*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
189*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
190*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_dup_f32(w); w += 1;
191*4bdc9457SAndroid Build Coastguard Worker vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
192*4bdc9457SAndroid Build Coastguard Worker vacc4567 = vfmaq_f32(vacc4567, vi4567, vw);
193*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
194*4bdc9457SAndroid Build Coastguard Worker }
195*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
196*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
197*4bdc9457SAndroid Build Coastguard Worker
198*4bdc9457SAndroid Build Coastguard Worker vout0123 = vmaxq_f32(vout0123, vmin);
199*4bdc9457SAndroid Build Coastguard Worker vout4567 = vmaxq_f32(vout4567, vmin);
200*4bdc9457SAndroid Build Coastguard Worker
201*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123);
202*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567);
203*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
204*4bdc9457SAndroid Build Coastguard Worker n -= 1;
205*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
206*4bdc9457SAndroid Build Coastguard Worker }
207*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
208*4bdc9457SAndroid Build Coastguard Worker input += 8;
209*4bdc9457SAndroid Build Coastguard Worker }
210*4bdc9457SAndroid Build Coastguard Worker output_decrement += 4 * sizeof(float);
211*4bdc9457SAndroid Build Coastguard Worker if (mc & (4 * sizeof(float))) {
212*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
213*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
214*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
215*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
216*4bdc9457SAndroid Build Coastguard Worker while (n >= 2) {
217*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
218*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n0 = vld1q_dup_f32(w); w += 1;
219*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n1 = vld1q_dup_f32(w); w += 1;
220*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
221*4bdc9457SAndroid Build Coastguard Worker do {
222*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
223*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
224*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
225*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_f32(w); w += 2;
226*4bdc9457SAndroid Build Coastguard Worker
227*4bdc9457SAndroid Build Coastguard Worker vacc0123n0 = vfmaq_lane_f32(vacc0123n0, vi0123, vw, 0);
228*4bdc9457SAndroid Build Coastguard Worker vacc0123n1 = vfmaq_lane_f32(vacc0123n1, vi0123, vw, 1);
229*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
230*4bdc9457SAndroid Build Coastguard Worker }
231*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n0 = vminq_f32(vacc0123n0, vmax);
232*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n1 = vminq_f32(vacc0123n1, vmax);
233*4bdc9457SAndroid Build Coastguard Worker
234*4bdc9457SAndroid Build Coastguard Worker vout0123n0 = vmaxq_f32(vout0123n0, vmin);
235*4bdc9457SAndroid Build Coastguard Worker vout0123n1 = vmaxq_f32(vout0123n1, vmin);
236*4bdc9457SAndroid Build Coastguard Worker
237*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n0);
238*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
239*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n1);
240*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
241*4bdc9457SAndroid Build Coastguard Worker n -= 2;
242*4bdc9457SAndroid Build Coastguard Worker }
243*4bdc9457SAndroid Build Coastguard Worker
244*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
245*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
246*4bdc9457SAndroid Build Coastguard Worker do {
247*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
248*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
249*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
250*4bdc9457SAndroid Build Coastguard Worker do {
251*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
252*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
253*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
254*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_dup_f32(w); w += 1;
255*4bdc9457SAndroid Build Coastguard Worker vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
256*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
257*4bdc9457SAndroid Build Coastguard Worker }
258*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
259*4bdc9457SAndroid Build Coastguard Worker
260*4bdc9457SAndroid Build Coastguard Worker vout0123 = vmaxq_f32(vout0123, vmin);
261*4bdc9457SAndroid Build Coastguard Worker
262*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123);
263*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
264*4bdc9457SAndroid Build Coastguard Worker n -= 1;
265*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
266*4bdc9457SAndroid Build Coastguard Worker }
267*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
268*4bdc9457SAndroid Build Coastguard Worker input += 4;
269*4bdc9457SAndroid Build Coastguard Worker }
270*4bdc9457SAndroid Build Coastguard Worker output_decrement += 2 * sizeof(float);
271*4bdc9457SAndroid Build Coastguard Worker if (mc & (2 * sizeof(float))) {
272*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
273*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
274*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
275*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
276*4bdc9457SAndroid Build Coastguard Worker while (n >= 2) {
277*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
278*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n0 = vld1_dup_f32(w); w += 1;
279*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n1 = vld1_dup_f32(w); w += 1;
280*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
281*4bdc9457SAndroid Build Coastguard Worker do {
282*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
283*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi01 = vld1_f32(input);
284*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
285*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_f32(w); w += 2;
286*4bdc9457SAndroid Build Coastguard Worker
287*4bdc9457SAndroid Build Coastguard Worker vacc01n0 = vfma_lane_f32(vacc01n0, vi01, vw, 0);
288*4bdc9457SAndroid Build Coastguard Worker vacc01n1 = vfma_lane_f32(vacc01n1, vi01, vw, 1);
289*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
290*4bdc9457SAndroid Build Coastguard Worker }
291*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n0 = vmin_f32(vacc01n0, vget_low_f32(vmax));
292*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n1 = vmin_f32(vacc01n1, vget_low_f32(vmax));
293*4bdc9457SAndroid Build Coastguard Worker
294*4bdc9457SAndroid Build Coastguard Worker vout01n0 = vmax_f32(vout01n0, vget_low_f32(vmin));
295*4bdc9457SAndroid Build Coastguard Worker vout01n1 = vmax_f32(vout01n1, vget_low_f32(vmin));
296*4bdc9457SAndroid Build Coastguard Worker
297*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n0);
298*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
299*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n1);
300*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
301*4bdc9457SAndroid Build Coastguard Worker n -= 2;
302*4bdc9457SAndroid Build Coastguard Worker }
303*4bdc9457SAndroid Build Coastguard Worker
304*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
305*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
306*4bdc9457SAndroid Build Coastguard Worker do {
307*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
308*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
309*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
310*4bdc9457SAndroid Build Coastguard Worker do {
311*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
312*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi01 = vld1_f32(input);
313*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
314*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_dup_f32(w); w += 1;
315*4bdc9457SAndroid Build Coastguard Worker vacc01 = vfma_f32(vacc01, vi01, vw);
316*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
317*4bdc9457SAndroid Build Coastguard Worker }
318*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
319*4bdc9457SAndroid Build Coastguard Worker vout01 = vmax_f32(vout01, vget_low_f32(vmin));
320*4bdc9457SAndroid Build Coastguard Worker
321*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output, vout01);
322*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
323*4bdc9457SAndroid Build Coastguard Worker n -= 1;
324*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
325*4bdc9457SAndroid Build Coastguard Worker }
326*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
327*4bdc9457SAndroid Build Coastguard Worker input += 2;
328*4bdc9457SAndroid Build Coastguard Worker }
329*4bdc9457SAndroid Build Coastguard Worker output_decrement += 1 * sizeof(float);
330*4bdc9457SAndroid Build Coastguard Worker if (mc & (1 * sizeof(float))) {
331*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
332*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
333*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
334*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
335*4bdc9457SAndroid Build Coastguard Worker while (n >= 2) {
336*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
337*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n0 = vld1_dup_f32(w); w += 1;
338*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n1 = vld1_dup_f32(w); w += 1;
339*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
340*4bdc9457SAndroid Build Coastguard Worker do {
341*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
342*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi0 = vld1_dup_f32(input);
343*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
344*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_f32(w); w += 2;
345*4bdc9457SAndroid Build Coastguard Worker
346*4bdc9457SAndroid Build Coastguard Worker vacc0n0 = vfma_lane_f32(vacc0n0, vi0, vw, 0);
347*4bdc9457SAndroid Build Coastguard Worker vacc0n1 = vfma_lane_f32(vacc0n1, vi0, vw, 1);
348*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
349*4bdc9457SAndroid Build Coastguard Worker }
350*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n0 = vmin_f32(vacc0n0, vget_low_f32(vmax));
351*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n1 = vmin_f32(vacc0n1, vget_low_f32(vmax));
352*4bdc9457SAndroid Build Coastguard Worker
353*4bdc9457SAndroid Build Coastguard Worker vout0n0 = vmax_f32(vout0n0, vget_low_f32(vmin));
354*4bdc9457SAndroid Build Coastguard Worker vout0n1 = vmax_f32(vout0n1, vget_low_f32(vmin));
355*4bdc9457SAndroid Build Coastguard Worker
356*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n0, 0);
357*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
358*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n1, 0);
359*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
360*4bdc9457SAndroid Build Coastguard Worker n -= 2;
361*4bdc9457SAndroid Build Coastguard Worker }
362*4bdc9457SAndroid Build Coastguard Worker
363*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
364*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
365*4bdc9457SAndroid Build Coastguard Worker do {
366*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
367*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
368*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
369*4bdc9457SAndroid Build Coastguard Worker do {
370*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
371*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi0 = vld1_dup_f32(input);
372*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
373*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_dup_f32(w); w += 1;
374*4bdc9457SAndroid Build Coastguard Worker vacc0 = vfma_f32(vacc0, vi0, vw);
375*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
376*4bdc9457SAndroid Build Coastguard Worker }
377*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
378*4bdc9457SAndroid Build Coastguard Worker vout0 = vmax_f32(vout0, vget_low_f32(vmin));
379*4bdc9457SAndroid Build Coastguard Worker
380*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output, vout0, 1);
381*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
382*4bdc9457SAndroid Build Coastguard Worker n -= 1;
383*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
384*4bdc9457SAndroid Build Coastguard Worker }
385*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
386*4bdc9457SAndroid Build Coastguard Worker input += 1;
387*4bdc9457SAndroid Build Coastguard Worker }
388*4bdc9457SAndroid Build Coastguard Worker }
389*4bdc9457SAndroid Build Coastguard Worker }
390