1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f32-spmm/neon-blocked.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2019 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <arm_neon.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/spmm.h>
15*4bdc9457SAndroid Build Coastguard Worker
16*4bdc9457SAndroid Build Coastguard Worker
xnn_f32_spmm_minmax_ukernel_8x4__neonfma(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_spmm_minmax_ukernel_8x4__neonfma(
18*4bdc9457SAndroid Build Coastguard Worker size_t mc,
19*4bdc9457SAndroid Build Coastguard Worker size_t nc,
20*4bdc9457SAndroid Build Coastguard Worker const float*restrict input,
21*4bdc9457SAndroid Build Coastguard Worker const float*restrict weights,
22*4bdc9457SAndroid Build Coastguard Worker const int32_t*restrict widx_dmap,
23*4bdc9457SAndroid Build Coastguard Worker const uint32_t*restrict nidx_nnzmap,
24*4bdc9457SAndroid Build Coastguard Worker float*restrict output,
25*4bdc9457SAndroid Build Coastguard Worker size_t output_stride,
26*4bdc9457SAndroid Build Coastguard Worker const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker assert(mc != 0);
29*4bdc9457SAndroid Build Coastguard Worker assert(mc % sizeof(float) == 0);
30*4bdc9457SAndroid Build Coastguard Worker assert(nc != 0);
31*4bdc9457SAndroid Build Coastguard Worker
32*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min);
33*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max);
34*4bdc9457SAndroid Build Coastguard Worker size_t output_decrement = output_stride * nc - 8 * sizeof(float);
35*4bdc9457SAndroid Build Coastguard Worker while XNN_LIKELY(mc >= 8 * sizeof(float)) {
36*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
37*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
38*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
39*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
40*4bdc9457SAndroid Build Coastguard Worker while (n >= 4) {
41*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
42*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n0 = vld1q_dup_f32(w); w += 1;
43*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n0 = vacc0123n0;
44*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n1 = vld1q_dup_f32(w); w += 1;
45*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n1 = vacc0123n1;
46*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n2 = vld1q_dup_f32(w); w += 1;
47*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n2 = vacc0123n2;
48*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n3 = vld1q_dup_f32(w); w += 1;
49*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567n3 = vacc0123n3;
50*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
51*4bdc9457SAndroid Build Coastguard Worker do {
52*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
53*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
54*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
55*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
56*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(input + 16);
57*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_f32(w); w += 4;
58*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(w + 32);
59*4bdc9457SAndroid Build Coastguard Worker vacc0123n0 = vfmaq_laneq_f32(vacc0123n0, vi0123, vw, 0);
60*4bdc9457SAndroid Build Coastguard Worker vacc4567n0 = vfmaq_laneq_f32(vacc4567n0, vi4567, vw, 0);
61*4bdc9457SAndroid Build Coastguard Worker vacc0123n1 = vfmaq_laneq_f32(vacc0123n1, vi0123, vw, 1);
62*4bdc9457SAndroid Build Coastguard Worker vacc4567n1 = vfmaq_laneq_f32(vacc4567n1, vi4567, vw, 1);
63*4bdc9457SAndroid Build Coastguard Worker vacc0123n2 = vfmaq_laneq_f32(vacc0123n2, vi0123, vw, 2);
64*4bdc9457SAndroid Build Coastguard Worker vacc4567n2 = vfmaq_laneq_f32(vacc4567n2, vi4567, vw, 2);
65*4bdc9457SAndroid Build Coastguard Worker vacc0123n3 = vfmaq_laneq_f32(vacc0123n3, vi0123, vw, 3);
66*4bdc9457SAndroid Build Coastguard Worker vacc4567n3 = vfmaq_laneq_f32(vacc4567n3, vi4567, vw, 3);
67*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
68*4bdc9457SAndroid Build Coastguard Worker }
69*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n0 = vminq_f32(vacc0123n0, vmax);
70*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n0 = vminq_f32(vacc4567n0, vmax);
71*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n1 = vminq_f32(vacc0123n1, vmax);
72*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n1 = vminq_f32(vacc4567n1, vmax);
73*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n2 = vminq_f32(vacc0123n2, vmax);
74*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n2 = vminq_f32(vacc4567n2, vmax);
75*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n3 = vminq_f32(vacc0123n3, vmax);
76*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567n3 = vminq_f32(vacc4567n3, vmax);
77*4bdc9457SAndroid Build Coastguard Worker
78*4bdc9457SAndroid Build Coastguard Worker vout0123n0 = vmaxq_f32(vout0123n0, vmin);
79*4bdc9457SAndroid Build Coastguard Worker vout4567n0 = vmaxq_f32(vout4567n0, vmin);
80*4bdc9457SAndroid Build Coastguard Worker vout0123n1 = vmaxq_f32(vout0123n1, vmin);
81*4bdc9457SAndroid Build Coastguard Worker vout4567n1 = vmaxq_f32(vout4567n1, vmin);
82*4bdc9457SAndroid Build Coastguard Worker vout0123n2 = vmaxq_f32(vout0123n2, vmin);
83*4bdc9457SAndroid Build Coastguard Worker vout4567n2 = vmaxq_f32(vout4567n2, vmin);
84*4bdc9457SAndroid Build Coastguard Worker vout0123n3 = vmaxq_f32(vout0123n3, vmin);
85*4bdc9457SAndroid Build Coastguard Worker vout4567n3 = vmaxq_f32(vout4567n3, vmin);
86*4bdc9457SAndroid Build Coastguard Worker
87*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n0);
88*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n0);
89*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
90*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n1);
91*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n1);
92*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
93*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n2);
94*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n2);
95*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
96*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n3);
97*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567n3);
98*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
99*4bdc9457SAndroid Build Coastguard Worker n -= 4;
100*4bdc9457SAndroid Build Coastguard Worker }
101*4bdc9457SAndroid Build Coastguard Worker
102*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
103*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
104*4bdc9457SAndroid Build Coastguard Worker do {
105*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
106*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
107*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc4567 = vacc0123;
108*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
109*4bdc9457SAndroid Build Coastguard Worker do {
110*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
111*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
112*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi4567 = vld1q_f32(input + 4);
113*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
114*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(input + 16);
115*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_dup_f32(w); w += 1;
116*4bdc9457SAndroid Build Coastguard Worker __builtin_prefetch(w + 32);
117*4bdc9457SAndroid Build Coastguard Worker vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
118*4bdc9457SAndroid Build Coastguard Worker vacc4567 = vfmaq_f32(vacc4567, vi4567, vw);
119*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
120*4bdc9457SAndroid Build Coastguard Worker }
121*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
122*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
123*4bdc9457SAndroid Build Coastguard Worker
124*4bdc9457SAndroid Build Coastguard Worker vout0123 = vmaxq_f32(vout0123, vmin);
125*4bdc9457SAndroid Build Coastguard Worker vout4567 = vmaxq_f32(vout4567, vmin);
126*4bdc9457SAndroid Build Coastguard Worker
127*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123);
128*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 4, vout4567);
129*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
130*4bdc9457SAndroid Build Coastguard Worker n -= 1;
131*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
132*4bdc9457SAndroid Build Coastguard Worker }
133*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
134*4bdc9457SAndroid Build Coastguard Worker input += 8;
135*4bdc9457SAndroid Build Coastguard Worker mc -= 8 * sizeof(float);
136*4bdc9457SAndroid Build Coastguard Worker }
137*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(mc != 0) {
138*4bdc9457SAndroid Build Coastguard Worker output_decrement += 4 * sizeof(float);
139*4bdc9457SAndroid Build Coastguard Worker if (mc & (4 * sizeof(float))) {
140*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
141*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
142*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
143*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
144*4bdc9457SAndroid Build Coastguard Worker while (n >= 4) {
145*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
146*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n0 = vld1q_dup_f32(w); w += 1;
147*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n1 = vld1q_dup_f32(w); w += 1;
148*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n2 = vld1q_dup_f32(w); w += 1;
149*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123n3 = vld1q_dup_f32(w); w += 1;
150*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
151*4bdc9457SAndroid Build Coastguard Worker do {
152*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
153*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
154*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
155*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_f32(w); w += 4;
156*4bdc9457SAndroid Build Coastguard Worker
157*4bdc9457SAndroid Build Coastguard Worker vacc0123n0 = vfmaq_laneq_f32(vacc0123n0, vi0123, vw, 0);
158*4bdc9457SAndroid Build Coastguard Worker vacc0123n1 = vfmaq_laneq_f32(vacc0123n1, vi0123, vw, 1);
159*4bdc9457SAndroid Build Coastguard Worker vacc0123n2 = vfmaq_laneq_f32(vacc0123n2, vi0123, vw, 2);
160*4bdc9457SAndroid Build Coastguard Worker vacc0123n3 = vfmaq_laneq_f32(vacc0123n3, vi0123, vw, 3);
161*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
162*4bdc9457SAndroid Build Coastguard Worker }
163*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n0 = vminq_f32(vacc0123n0, vmax);
164*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n1 = vminq_f32(vacc0123n1, vmax);
165*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n2 = vminq_f32(vacc0123n2, vmax);
166*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123n3 = vminq_f32(vacc0123n3, vmax);
167*4bdc9457SAndroid Build Coastguard Worker
168*4bdc9457SAndroid Build Coastguard Worker vout0123n0 = vmaxq_f32(vout0123n0, vmin);
169*4bdc9457SAndroid Build Coastguard Worker vout0123n1 = vmaxq_f32(vout0123n1, vmin);
170*4bdc9457SAndroid Build Coastguard Worker vout0123n2 = vmaxq_f32(vout0123n2, vmin);
171*4bdc9457SAndroid Build Coastguard Worker vout0123n3 = vmaxq_f32(vout0123n3, vmin);
172*4bdc9457SAndroid Build Coastguard Worker
173*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n0);
174*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
175*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n1);
176*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
177*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n2);
178*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
179*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123n3);
180*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
181*4bdc9457SAndroid Build Coastguard Worker n -= 4;
182*4bdc9457SAndroid Build Coastguard Worker }
183*4bdc9457SAndroid Build Coastguard Worker
184*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
185*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
186*4bdc9457SAndroid Build Coastguard Worker do {
187*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
188*4bdc9457SAndroid Build Coastguard Worker float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
189*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
190*4bdc9457SAndroid Build Coastguard Worker do {
191*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
192*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vi0123 = vld1q_f32(input);
193*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
194*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_dup_f32(w); w += 1;
195*4bdc9457SAndroid Build Coastguard Worker vacc0123 = vfmaq_f32(vacc0123, vi0123, vw);
196*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
197*4bdc9457SAndroid Build Coastguard Worker }
198*4bdc9457SAndroid Build Coastguard Worker float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
199*4bdc9457SAndroid Build Coastguard Worker
200*4bdc9457SAndroid Build Coastguard Worker vout0123 = vmaxq_f32(vout0123, vmin);
201*4bdc9457SAndroid Build Coastguard Worker
202*4bdc9457SAndroid Build Coastguard Worker vst1q_f32(output + 0, vout0123);
203*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
204*4bdc9457SAndroid Build Coastguard Worker n -= 1;
205*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
206*4bdc9457SAndroid Build Coastguard Worker }
207*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
208*4bdc9457SAndroid Build Coastguard Worker input += 4;
209*4bdc9457SAndroid Build Coastguard Worker }
210*4bdc9457SAndroid Build Coastguard Worker output_decrement += 2 * sizeof(float);
211*4bdc9457SAndroid Build Coastguard Worker if (mc & (2 * sizeof(float))) {
212*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
213*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
214*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
215*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
216*4bdc9457SAndroid Build Coastguard Worker while (n >= 4) {
217*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
218*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n0 = vld1_dup_f32(w); w += 1;
219*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n1 = vld1_dup_f32(w); w += 1;
220*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n2 = vld1_dup_f32(w); w += 1;
221*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01n3 = vld1_dup_f32(w); w += 1;
222*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
223*4bdc9457SAndroid Build Coastguard Worker do {
224*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
225*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi01 = vld1_f32(input);
226*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
227*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_f32(w); w += 4;
228*4bdc9457SAndroid Build Coastguard Worker
229*4bdc9457SAndroid Build Coastguard Worker vacc01n0 = vfma_laneq_f32(vacc01n0, vi01, vw, 0);
230*4bdc9457SAndroid Build Coastguard Worker vacc01n1 = vfma_laneq_f32(vacc01n1, vi01, vw, 1);
231*4bdc9457SAndroid Build Coastguard Worker vacc01n2 = vfma_laneq_f32(vacc01n2, vi01, vw, 2);
232*4bdc9457SAndroid Build Coastguard Worker vacc01n3 = vfma_laneq_f32(vacc01n3, vi01, vw, 3);
233*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
234*4bdc9457SAndroid Build Coastguard Worker }
235*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n0 = vmin_f32(vacc01n0, vget_low_f32(vmax));
236*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n1 = vmin_f32(vacc01n1, vget_low_f32(vmax));
237*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n2 = vmin_f32(vacc01n2, vget_low_f32(vmax));
238*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01n3 = vmin_f32(vacc01n3, vget_low_f32(vmax));
239*4bdc9457SAndroid Build Coastguard Worker
240*4bdc9457SAndroid Build Coastguard Worker vout01n0 = vmax_f32(vout01n0, vget_low_f32(vmin));
241*4bdc9457SAndroid Build Coastguard Worker vout01n1 = vmax_f32(vout01n1, vget_low_f32(vmin));
242*4bdc9457SAndroid Build Coastguard Worker vout01n2 = vmax_f32(vout01n2, vget_low_f32(vmin));
243*4bdc9457SAndroid Build Coastguard Worker vout01n3 = vmax_f32(vout01n3, vget_low_f32(vmin));
244*4bdc9457SAndroid Build Coastguard Worker
245*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n0);
246*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
247*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n1);
248*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
249*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n2);
250*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
251*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output + 0, vout01n3);
252*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
253*4bdc9457SAndroid Build Coastguard Worker n -= 4;
254*4bdc9457SAndroid Build Coastguard Worker }
255*4bdc9457SAndroid Build Coastguard Worker
256*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
257*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
258*4bdc9457SAndroid Build Coastguard Worker do {
259*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
260*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
261*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
262*4bdc9457SAndroid Build Coastguard Worker do {
263*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
264*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi01 = vld1_f32(input);
265*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
266*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_dup_f32(w); w += 1;
267*4bdc9457SAndroid Build Coastguard Worker vacc01 = vfma_f32(vacc01, vi01, vw);
268*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
269*4bdc9457SAndroid Build Coastguard Worker }
270*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
271*4bdc9457SAndroid Build Coastguard Worker vout01 = vmax_f32(vout01, vget_low_f32(vmin));
272*4bdc9457SAndroid Build Coastguard Worker
273*4bdc9457SAndroid Build Coastguard Worker vst1_f32(output, vout01);
274*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
275*4bdc9457SAndroid Build Coastguard Worker n -= 1;
276*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
277*4bdc9457SAndroid Build Coastguard Worker }
278*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
279*4bdc9457SAndroid Build Coastguard Worker input += 2;
280*4bdc9457SAndroid Build Coastguard Worker }
281*4bdc9457SAndroid Build Coastguard Worker output_decrement += 1 * sizeof(float);
282*4bdc9457SAndroid Build Coastguard Worker if (mc & (1 * sizeof(float))) {
283*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
284*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
285*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
286*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
287*4bdc9457SAndroid Build Coastguard Worker while (n >= 4) {
288*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
289*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n0 = vld1_dup_f32(w); w += 1;
290*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n1 = vld1_dup_f32(w); w += 1;
291*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n2 = vld1_dup_f32(w); w += 1;
292*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0n3 = vld1_dup_f32(w); w += 1;
293*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
294*4bdc9457SAndroid Build Coastguard Worker do {
295*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
296*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi0 = vld1_dup_f32(input);
297*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
298*4bdc9457SAndroid Build Coastguard Worker const float32x4_t vw = vld1q_f32(w); w += 4;
299*4bdc9457SAndroid Build Coastguard Worker
300*4bdc9457SAndroid Build Coastguard Worker vacc0n0 = vfma_laneq_f32(vacc0n0, vi0, vw, 0);
301*4bdc9457SAndroid Build Coastguard Worker vacc0n1 = vfma_laneq_f32(vacc0n1, vi0, vw, 1);
302*4bdc9457SAndroid Build Coastguard Worker vacc0n2 = vfma_laneq_f32(vacc0n2, vi0, vw, 2);
303*4bdc9457SAndroid Build Coastguard Worker vacc0n3 = vfma_laneq_f32(vacc0n3, vi0, vw, 3);
304*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
305*4bdc9457SAndroid Build Coastguard Worker }
306*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n0 = vmin_f32(vacc0n0, vget_low_f32(vmax));
307*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n1 = vmin_f32(vacc0n1, vget_low_f32(vmax));
308*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n2 = vmin_f32(vacc0n2, vget_low_f32(vmax));
309*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0n3 = vmin_f32(vacc0n3, vget_low_f32(vmax));
310*4bdc9457SAndroid Build Coastguard Worker
311*4bdc9457SAndroid Build Coastguard Worker vout0n0 = vmax_f32(vout0n0, vget_low_f32(vmin));
312*4bdc9457SAndroid Build Coastguard Worker vout0n1 = vmax_f32(vout0n1, vget_low_f32(vmin));
313*4bdc9457SAndroid Build Coastguard Worker vout0n2 = vmax_f32(vout0n2, vget_low_f32(vmin));
314*4bdc9457SAndroid Build Coastguard Worker vout0n3 = vmax_f32(vout0n3, vget_low_f32(vmin));
315*4bdc9457SAndroid Build Coastguard Worker
316*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n0, 0);
317*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
318*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n1, 0);
319*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
320*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n2, 0);
321*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
322*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output + 0, vout0n3, 0);
323*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
324*4bdc9457SAndroid Build Coastguard Worker n -= 4;
325*4bdc9457SAndroid Build Coastguard Worker }
326*4bdc9457SAndroid Build Coastguard Worker
327*4bdc9457SAndroid Build Coastguard Worker // clean up loop, fall back to nr=1
328*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(n != 0) {
329*4bdc9457SAndroid Build Coastguard Worker do {
330*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
331*4bdc9457SAndroid Build Coastguard Worker float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
332*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
333*4bdc9457SAndroid Build Coastguard Worker do {
334*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
335*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vi0 = vld1_dup_f32(input);
336*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
337*4bdc9457SAndroid Build Coastguard Worker const float32x2_t vw = vld1_dup_f32(w); w += 1;
338*4bdc9457SAndroid Build Coastguard Worker vacc0 = vfma_f32(vacc0, vi0, vw);
339*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
340*4bdc9457SAndroid Build Coastguard Worker }
341*4bdc9457SAndroid Build Coastguard Worker float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
342*4bdc9457SAndroid Build Coastguard Worker vout0 = vmax_f32(vout0, vget_low_f32(vmin));
343*4bdc9457SAndroid Build Coastguard Worker
344*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f32(output, vout0, 1);
345*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
346*4bdc9457SAndroid Build Coastguard Worker n -= 1;
347*4bdc9457SAndroid Build Coastguard Worker } while (n != 0);
348*4bdc9457SAndroid Build Coastguard Worker }
349*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
350*4bdc9457SAndroid Build Coastguard Worker input += 1;
351*4bdc9457SAndroid Build Coastguard Worker }
352*4bdc9457SAndroid Build Coastguard Worker }
353*4bdc9457SAndroid Build Coastguard Worker }
354