xref: /aosp_15_r20/external/XNNPACK/src/f32-spmm/gen/32x1-minmax-neon-x2.c (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker //   Template: src/f32-spmm/neon.c.in
3*4bdc9457SAndroid Build Coastguard Worker //   Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2019 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker 
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker 
12*4bdc9457SAndroid Build Coastguard Worker #include <arm_neon.h>
13*4bdc9457SAndroid Build Coastguard Worker 
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/spmm.h>
15*4bdc9457SAndroid Build Coastguard Worker 
16*4bdc9457SAndroid Build Coastguard Worker 
xnn_f32_spmm_minmax_ukernel_32x1__neon_x2(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_spmm_minmax_ukernel_32x1__neon_x2(
18*4bdc9457SAndroid Build Coastguard Worker     size_t mc,
19*4bdc9457SAndroid Build Coastguard Worker     size_t nc,
20*4bdc9457SAndroid Build Coastguard Worker     const float*restrict input,
21*4bdc9457SAndroid Build Coastguard Worker     const float*restrict weights,
22*4bdc9457SAndroid Build Coastguard Worker     const int32_t*restrict widx_dmap,
23*4bdc9457SAndroid Build Coastguard Worker     const uint32_t*restrict nidx_nnzmap,
24*4bdc9457SAndroid Build Coastguard Worker     float*restrict output,
25*4bdc9457SAndroid Build Coastguard Worker     size_t output_stride,
26*4bdc9457SAndroid Build Coastguard Worker     const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker   assert(mc != 0);
29*4bdc9457SAndroid Build Coastguard Worker   assert(mc % sizeof(float) == 0);
30*4bdc9457SAndroid Build Coastguard Worker   assert(nc != 0);
31*4bdc9457SAndroid Build Coastguard Worker 
32*4bdc9457SAndroid Build Coastguard Worker   const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
33*4bdc9457SAndroid Build Coastguard Worker   const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
34*4bdc9457SAndroid Build Coastguard Worker   size_t output_decrement = output_stride * nc - 32 * sizeof(float);
35*4bdc9457SAndroid Build Coastguard Worker   while XNN_LIKELY(mc >= 32 * sizeof(float)) {
36*4bdc9457SAndroid Build Coastguard Worker     const float*restrict w = weights;
37*4bdc9457SAndroid Build Coastguard Worker     const int32_t* dmap = widx_dmap;
38*4bdc9457SAndroid Build Coastguard Worker     const uint32_t* nnzmap = nidx_nnzmap;
39*4bdc9457SAndroid Build Coastguard Worker     size_t n = nc;
40*4bdc9457SAndroid Build Coastguard Worker     do {
41*4bdc9457SAndroid Build Coastguard Worker       uint32_t nnz = *nnzmap++;
42*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc0123x0 = vld1q_dup_f32(w); w += 1;
43*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc0123x1 = vmovq_n_f32(0.0f);
44*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc4567x0 = vacc0123x0;
45*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc4567x1 = vmovq_n_f32(0.0f);
46*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc89ABx0 = vacc0123x0;
47*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc89ABx1 = vmovq_n_f32(0.0f);
48*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccCDEFx0 = vacc0123x0;
49*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccCDEFx1 = vmovq_n_f32(0.0f);
50*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccGHIJx0 = vacc0123x0;
51*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccGHIJx1 = vmovq_n_f32(0.0f);
52*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccKLMNx0 = vacc0123x0;
53*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccKLMNx1 = vmovq_n_f32(0.0f);
54*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccOPQRx0 = vacc0123x0;
55*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccOPQRx1 = vmovq_n_f32(0.0f);
56*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccSTUVx0 = vacc0123x0;
57*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccSTUVx1 = vmovq_n_f32(0.0f);
58*4bdc9457SAndroid Build Coastguard Worker       for (; nnz >= 2; nnz -= 2) {
59*4bdc9457SAndroid Build Coastguard Worker         const intptr_t diff0 = dmap[0];
60*4bdc9457SAndroid Build Coastguard Worker         const intptr_t diff1 = dmap[1];
61*4bdc9457SAndroid Build Coastguard Worker         dmap += 2;
62*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi0123x0 = vld1q_f32(input);
63*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi4567x0 = vld1q_f32(input + 4);
64*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi89ABx0 = vld1q_f32(input + 8);
65*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viCDEFx0 = vld1q_f32(input + 12);
66*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viGHIJx0 = vld1q_f32(input + 16);
67*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viKLMNx0 = vld1q_f32(input + 20);
68*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viOPQRx0 = vld1q_f32(input + 24);
69*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viSTUVx0 = vld1q_f32(input + 28);
70*4bdc9457SAndroid Build Coastguard Worker         input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff0);
71*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(input + 16);
72*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(input + 32);
73*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vw0 = vld1q_dup_f32(w); w += 1;
74*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(w + 32);
75*4bdc9457SAndroid Build Coastguard Worker         vacc0123x0 = vmlaq_f32(vacc0123x0, vi0123x0, vw0);
76*4bdc9457SAndroid Build Coastguard Worker         vacc4567x0 = vmlaq_f32(vacc4567x0, vi4567x0, vw0);
77*4bdc9457SAndroid Build Coastguard Worker         vacc89ABx0 = vmlaq_f32(vacc89ABx0, vi89ABx0, vw0);
78*4bdc9457SAndroid Build Coastguard Worker         vaccCDEFx0 = vmlaq_f32(vaccCDEFx0, viCDEFx0, vw0);
79*4bdc9457SAndroid Build Coastguard Worker         vaccGHIJx0 = vmlaq_f32(vaccGHIJx0, viGHIJx0, vw0);
80*4bdc9457SAndroid Build Coastguard Worker         vaccKLMNx0 = vmlaq_f32(vaccKLMNx0, viKLMNx0, vw0);
81*4bdc9457SAndroid Build Coastguard Worker         vaccOPQRx0 = vmlaq_f32(vaccOPQRx0, viOPQRx0, vw0);
82*4bdc9457SAndroid Build Coastguard Worker         vaccSTUVx0 = vmlaq_f32(vaccSTUVx0, viSTUVx0, vw0);
83*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi0123x1 = vld1q_f32(input);
84*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi4567x1 = vld1q_f32(input + 4);
85*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vi89ABx1 = vld1q_f32(input + 8);
86*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viCDEFx1 = vld1q_f32(input + 12);
87*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viGHIJx1 = vld1q_f32(input + 16);
88*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viKLMNx1 = vld1q_f32(input + 20);
89*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viOPQRx1 = vld1q_f32(input + 24);
90*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t viSTUVx1 = vld1q_f32(input + 28);
91*4bdc9457SAndroid Build Coastguard Worker         input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff1);
92*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(input + 16);
93*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(input + 32);
94*4bdc9457SAndroid Build Coastguard Worker         const float32x4_t vw1 = vld1q_dup_f32(w); w += 1;
95*4bdc9457SAndroid Build Coastguard Worker         __builtin_prefetch(w + 32);
96*4bdc9457SAndroid Build Coastguard Worker         vacc0123x1 = vmlaq_f32(vacc0123x1, vi0123x1, vw1);
97*4bdc9457SAndroid Build Coastguard Worker         vacc4567x1 = vmlaq_f32(vacc4567x1, vi4567x1, vw1);
98*4bdc9457SAndroid Build Coastguard Worker         vacc89ABx1 = vmlaq_f32(vacc89ABx1, vi89ABx1, vw1);
99*4bdc9457SAndroid Build Coastguard Worker         vaccCDEFx1 = vmlaq_f32(vaccCDEFx1, viCDEFx1, vw1);
100*4bdc9457SAndroid Build Coastguard Worker         vaccGHIJx1 = vmlaq_f32(vaccGHIJx1, viGHIJx1, vw1);
101*4bdc9457SAndroid Build Coastguard Worker         vaccKLMNx1 = vmlaq_f32(vaccKLMNx1, viKLMNx1, vw1);
102*4bdc9457SAndroid Build Coastguard Worker         vaccOPQRx1 = vmlaq_f32(vaccOPQRx1, viOPQRx1, vw1);
103*4bdc9457SAndroid Build Coastguard Worker         vaccSTUVx1 = vmlaq_f32(vaccSTUVx1, viSTUVx1, vw1);
104*4bdc9457SAndroid Build Coastguard Worker       }
105*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc0123 = vacc0123x0;
106*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc4567 = vacc4567x0;
107*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vacc89AB = vacc89ABx0;
108*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccCDEF = vaccCDEFx0;
109*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccGHIJ = vaccGHIJx0;
110*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccKLMN = vaccKLMNx0;
111*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccOPQR = vaccOPQRx0;
112*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vaccSTUV = vaccSTUVx0;
113*4bdc9457SAndroid Build Coastguard Worker       vacc0123 = vaddq_f32(vacc0123, vacc0123x1);
114*4bdc9457SAndroid Build Coastguard Worker       vacc4567 = vaddq_f32(vacc4567, vacc4567x1);
115*4bdc9457SAndroid Build Coastguard Worker       vacc89AB = vaddq_f32(vacc89AB, vacc89ABx1);
116*4bdc9457SAndroid Build Coastguard Worker       vaccCDEF = vaddq_f32(vaccCDEF, vaccCDEFx1);
117*4bdc9457SAndroid Build Coastguard Worker       vaccGHIJ = vaddq_f32(vaccGHIJ, vaccGHIJx1);
118*4bdc9457SAndroid Build Coastguard Worker       vaccKLMN = vaddq_f32(vaccKLMN, vaccKLMNx1);
119*4bdc9457SAndroid Build Coastguard Worker       vaccOPQR = vaddq_f32(vaccOPQR, vaccOPQRx1);
120*4bdc9457SAndroid Build Coastguard Worker       vaccSTUV = vaddq_f32(vaccSTUV, vaccSTUVx1);
121*4bdc9457SAndroid Build Coastguard Worker       if XNN_LIKELY(nnz != 0) {
122*4bdc9457SAndroid Build Coastguard Worker         do {
123*4bdc9457SAndroid Build Coastguard Worker           const intptr_t diff = *dmap++;
124*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t vi0123 = vld1q_f32(input);
125*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t vi4567 = vld1q_f32(input + 4);
126*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t vi89AB = vld1q_f32(input + 8);
127*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t viCDEF = vld1q_f32(input + 12);
128*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t viGHIJ = vld1q_f32(input + 16);
129*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t viKLMN = vld1q_f32(input + 20);
130*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t viOPQR = vld1q_f32(input + 24);
131*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t viSTUV = vld1q_f32(input + 28);
132*4bdc9457SAndroid Build Coastguard Worker           input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
133*4bdc9457SAndroid Build Coastguard Worker           __builtin_prefetch(input + 16);
134*4bdc9457SAndroid Build Coastguard Worker           __builtin_prefetch(input + 32);
135*4bdc9457SAndroid Build Coastguard Worker           const float32x4_t vw = vld1q_dup_f32(w); w += 1;
136*4bdc9457SAndroid Build Coastguard Worker           __builtin_prefetch(w + 32);
137*4bdc9457SAndroid Build Coastguard Worker           vacc0123 = vmlaq_f32(vacc0123, vi0123, vw);
138*4bdc9457SAndroid Build Coastguard Worker           vacc4567 = vmlaq_f32(vacc4567, vi4567, vw);
139*4bdc9457SAndroid Build Coastguard Worker           vacc89AB = vmlaq_f32(vacc89AB, vi89AB, vw);
140*4bdc9457SAndroid Build Coastguard Worker           vaccCDEF = vmlaq_f32(vaccCDEF, viCDEF, vw);
141*4bdc9457SAndroid Build Coastguard Worker           vaccGHIJ = vmlaq_f32(vaccGHIJ, viGHIJ, vw);
142*4bdc9457SAndroid Build Coastguard Worker           vaccKLMN = vmlaq_f32(vaccKLMN, viKLMN, vw);
143*4bdc9457SAndroid Build Coastguard Worker           vaccOPQR = vmlaq_f32(vaccOPQR, viOPQR, vw);
144*4bdc9457SAndroid Build Coastguard Worker           vaccSTUV = vmlaq_f32(vaccSTUV, viSTUV, vw);
145*4bdc9457SAndroid Build Coastguard Worker         } while (--nnz != 0);
146*4bdc9457SAndroid Build Coastguard Worker       }
147*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
148*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
149*4bdc9457SAndroid Build Coastguard Worker       float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
150*4bdc9457SAndroid Build Coastguard Worker       float32x4_t voutCDEF = vminq_f32(vaccCDEF, vmax);
151*4bdc9457SAndroid Build Coastguard Worker       float32x4_t voutGHIJ = vminq_f32(vaccGHIJ, vmax);
152*4bdc9457SAndroid Build Coastguard Worker       float32x4_t voutKLMN = vminq_f32(vaccKLMN, vmax);
153*4bdc9457SAndroid Build Coastguard Worker       float32x4_t voutOPQR = vminq_f32(vaccOPQR, vmax);
154*4bdc9457SAndroid Build Coastguard Worker       float32x4_t voutSTUV = vminq_f32(vaccSTUV, vmax);
155*4bdc9457SAndroid Build Coastguard Worker       vout0123 = vmaxq_f32(vout0123, vmin);
156*4bdc9457SAndroid Build Coastguard Worker       vout4567 = vmaxq_f32(vout4567, vmin);
157*4bdc9457SAndroid Build Coastguard Worker       vout89AB = vmaxq_f32(vout89AB, vmin);
158*4bdc9457SAndroid Build Coastguard Worker       voutCDEF = vmaxq_f32(voutCDEF, vmin);
159*4bdc9457SAndroid Build Coastguard Worker       voutGHIJ = vmaxq_f32(voutGHIJ, vmin);
160*4bdc9457SAndroid Build Coastguard Worker       voutKLMN = vmaxq_f32(voutKLMN, vmin);
161*4bdc9457SAndroid Build Coastguard Worker       voutOPQR = vmaxq_f32(voutOPQR, vmin);
162*4bdc9457SAndroid Build Coastguard Worker       voutSTUV = vmaxq_f32(voutSTUV, vmin);
163*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output, vout0123);
164*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 4, vout4567);
165*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 8, vout89AB);
166*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 12, voutCDEF);
167*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 16, voutGHIJ);
168*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 20, voutKLMN);
169*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 24, voutOPQR);
170*4bdc9457SAndroid Build Coastguard Worker       vst1q_f32(output + 28, voutSTUV);
171*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output + output_stride);
172*4bdc9457SAndroid Build Coastguard Worker     } while (--n != 0);
173*4bdc9457SAndroid Build Coastguard Worker     output = (float*restrict) ((uintptr_t) output - output_decrement);
174*4bdc9457SAndroid Build Coastguard Worker     input += 32;
175*4bdc9457SAndroid Build Coastguard Worker     mc -= 32 * sizeof(float);
176*4bdc9457SAndroid Build Coastguard Worker   }
177*4bdc9457SAndroid Build Coastguard Worker   if XNN_UNLIKELY(mc != 0) {
178*4bdc9457SAndroid Build Coastguard Worker     output_decrement += 16 * sizeof(float);
179*4bdc9457SAndroid Build Coastguard Worker     if (mc & (16 * sizeof(float))) {
180*4bdc9457SAndroid Build Coastguard Worker       const float*restrict w = weights;
181*4bdc9457SAndroid Build Coastguard Worker       const int32_t* dmap = widx_dmap;
182*4bdc9457SAndroid Build Coastguard Worker       const uint32_t* nnzmap = nidx_nnzmap;
183*4bdc9457SAndroid Build Coastguard Worker       size_t n = nc;
184*4bdc9457SAndroid Build Coastguard Worker       do {
185*4bdc9457SAndroid Build Coastguard Worker         uint32_t nnz = *nnzmap++;
186*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
187*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc4567 = vacc0123;
188*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc89AB = vacc0123;
189*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vaccCDEF = vacc0123;
190*4bdc9457SAndroid Build Coastguard Worker         if XNN_LIKELY(nnz != 0) {
191*4bdc9457SAndroid Build Coastguard Worker           do {
192*4bdc9457SAndroid Build Coastguard Worker             const intptr_t diff = *dmap++;
193*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi0123 = vld1q_f32(input);
194*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi4567 = vld1q_f32(input + 4);
195*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi89AB = vld1q_f32(input + 8);
196*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t viCDEF = vld1q_f32(input + 12);
197*4bdc9457SAndroid Build Coastguard Worker             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
198*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vw = vld1q_dup_f32(w); w += 1;
199*4bdc9457SAndroid Build Coastguard Worker             vacc0123 = vmlaq_f32(vacc0123, vi0123, vw);
200*4bdc9457SAndroid Build Coastguard Worker             vacc4567 = vmlaq_f32(vacc4567, vi4567, vw);
201*4bdc9457SAndroid Build Coastguard Worker             vacc89AB = vmlaq_f32(vacc89AB, vi89AB, vw);
202*4bdc9457SAndroid Build Coastguard Worker             vaccCDEF = vmlaq_f32(vaccCDEF, viCDEF, vw);
203*4bdc9457SAndroid Build Coastguard Worker           } while (--nnz != 0);
204*4bdc9457SAndroid Build Coastguard Worker         }
205*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
206*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
207*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout89AB = vminq_f32(vacc89AB, vmax);
208*4bdc9457SAndroid Build Coastguard Worker         float32x4_t voutCDEF = vminq_f32(vaccCDEF, vmax);
209*4bdc9457SAndroid Build Coastguard Worker         vout0123 = vmaxq_f32(vout0123, vmin);
210*4bdc9457SAndroid Build Coastguard Worker         vout4567 = vmaxq_f32(vout4567, vmin);
211*4bdc9457SAndroid Build Coastguard Worker         vout89AB = vmaxq_f32(vout89AB, vmin);
212*4bdc9457SAndroid Build Coastguard Worker         voutCDEF = vmaxq_f32(voutCDEF, vmin);
213*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output, vout0123);
214*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output + 4, vout4567);
215*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output + 8, vout89AB);
216*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output + 12, voutCDEF);
217*4bdc9457SAndroid Build Coastguard Worker         output = (float*restrict) ((uintptr_t) output + output_stride);
218*4bdc9457SAndroid Build Coastguard Worker       } while (--n != 0);
219*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output - output_decrement);
220*4bdc9457SAndroid Build Coastguard Worker       input += 16;
221*4bdc9457SAndroid Build Coastguard Worker     }
222*4bdc9457SAndroid Build Coastguard Worker     output_decrement += 8 * sizeof(float);
223*4bdc9457SAndroid Build Coastguard Worker     if (mc & (8 * sizeof(float))) {
224*4bdc9457SAndroid Build Coastguard Worker       const float*restrict w = weights;
225*4bdc9457SAndroid Build Coastguard Worker       const int32_t* dmap = widx_dmap;
226*4bdc9457SAndroid Build Coastguard Worker       const uint32_t* nnzmap = nidx_nnzmap;
227*4bdc9457SAndroid Build Coastguard Worker       size_t n = nc;
228*4bdc9457SAndroid Build Coastguard Worker       do {
229*4bdc9457SAndroid Build Coastguard Worker         uint32_t nnz = *nnzmap++;
230*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
231*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc4567 = vacc0123;
232*4bdc9457SAndroid Build Coastguard Worker         if XNN_LIKELY(nnz != 0) {
233*4bdc9457SAndroid Build Coastguard Worker           do {
234*4bdc9457SAndroid Build Coastguard Worker             const intptr_t diff = *dmap++;
235*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi0123 = vld1q_f32(input);
236*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi4567 = vld1q_f32(input + 4);
237*4bdc9457SAndroid Build Coastguard Worker             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
238*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vw = vld1q_dup_f32(w); w += 1;
239*4bdc9457SAndroid Build Coastguard Worker             vacc0123 = vmlaq_f32(vacc0123, vi0123, vw);
240*4bdc9457SAndroid Build Coastguard Worker             vacc4567 = vmlaq_f32(vacc4567, vi4567, vw);
241*4bdc9457SAndroid Build Coastguard Worker           } while (--nnz != 0);
242*4bdc9457SAndroid Build Coastguard Worker         }
243*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
244*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout4567 = vminq_f32(vacc4567, vmax);
245*4bdc9457SAndroid Build Coastguard Worker         vout0123 = vmaxq_f32(vout0123, vmin);
246*4bdc9457SAndroid Build Coastguard Worker         vout4567 = vmaxq_f32(vout4567, vmin);
247*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output, vout0123);
248*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output + 4, vout4567);
249*4bdc9457SAndroid Build Coastguard Worker         output = (float*restrict) ((uintptr_t) output + output_stride);
250*4bdc9457SAndroid Build Coastguard Worker       } while (--n != 0);
251*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output - output_decrement);
252*4bdc9457SAndroid Build Coastguard Worker       input += 8;
253*4bdc9457SAndroid Build Coastguard Worker     }
254*4bdc9457SAndroid Build Coastguard Worker     output_decrement += 4 * sizeof(float);
255*4bdc9457SAndroid Build Coastguard Worker     if (mc & (4 * sizeof(float))) {
256*4bdc9457SAndroid Build Coastguard Worker       const float*restrict w = weights;
257*4bdc9457SAndroid Build Coastguard Worker       const int32_t* dmap = widx_dmap;
258*4bdc9457SAndroid Build Coastguard Worker       const uint32_t* nnzmap = nidx_nnzmap;
259*4bdc9457SAndroid Build Coastguard Worker       size_t n = nc;
260*4bdc9457SAndroid Build Coastguard Worker       do {
261*4bdc9457SAndroid Build Coastguard Worker         uint32_t nnz = *nnzmap++;
262*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vacc0123 = vld1q_dup_f32(w); w += 1;
263*4bdc9457SAndroid Build Coastguard Worker         if XNN_LIKELY(nnz != 0) {
264*4bdc9457SAndroid Build Coastguard Worker           do {
265*4bdc9457SAndroid Build Coastguard Worker             const intptr_t diff = *dmap++;
266*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vi0123 = vld1q_f32(input);
267*4bdc9457SAndroid Build Coastguard Worker             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
268*4bdc9457SAndroid Build Coastguard Worker             const float32x4_t vw = vld1q_dup_f32(w); w += 1;
269*4bdc9457SAndroid Build Coastguard Worker             vacc0123 = vmlaq_f32(vacc0123, vi0123, vw);
270*4bdc9457SAndroid Build Coastguard Worker           } while (--nnz != 0);
271*4bdc9457SAndroid Build Coastguard Worker         }
272*4bdc9457SAndroid Build Coastguard Worker         float32x4_t vout0123 = vminq_f32(vacc0123, vmax);
273*4bdc9457SAndroid Build Coastguard Worker         vout0123 = vmaxq_f32(vout0123, vmin);
274*4bdc9457SAndroid Build Coastguard Worker         vst1q_f32(output, vout0123);
275*4bdc9457SAndroid Build Coastguard Worker         output = (float*restrict) ((uintptr_t) output + output_stride);
276*4bdc9457SAndroid Build Coastguard Worker       } while (--n != 0);
277*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output - output_decrement);
278*4bdc9457SAndroid Build Coastguard Worker       input += 4;
279*4bdc9457SAndroid Build Coastguard Worker     }
280*4bdc9457SAndroid Build Coastguard Worker     output_decrement += 2 * sizeof(float);
281*4bdc9457SAndroid Build Coastguard Worker     if (mc & (2 * sizeof(float))) {
282*4bdc9457SAndroid Build Coastguard Worker       const float*restrict w = weights;
283*4bdc9457SAndroid Build Coastguard Worker       const int32_t* dmap = widx_dmap;
284*4bdc9457SAndroid Build Coastguard Worker       const uint32_t* nnzmap = nidx_nnzmap;
285*4bdc9457SAndroid Build Coastguard Worker       size_t n = nc;
286*4bdc9457SAndroid Build Coastguard Worker       do {
287*4bdc9457SAndroid Build Coastguard Worker         uint32_t nnz = *nnzmap++;
288*4bdc9457SAndroid Build Coastguard Worker         float32x2_t vacc01 = vld1_dup_f32(w); w += 1;
289*4bdc9457SAndroid Build Coastguard Worker         if XNN_LIKELY(nnz != 0) {
290*4bdc9457SAndroid Build Coastguard Worker           do {
291*4bdc9457SAndroid Build Coastguard Worker             const intptr_t diff = *dmap++;
292*4bdc9457SAndroid Build Coastguard Worker             const float32x2_t vi01 = vld1_f32(input);
293*4bdc9457SAndroid Build Coastguard Worker             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
294*4bdc9457SAndroid Build Coastguard Worker             const float32x2_t vw = vld1_dup_f32(w); w += 1;
295*4bdc9457SAndroid Build Coastguard Worker             vacc01 = vmla_f32(vacc01, vi01, vw);
296*4bdc9457SAndroid Build Coastguard Worker           } while (--nnz != 0);
297*4bdc9457SAndroid Build Coastguard Worker         }
298*4bdc9457SAndroid Build Coastguard Worker         float32x2_t vout01 = vmin_f32(vacc01, vget_low_f32(vmax));
299*4bdc9457SAndroid Build Coastguard Worker         vout01 = vmax_f32(vout01, vget_low_f32(vmin));
300*4bdc9457SAndroid Build Coastguard Worker         vst1_f32(output, vout01);
301*4bdc9457SAndroid Build Coastguard Worker         output = (float*restrict) ((uintptr_t) output + output_stride);
302*4bdc9457SAndroid Build Coastguard Worker       } while (--n != 0);
303*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output - output_decrement);
304*4bdc9457SAndroid Build Coastguard Worker       input += 2;
305*4bdc9457SAndroid Build Coastguard Worker     }
306*4bdc9457SAndroid Build Coastguard Worker     output_decrement += 1 * sizeof(float);
307*4bdc9457SAndroid Build Coastguard Worker     if (mc & (1 * sizeof(float))) {
308*4bdc9457SAndroid Build Coastguard Worker       const float*restrict w = weights;
309*4bdc9457SAndroid Build Coastguard Worker       const int32_t* dmap = widx_dmap;
310*4bdc9457SAndroid Build Coastguard Worker       const uint32_t* nnzmap = nidx_nnzmap;
311*4bdc9457SAndroid Build Coastguard Worker       size_t n = nc;
312*4bdc9457SAndroid Build Coastguard Worker       do {
313*4bdc9457SAndroid Build Coastguard Worker         uint32_t nnz = *nnzmap++;
314*4bdc9457SAndroid Build Coastguard Worker         float32x2_t vacc0 = vld1_dup_f32(w); w += 1;
315*4bdc9457SAndroid Build Coastguard Worker         if XNN_LIKELY(nnz != 0) {
316*4bdc9457SAndroid Build Coastguard Worker           do {
317*4bdc9457SAndroid Build Coastguard Worker             const intptr_t diff = *dmap++;
318*4bdc9457SAndroid Build Coastguard Worker             const float32x2_t vi0 = vld1_dup_f32(input);
319*4bdc9457SAndroid Build Coastguard Worker             input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
320*4bdc9457SAndroid Build Coastguard Worker             const float32x2_t vw = vld1_dup_f32(w); w += 1;
321*4bdc9457SAndroid Build Coastguard Worker             vacc0 = vmla_f32(vacc0, vi0, vw);
322*4bdc9457SAndroid Build Coastguard Worker           } while (--nnz != 0);
323*4bdc9457SAndroid Build Coastguard Worker         }
324*4bdc9457SAndroid Build Coastguard Worker         float32x2_t vout0 = vmin_f32(vacc0, vget_low_f32(vmax));
325*4bdc9457SAndroid Build Coastguard Worker         vout0 = vmax_f32(vout0, vget_low_f32(vmin));
326*4bdc9457SAndroid Build Coastguard Worker         vst1_lane_f32(output, vout0, 0);
327*4bdc9457SAndroid Build Coastguard Worker         output = (float*restrict) ((uintptr_t) output + output_stride);
328*4bdc9457SAndroid Build Coastguard Worker       } while (--n != 0);
329*4bdc9457SAndroid Build Coastguard Worker       output = (float*restrict) ((uintptr_t) output - output_decrement);
330*4bdc9457SAndroid Build Coastguard Worker       input += 1;
331*4bdc9457SAndroid Build Coastguard Worker     }
332*4bdc9457SAndroid Build Coastguard Worker   }
333*4bdc9457SAndroid Build Coastguard Worker }
334