1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f32-spmm/wasmsimd.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2020 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <wasm_simd128.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/spmm.h>
15*4bdc9457SAndroid Build Coastguard Worker
16*4bdc9457SAndroid Build Coastguard Worker
xnn_f32_spmm_minmax_ukernel_16x1__wasmsimd_x86_x4(size_t mc,size_t nc,const float * restrict input,const float * restrict weights,const int32_t * restrict widx_dmap,const uint32_t * restrict nidx_nnzmap,float * restrict output,size_t output_stride,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_spmm_minmax_ukernel_16x1__wasmsimd_x86_x4(
18*4bdc9457SAndroid Build Coastguard Worker size_t mc,
19*4bdc9457SAndroid Build Coastguard Worker size_t nc,
20*4bdc9457SAndroid Build Coastguard Worker const float*restrict input,
21*4bdc9457SAndroid Build Coastguard Worker const float*restrict weights,
22*4bdc9457SAndroid Build Coastguard Worker const int32_t*restrict widx_dmap,
23*4bdc9457SAndroid Build Coastguard Worker const uint32_t*restrict nidx_nnzmap,
24*4bdc9457SAndroid Build Coastguard Worker float*restrict output,
25*4bdc9457SAndroid Build Coastguard Worker size_t output_stride,
26*4bdc9457SAndroid Build Coastguard Worker const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
27*4bdc9457SAndroid Build Coastguard Worker {
28*4bdc9457SAndroid Build Coastguard Worker assert(mc != 0);
29*4bdc9457SAndroid Build Coastguard Worker assert(mc % sizeof(float) == 0);
30*4bdc9457SAndroid Build Coastguard Worker assert(nc != 0);
31*4bdc9457SAndroid Build Coastguard Worker
32*4bdc9457SAndroid Build Coastguard Worker const v128_t vmin = wasm_v128_load64_splat(params->wasmsimd.min);
33*4bdc9457SAndroid Build Coastguard Worker const v128_t vmax = wasm_v128_load64_splat(params->wasmsimd.max);
34*4bdc9457SAndroid Build Coastguard Worker size_t output_decrement = output_stride * nc - 16 * sizeof(float);
35*4bdc9457SAndroid Build Coastguard Worker while XNN_LIKELY(mc >= 16 * sizeof(float)) {
36*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
37*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
38*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
39*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
40*4bdc9457SAndroid Build Coastguard Worker do {
41*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
42*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123x0 = wasm_v128_load32_splat(w);
43*4bdc9457SAndroid Build Coastguard Worker w += 1;
44*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123x1 = wasm_f32x4_const_splat(0.0f);
45*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123x2 = wasm_f32x4_const_splat(0.0f);
46*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123x3 = wasm_f32x4_const_splat(0.0f);
47*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567x0 = vacc0123x0;
48*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567x1 = wasm_f32x4_const_splat(0.0f);
49*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567x2 = wasm_f32x4_const_splat(0.0f);
50*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567x3 = wasm_f32x4_const_splat(0.0f);
51*4bdc9457SAndroid Build Coastguard Worker v128_t vacc89ABx0 = vacc0123x0;
52*4bdc9457SAndroid Build Coastguard Worker v128_t vacc89ABx1 = wasm_f32x4_const_splat(0.0f);
53*4bdc9457SAndroid Build Coastguard Worker v128_t vacc89ABx2 = wasm_f32x4_const_splat(0.0f);
54*4bdc9457SAndroid Build Coastguard Worker v128_t vacc89ABx3 = wasm_f32x4_const_splat(0.0f);
55*4bdc9457SAndroid Build Coastguard Worker v128_t vaccCDEFx0 = vacc0123x0;
56*4bdc9457SAndroid Build Coastguard Worker v128_t vaccCDEFx1 = wasm_f32x4_const_splat(0.0f);
57*4bdc9457SAndroid Build Coastguard Worker v128_t vaccCDEFx2 = wasm_f32x4_const_splat(0.0f);
58*4bdc9457SAndroid Build Coastguard Worker v128_t vaccCDEFx3 = wasm_f32x4_const_splat(0.0f);
59*4bdc9457SAndroid Build Coastguard Worker for (; nnz >= 4; nnz -= 4) {
60*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff0 = dmap[0];
61*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff1 = dmap[1];
62*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff2 = dmap[2];
63*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff3 = dmap[3];
64*4bdc9457SAndroid Build Coastguard Worker dmap += 4;
65*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123x0 = wasm_v128_load(input);
66*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567x0 = wasm_v128_load(input + 4);
67*4bdc9457SAndroid Build Coastguard Worker const v128_t vi89ABx0 = wasm_v128_load(input + 8);
68*4bdc9457SAndroid Build Coastguard Worker const v128_t viCDEFx0 = wasm_v128_load(input + 12);
69*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff0);
70*4bdc9457SAndroid Build Coastguard Worker const v128_t vw0 = wasm_v128_load32_splat(w);
71*4bdc9457SAndroid Build Coastguard Worker w += 1;
72*4bdc9457SAndroid Build Coastguard Worker vacc0123x0 = wasm_f32x4_add(vacc0123x0, wasm_f32x4_mul(vi0123x0, vw0));
73*4bdc9457SAndroid Build Coastguard Worker vacc4567x0 = wasm_f32x4_add(vacc4567x0, wasm_f32x4_mul(vi4567x0, vw0));
74*4bdc9457SAndroid Build Coastguard Worker vacc89ABx0 = wasm_f32x4_add(vacc89ABx0, wasm_f32x4_mul(vi89ABx0, vw0));
75*4bdc9457SAndroid Build Coastguard Worker vaccCDEFx0 = wasm_f32x4_add(vaccCDEFx0, wasm_f32x4_mul(viCDEFx0, vw0));
76*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123x1 = wasm_v128_load(input);
77*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567x1 = wasm_v128_load(input + 4);
78*4bdc9457SAndroid Build Coastguard Worker const v128_t vi89ABx1 = wasm_v128_load(input + 8);
79*4bdc9457SAndroid Build Coastguard Worker const v128_t viCDEFx1 = wasm_v128_load(input + 12);
80*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff1);
81*4bdc9457SAndroid Build Coastguard Worker const v128_t vw1 = wasm_v128_load32_splat(w);
82*4bdc9457SAndroid Build Coastguard Worker w += 1;
83*4bdc9457SAndroid Build Coastguard Worker vacc0123x1 = wasm_f32x4_add(vacc0123x1, wasm_f32x4_mul(vi0123x1, vw1));
84*4bdc9457SAndroid Build Coastguard Worker vacc4567x1 = wasm_f32x4_add(vacc4567x1, wasm_f32x4_mul(vi4567x1, vw1));
85*4bdc9457SAndroid Build Coastguard Worker vacc89ABx1 = wasm_f32x4_add(vacc89ABx1, wasm_f32x4_mul(vi89ABx1, vw1));
86*4bdc9457SAndroid Build Coastguard Worker vaccCDEFx1 = wasm_f32x4_add(vaccCDEFx1, wasm_f32x4_mul(viCDEFx1, vw1));
87*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123x2 = wasm_v128_load(input);
88*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567x2 = wasm_v128_load(input + 4);
89*4bdc9457SAndroid Build Coastguard Worker const v128_t vi89ABx2 = wasm_v128_load(input + 8);
90*4bdc9457SAndroid Build Coastguard Worker const v128_t viCDEFx2 = wasm_v128_load(input + 12);
91*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff2);
92*4bdc9457SAndroid Build Coastguard Worker const v128_t vw2 = wasm_v128_load32_splat(w);
93*4bdc9457SAndroid Build Coastguard Worker w += 1;
94*4bdc9457SAndroid Build Coastguard Worker vacc0123x2 = wasm_f32x4_add(vacc0123x2, wasm_f32x4_mul(vi0123x2, vw2));
95*4bdc9457SAndroid Build Coastguard Worker vacc4567x2 = wasm_f32x4_add(vacc4567x2, wasm_f32x4_mul(vi4567x2, vw2));
96*4bdc9457SAndroid Build Coastguard Worker vacc89ABx2 = wasm_f32x4_add(vacc89ABx2, wasm_f32x4_mul(vi89ABx2, vw2));
97*4bdc9457SAndroid Build Coastguard Worker vaccCDEFx2 = wasm_f32x4_add(vaccCDEFx2, wasm_f32x4_mul(viCDEFx2, vw2));
98*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123x3 = wasm_v128_load(input);
99*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567x3 = wasm_v128_load(input + 4);
100*4bdc9457SAndroid Build Coastguard Worker const v128_t vi89ABx3 = wasm_v128_load(input + 8);
101*4bdc9457SAndroid Build Coastguard Worker const v128_t viCDEFx3 = wasm_v128_load(input + 12);
102*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff3);
103*4bdc9457SAndroid Build Coastguard Worker const v128_t vw3 = wasm_v128_load32_splat(w);
104*4bdc9457SAndroid Build Coastguard Worker w += 1;
105*4bdc9457SAndroid Build Coastguard Worker vacc0123x3 = wasm_f32x4_add(vacc0123x3, wasm_f32x4_mul(vi0123x3, vw3));
106*4bdc9457SAndroid Build Coastguard Worker vacc4567x3 = wasm_f32x4_add(vacc4567x3, wasm_f32x4_mul(vi4567x3, vw3));
107*4bdc9457SAndroid Build Coastguard Worker vacc89ABx3 = wasm_f32x4_add(vacc89ABx3, wasm_f32x4_mul(vi89ABx3, vw3));
108*4bdc9457SAndroid Build Coastguard Worker vaccCDEFx3 = wasm_f32x4_add(vaccCDEFx3, wasm_f32x4_mul(viCDEFx3, vw3));
109*4bdc9457SAndroid Build Coastguard Worker }
110*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123 = vacc0123x0;
111*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567 = vacc4567x0;
112*4bdc9457SAndroid Build Coastguard Worker v128_t vacc89AB = vacc89ABx0;
113*4bdc9457SAndroid Build Coastguard Worker v128_t vaccCDEF = vaccCDEFx0;
114*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, vacc0123x1);
115*4bdc9457SAndroid Build Coastguard Worker vacc4567 = wasm_f32x4_add(vacc4567, vacc4567x1);
116*4bdc9457SAndroid Build Coastguard Worker vacc89AB = wasm_f32x4_add(vacc89AB, vacc89ABx1);
117*4bdc9457SAndroid Build Coastguard Worker vaccCDEF = wasm_f32x4_add(vaccCDEF, vaccCDEFx1);
118*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, vacc0123x2);
119*4bdc9457SAndroid Build Coastguard Worker vacc4567 = wasm_f32x4_add(vacc4567, vacc4567x2);
120*4bdc9457SAndroid Build Coastguard Worker vacc89AB = wasm_f32x4_add(vacc89AB, vacc89ABx2);
121*4bdc9457SAndroid Build Coastguard Worker vaccCDEF = wasm_f32x4_add(vaccCDEF, vaccCDEFx2);
122*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, vacc0123x3);
123*4bdc9457SAndroid Build Coastguard Worker vacc4567 = wasm_f32x4_add(vacc4567, vacc4567x3);
124*4bdc9457SAndroid Build Coastguard Worker vacc89AB = wasm_f32x4_add(vacc89AB, vacc89ABx3);
125*4bdc9457SAndroid Build Coastguard Worker vaccCDEF = wasm_f32x4_add(vaccCDEF, vaccCDEFx3);
126*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
127*4bdc9457SAndroid Build Coastguard Worker do {
128*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
129*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123 = wasm_v128_load(input);
130*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567 = wasm_v128_load(input + 4);
131*4bdc9457SAndroid Build Coastguard Worker const v128_t vi89AB = wasm_v128_load(input + 8);
132*4bdc9457SAndroid Build Coastguard Worker const v128_t viCDEF = wasm_v128_load(input + 12);
133*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
134*4bdc9457SAndroid Build Coastguard Worker const v128_t vw = wasm_v128_load32_splat(w); w += 1;
135*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
136*4bdc9457SAndroid Build Coastguard Worker vacc4567 = wasm_f32x4_add(vacc4567, wasm_f32x4_mul(vi4567, vw));
137*4bdc9457SAndroid Build Coastguard Worker vacc89AB = wasm_f32x4_add(vacc89AB, wasm_f32x4_mul(vi89AB, vw));
138*4bdc9457SAndroid Build Coastguard Worker vaccCDEF = wasm_f32x4_add(vaccCDEF, wasm_f32x4_mul(viCDEF, vw));
139*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
140*4bdc9457SAndroid Build Coastguard Worker }
141*4bdc9457SAndroid Build Coastguard Worker v128_t vout0123 = wasm_f32x4_pmin(vmax, vacc0123);
142*4bdc9457SAndroid Build Coastguard Worker v128_t vout4567 = wasm_f32x4_pmin(vmax, vacc4567);
143*4bdc9457SAndroid Build Coastguard Worker v128_t vout89AB = wasm_f32x4_pmin(vmax, vacc89AB);
144*4bdc9457SAndroid Build Coastguard Worker v128_t voutCDEF = wasm_f32x4_pmin(vmax, vaccCDEF);
145*4bdc9457SAndroid Build Coastguard Worker vout0123 = wasm_f32x4_pmax(vmin, vout0123);
146*4bdc9457SAndroid Build Coastguard Worker vout4567 = wasm_f32x4_pmax(vmin, vout4567);
147*4bdc9457SAndroid Build Coastguard Worker vout89AB = wasm_f32x4_pmax(vmin, vout89AB);
148*4bdc9457SAndroid Build Coastguard Worker voutCDEF = wasm_f32x4_pmax(vmin, voutCDEF);
149*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output, vout0123);
150*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output + 4, vout4567);
151*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output + 8, vout89AB);
152*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output + 12, voutCDEF);
153*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
154*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
155*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
156*4bdc9457SAndroid Build Coastguard Worker input += 16;
157*4bdc9457SAndroid Build Coastguard Worker mc -= 16 * sizeof(float);
158*4bdc9457SAndroid Build Coastguard Worker }
159*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(mc != 0) {
160*4bdc9457SAndroid Build Coastguard Worker output_decrement += 8 * sizeof(float);
161*4bdc9457SAndroid Build Coastguard Worker if (mc & (8 * sizeof(float))) {
162*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
163*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
164*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
165*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
166*4bdc9457SAndroid Build Coastguard Worker do {
167*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
168*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123 = wasm_v128_load32_splat(w); w += 1;
169*4bdc9457SAndroid Build Coastguard Worker v128_t vacc4567 = vacc0123;
170*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
171*4bdc9457SAndroid Build Coastguard Worker do {
172*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
173*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123 = wasm_v128_load(input);
174*4bdc9457SAndroid Build Coastguard Worker const v128_t vi4567 = wasm_v128_load(input + 4);
175*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
176*4bdc9457SAndroid Build Coastguard Worker const v128_t vw = wasm_v128_load32_splat(w); w += 1;
177*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
178*4bdc9457SAndroid Build Coastguard Worker vacc4567 = wasm_f32x4_add(vacc4567, wasm_f32x4_mul(vi4567, vw));
179*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
180*4bdc9457SAndroid Build Coastguard Worker }
181*4bdc9457SAndroid Build Coastguard Worker v128_t vout0123 = wasm_f32x4_pmin(vmax, vacc0123);
182*4bdc9457SAndroid Build Coastguard Worker v128_t vout4567 = wasm_f32x4_pmin(vmax, vacc4567);
183*4bdc9457SAndroid Build Coastguard Worker vout0123 = wasm_f32x4_pmax(vmin, vout0123);
184*4bdc9457SAndroid Build Coastguard Worker vout4567 = wasm_f32x4_pmax(vmin, vout4567);
185*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output, vout0123);
186*4bdc9457SAndroid Build Coastguard Worker
187*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output + 4, vout4567);
188*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
189*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
190*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
191*4bdc9457SAndroid Build Coastguard Worker input += 8;
192*4bdc9457SAndroid Build Coastguard Worker }
193*4bdc9457SAndroid Build Coastguard Worker output_decrement += 4 * sizeof(float);
194*4bdc9457SAndroid Build Coastguard Worker if (mc & (4 * sizeof(float))) {
195*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
196*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
197*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
198*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
199*4bdc9457SAndroid Build Coastguard Worker do {
200*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
201*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0123 = wasm_v128_load32_splat(w); w += 1;
202*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
203*4bdc9457SAndroid Build Coastguard Worker do {
204*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
205*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0123 = wasm_v128_load(input);
206*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
207*4bdc9457SAndroid Build Coastguard Worker const v128_t vw = wasm_v128_load32_splat(w); w += 1;
208*4bdc9457SAndroid Build Coastguard Worker vacc0123 = wasm_f32x4_add(vacc0123, wasm_f32x4_mul(vi0123, vw));
209*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
210*4bdc9457SAndroid Build Coastguard Worker }
211*4bdc9457SAndroid Build Coastguard Worker v128_t vout0123 = wasm_f32x4_pmin(vmax, vacc0123);
212*4bdc9457SAndroid Build Coastguard Worker vout0123 = wasm_f32x4_pmax(vmin, vout0123);
213*4bdc9457SAndroid Build Coastguard Worker wasm_v128_store(output, vout0123);
214*4bdc9457SAndroid Build Coastguard Worker
215*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
216*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
217*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
218*4bdc9457SAndroid Build Coastguard Worker input += 4;
219*4bdc9457SAndroid Build Coastguard Worker }
220*4bdc9457SAndroid Build Coastguard Worker output_decrement += 2 * sizeof(float);
221*4bdc9457SAndroid Build Coastguard Worker if (mc & (2 * sizeof(float))) {
222*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
223*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
224*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
225*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
226*4bdc9457SAndroid Build Coastguard Worker do {
227*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
228*4bdc9457SAndroid Build Coastguard Worker v128_t vacc01 = wasm_v128_load32_splat(w); w += 1;
229*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
230*4bdc9457SAndroid Build Coastguard Worker do {
231*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
232*4bdc9457SAndroid Build Coastguard Worker const v128_t vi01 = wasm_v128_load64_splat(input);
233*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
234*4bdc9457SAndroid Build Coastguard Worker const v128_t vw = wasm_v128_load32_splat(w); w += 1;
235*4bdc9457SAndroid Build Coastguard Worker vacc01 = wasm_f32x4_add(vacc01, wasm_f32x4_mul(vi01, vw));
236*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
237*4bdc9457SAndroid Build Coastguard Worker }
238*4bdc9457SAndroid Build Coastguard Worker v128_t vout01 = wasm_f32x4_pmin(vmax, vacc01);
239*4bdc9457SAndroid Build Coastguard Worker vout01 = wasm_f32x4_pmax(vmin, vout01);
240*4bdc9457SAndroid Build Coastguard Worker *((double*) output) = wasm_f64x2_extract_lane(vout01, 0);
241*4bdc9457SAndroid Build Coastguard Worker
242*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
243*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
244*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
245*4bdc9457SAndroid Build Coastguard Worker input += 2;
246*4bdc9457SAndroid Build Coastguard Worker }
247*4bdc9457SAndroid Build Coastguard Worker output_decrement += 1 * sizeof(float);
248*4bdc9457SAndroid Build Coastguard Worker if (mc & (1 * sizeof(float))) {
249*4bdc9457SAndroid Build Coastguard Worker const float*restrict w = weights;
250*4bdc9457SAndroid Build Coastguard Worker const int32_t* dmap = widx_dmap;
251*4bdc9457SAndroid Build Coastguard Worker const uint32_t* nnzmap = nidx_nnzmap;
252*4bdc9457SAndroid Build Coastguard Worker size_t n = nc;
253*4bdc9457SAndroid Build Coastguard Worker do {
254*4bdc9457SAndroid Build Coastguard Worker uint32_t nnz = *nnzmap++;
255*4bdc9457SAndroid Build Coastguard Worker v128_t vacc0 = wasm_v128_load32_splat(w); w += 1;
256*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(nnz != 0) {
257*4bdc9457SAndroid Build Coastguard Worker do {
258*4bdc9457SAndroid Build Coastguard Worker const intptr_t diff = *dmap++;
259*4bdc9457SAndroid Build Coastguard Worker const v128_t vi0 = wasm_v128_load32_splat(input);
260*4bdc9457SAndroid Build Coastguard Worker input = (const float*restrict) ((uintptr_t) input + (uintptr_t) diff);
261*4bdc9457SAndroid Build Coastguard Worker const v128_t vw = wasm_v128_load32_splat(w); w += 1;
262*4bdc9457SAndroid Build Coastguard Worker vacc0 = wasm_f32x4_add(vacc0, wasm_f32x4_mul(vi0, vw));
263*4bdc9457SAndroid Build Coastguard Worker } while (--nnz != 0);
264*4bdc9457SAndroid Build Coastguard Worker }
265*4bdc9457SAndroid Build Coastguard Worker v128_t vout0 = wasm_f32x4_pmin(vmax, vacc0);
266*4bdc9457SAndroid Build Coastguard Worker vout0 = wasm_f32x4_pmax(vmin, vout0);
267*4bdc9457SAndroid Build Coastguard Worker *output = wasm_f32x4_extract_lane(vout0, 0);
268*4bdc9457SAndroid Build Coastguard Worker
269*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output + output_stride);
270*4bdc9457SAndroid Build Coastguard Worker } while (--n != 0);
271*4bdc9457SAndroid Build Coastguard Worker output = (float*restrict) ((uintptr_t) output - output_decrement);
272*4bdc9457SAndroid Build Coastguard Worker input += 1;
273*4bdc9457SAndroid Build Coastguard Worker }
274*4bdc9457SAndroid Build Coastguard Worker }
275*4bdc9457SAndroid Build Coastguard Worker }
276