1*4bdc9457SAndroid Build Coastguard Worker // Auto-generated file. Do not edit!
2*4bdc9457SAndroid Build Coastguard Worker // Template: src/f32-dwconv/up-avx512.c.in
3*4bdc9457SAndroid Build Coastguard Worker // Generator: tools/xngen
4*4bdc9457SAndroid Build Coastguard Worker //
5*4bdc9457SAndroid Build Coastguard Worker // Copyright 2019 Google LLC
6*4bdc9457SAndroid Build Coastguard Worker //
7*4bdc9457SAndroid Build Coastguard Worker // This source code is licensed under the BSD-style license found in the
8*4bdc9457SAndroid Build Coastguard Worker // LICENSE file in the root directory of this source tree.
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker #include <assert.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker #include <immintrin.h>
13*4bdc9457SAndroid Build Coastguard Worker
14*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/dwconv.h>
15*4bdc9457SAndroid Build Coastguard Worker #include <xnnpack/intrinsics-polyfill.h>
16*4bdc9457SAndroid Build Coastguard Worker
17*4bdc9457SAndroid Build Coastguard Worker
xnn_f32_dwconv_minmax_ukernel_up16x25__avx512f_acc2(size_t channels,size_t output_width,const float ** input,const float * weights,float * output,size_t input_stride,size_t output_increment,size_t input_offset,const float * zero,const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])18*4bdc9457SAndroid Build Coastguard Worker void xnn_f32_dwconv_minmax_ukernel_up16x25__avx512f_acc2(
19*4bdc9457SAndroid Build Coastguard Worker size_t channels,
20*4bdc9457SAndroid Build Coastguard Worker size_t output_width,
21*4bdc9457SAndroid Build Coastguard Worker const float** input,
22*4bdc9457SAndroid Build Coastguard Worker const float* weights,
23*4bdc9457SAndroid Build Coastguard Worker float* output,
24*4bdc9457SAndroid Build Coastguard Worker size_t input_stride,
25*4bdc9457SAndroid Build Coastguard Worker size_t output_increment,
26*4bdc9457SAndroid Build Coastguard Worker size_t input_offset,
27*4bdc9457SAndroid Build Coastguard Worker const float* zero,
28*4bdc9457SAndroid Build Coastguard Worker const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
29*4bdc9457SAndroid Build Coastguard Worker {
30*4bdc9457SAndroid Build Coastguard Worker assert(channels != 0);
31*4bdc9457SAndroid Build Coastguard Worker assert(output_width != 0);
32*4bdc9457SAndroid Build Coastguard Worker
33*4bdc9457SAndroid Build Coastguard Worker const __m512 vmax = _mm512_set1_ps(params->scalar.max);
34*4bdc9457SAndroid Build Coastguard Worker const __m512 vmin = _mm512_set1_ps(params->scalar.min);
35*4bdc9457SAndroid Build Coastguard Worker do {
36*4bdc9457SAndroid Build Coastguard Worker const float* i0 = input[0];
37*4bdc9457SAndroid Build Coastguard Worker assert(i0 != NULL);
38*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i0 != zero) {
39*4bdc9457SAndroid Build Coastguard Worker i0 = (const float*) ((uintptr_t) i0 + input_offset);
40*4bdc9457SAndroid Build Coastguard Worker }
41*4bdc9457SAndroid Build Coastguard Worker const float* i1 = input[1];
42*4bdc9457SAndroid Build Coastguard Worker assert(i1 != NULL);
43*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i1 != zero) {
44*4bdc9457SAndroid Build Coastguard Worker i1 = (const float*) ((uintptr_t) i1 + input_offset);
45*4bdc9457SAndroid Build Coastguard Worker }
46*4bdc9457SAndroid Build Coastguard Worker const float* i2 = input[2];
47*4bdc9457SAndroid Build Coastguard Worker assert(i2 != NULL);
48*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i2 != zero) {
49*4bdc9457SAndroid Build Coastguard Worker i2 = (const float*) ((uintptr_t) i2 + input_offset);
50*4bdc9457SAndroid Build Coastguard Worker }
51*4bdc9457SAndroid Build Coastguard Worker const float* i3 = input[3];
52*4bdc9457SAndroid Build Coastguard Worker assert(i3 != NULL);
53*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i3 != zero) {
54*4bdc9457SAndroid Build Coastguard Worker i3 = (const float*) ((uintptr_t) i3 + input_offset);
55*4bdc9457SAndroid Build Coastguard Worker }
56*4bdc9457SAndroid Build Coastguard Worker const float* i4 = input[4];
57*4bdc9457SAndroid Build Coastguard Worker assert(i4 != NULL);
58*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i4 != zero) {
59*4bdc9457SAndroid Build Coastguard Worker i4 = (const float*) ((uintptr_t) i4 + input_offset);
60*4bdc9457SAndroid Build Coastguard Worker }
61*4bdc9457SAndroid Build Coastguard Worker const float* i5 = input[5];
62*4bdc9457SAndroid Build Coastguard Worker assert(i5 != NULL);
63*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i5 != zero) {
64*4bdc9457SAndroid Build Coastguard Worker i5 = (const float*) ((uintptr_t) i5 + input_offset);
65*4bdc9457SAndroid Build Coastguard Worker }
66*4bdc9457SAndroid Build Coastguard Worker const float* i6 = input[6];
67*4bdc9457SAndroid Build Coastguard Worker assert(i6 != NULL);
68*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i6 != zero) {
69*4bdc9457SAndroid Build Coastguard Worker i6 = (const float*) ((uintptr_t) i6 + input_offset);
70*4bdc9457SAndroid Build Coastguard Worker }
71*4bdc9457SAndroid Build Coastguard Worker const float* i7 = input[7];
72*4bdc9457SAndroid Build Coastguard Worker assert(i7 != NULL);
73*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i7 != zero) {
74*4bdc9457SAndroid Build Coastguard Worker i7 = (const float*) ((uintptr_t) i7 + input_offset);
75*4bdc9457SAndroid Build Coastguard Worker }
76*4bdc9457SAndroid Build Coastguard Worker const float* i8 = input[8];
77*4bdc9457SAndroid Build Coastguard Worker assert(i8 != NULL);
78*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i8 != zero) {
79*4bdc9457SAndroid Build Coastguard Worker i8 = (const float*) ((uintptr_t) i8 + input_offset);
80*4bdc9457SAndroid Build Coastguard Worker }
81*4bdc9457SAndroid Build Coastguard Worker const float* i9 = input[9];
82*4bdc9457SAndroid Build Coastguard Worker assert(i9 != NULL);
83*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i9 != zero) {
84*4bdc9457SAndroid Build Coastguard Worker i9 = (const float*) ((uintptr_t) i9 + input_offset);
85*4bdc9457SAndroid Build Coastguard Worker }
86*4bdc9457SAndroid Build Coastguard Worker const float* i10 = input[10];
87*4bdc9457SAndroid Build Coastguard Worker assert(i10 != NULL);
88*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i10 != zero) {
89*4bdc9457SAndroid Build Coastguard Worker i10 = (const float*) ((uintptr_t) i10 + input_offset);
90*4bdc9457SAndroid Build Coastguard Worker }
91*4bdc9457SAndroid Build Coastguard Worker const float* i11 = input[11];
92*4bdc9457SAndroid Build Coastguard Worker assert(i11 != NULL);
93*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i11 != zero) {
94*4bdc9457SAndroid Build Coastguard Worker i11 = (const float*) ((uintptr_t) i11 + input_offset);
95*4bdc9457SAndroid Build Coastguard Worker }
96*4bdc9457SAndroid Build Coastguard Worker const float* i12 = input[12];
97*4bdc9457SAndroid Build Coastguard Worker assert(i12 != NULL);
98*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i12 != zero) {
99*4bdc9457SAndroid Build Coastguard Worker i12 = (const float*) ((uintptr_t) i12 + input_offset);
100*4bdc9457SAndroid Build Coastguard Worker }
101*4bdc9457SAndroid Build Coastguard Worker const float* i13 = input[13];
102*4bdc9457SAndroid Build Coastguard Worker assert(i13 != NULL);
103*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i13 != zero) {
104*4bdc9457SAndroid Build Coastguard Worker i13 = (const float*) ((uintptr_t) i13 + input_offset);
105*4bdc9457SAndroid Build Coastguard Worker }
106*4bdc9457SAndroid Build Coastguard Worker const float* i14 = input[14];
107*4bdc9457SAndroid Build Coastguard Worker assert(i14 != NULL);
108*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i14 != zero) {
109*4bdc9457SAndroid Build Coastguard Worker i14 = (const float*) ((uintptr_t) i14 + input_offset);
110*4bdc9457SAndroid Build Coastguard Worker }
111*4bdc9457SAndroid Build Coastguard Worker const float* i15 = input[15];
112*4bdc9457SAndroid Build Coastguard Worker assert(i15 != NULL);
113*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i15 != zero) {
114*4bdc9457SAndroid Build Coastguard Worker i15 = (const float*) ((uintptr_t) i15 + input_offset);
115*4bdc9457SAndroid Build Coastguard Worker }
116*4bdc9457SAndroid Build Coastguard Worker const float* i16 = input[16];
117*4bdc9457SAndroid Build Coastguard Worker assert(i16 != NULL);
118*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i16 != zero) {
119*4bdc9457SAndroid Build Coastguard Worker i16 = (const float*) ((uintptr_t) i16 + input_offset);
120*4bdc9457SAndroid Build Coastguard Worker }
121*4bdc9457SAndroid Build Coastguard Worker const float* i17 = input[17];
122*4bdc9457SAndroid Build Coastguard Worker assert(i17 != NULL);
123*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i17 != zero) {
124*4bdc9457SAndroid Build Coastguard Worker i17 = (const float*) ((uintptr_t) i17 + input_offset);
125*4bdc9457SAndroid Build Coastguard Worker }
126*4bdc9457SAndroid Build Coastguard Worker const float* i18 = input[18];
127*4bdc9457SAndroid Build Coastguard Worker assert(i18 != NULL);
128*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i18 != zero) {
129*4bdc9457SAndroid Build Coastguard Worker i18 = (const float*) ((uintptr_t) i18 + input_offset);
130*4bdc9457SAndroid Build Coastguard Worker }
131*4bdc9457SAndroid Build Coastguard Worker const float* i19 = input[19];
132*4bdc9457SAndroid Build Coastguard Worker assert(i19 != NULL);
133*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i19 != zero) {
134*4bdc9457SAndroid Build Coastguard Worker i19 = (const float*) ((uintptr_t) i19 + input_offset);
135*4bdc9457SAndroid Build Coastguard Worker }
136*4bdc9457SAndroid Build Coastguard Worker const float* i20 = input[20];
137*4bdc9457SAndroid Build Coastguard Worker assert(i20 != NULL);
138*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i20 != zero) {
139*4bdc9457SAndroid Build Coastguard Worker i20 = (const float*) ((uintptr_t) i20 + input_offset);
140*4bdc9457SAndroid Build Coastguard Worker }
141*4bdc9457SAndroid Build Coastguard Worker const float* i21 = input[21];
142*4bdc9457SAndroid Build Coastguard Worker assert(i21 != NULL);
143*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i21 != zero) {
144*4bdc9457SAndroid Build Coastguard Worker i21 = (const float*) ((uintptr_t) i21 + input_offset);
145*4bdc9457SAndroid Build Coastguard Worker }
146*4bdc9457SAndroid Build Coastguard Worker const float* i22 = input[22];
147*4bdc9457SAndroid Build Coastguard Worker assert(i22 != NULL);
148*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i22 != zero) {
149*4bdc9457SAndroid Build Coastguard Worker i22 = (const float*) ((uintptr_t) i22 + input_offset);
150*4bdc9457SAndroid Build Coastguard Worker }
151*4bdc9457SAndroid Build Coastguard Worker const float* i23 = input[23];
152*4bdc9457SAndroid Build Coastguard Worker assert(i23 != NULL);
153*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i23 != zero) {
154*4bdc9457SAndroid Build Coastguard Worker i23 = (const float*) ((uintptr_t) i23 + input_offset);
155*4bdc9457SAndroid Build Coastguard Worker }
156*4bdc9457SAndroid Build Coastguard Worker const float* i24 = input[24];
157*4bdc9457SAndroid Build Coastguard Worker assert(i24 != NULL);
158*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(i24 != zero) {
159*4bdc9457SAndroid Build Coastguard Worker i24 = (const float*) ((uintptr_t) i24 + input_offset);
160*4bdc9457SAndroid Build Coastguard Worker }
161*4bdc9457SAndroid Build Coastguard Worker input = (const float**) ((uintptr_t) input + input_stride);
162*4bdc9457SAndroid Build Coastguard Worker
163*4bdc9457SAndroid Build Coastguard Worker size_t c = channels;
164*4bdc9457SAndroid Build Coastguard Worker const float* w = weights;
165*4bdc9457SAndroid Build Coastguard Worker for (; c >= 16; c -= 16) {
166*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEFp0 = _mm512_load_ps(w);
167*4bdc9457SAndroid Build Coastguard Worker
168*4bdc9457SAndroid Build Coastguard Worker
169*4bdc9457SAndroid Build Coastguard Worker const __m512 vi0x0123456789ABCDEF = _mm512_loadu_ps(i0);
170*4bdc9457SAndroid Build Coastguard Worker i0 += 16;
171*4bdc9457SAndroid Build Coastguard Worker
172*4bdc9457SAndroid Build Coastguard Worker const __m512 vk0x0123456789ABCDEF = _mm512_load_ps(w + 16);
173*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF, vacc0123456789ABCDEFp0);
174*4bdc9457SAndroid Build Coastguard Worker
175*4bdc9457SAndroid Build Coastguard Worker const __m512 vi1x0123456789ABCDEF = _mm512_loadu_ps(i1);
176*4bdc9457SAndroid Build Coastguard Worker i1 += 16;
177*4bdc9457SAndroid Build Coastguard Worker
178*4bdc9457SAndroid Build Coastguard Worker const __m512 vk1x0123456789ABCDEF = _mm512_load_ps(w + 32);
179*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEFp1 = _mm512_mul_ps(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
180*4bdc9457SAndroid Build Coastguard Worker
181*4bdc9457SAndroid Build Coastguard Worker const __m512 vi2x0123456789ABCDEF = _mm512_loadu_ps(i2);
182*4bdc9457SAndroid Build Coastguard Worker i2 += 16;
183*4bdc9457SAndroid Build Coastguard Worker
184*4bdc9457SAndroid Build Coastguard Worker const __m512 vk2x0123456789ABCDEF = _mm512_load_ps(w + 48);
185*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF, vacc0123456789ABCDEFp0);
186*4bdc9457SAndroid Build Coastguard Worker
187*4bdc9457SAndroid Build Coastguard Worker const __m512 vi3x0123456789ABCDEF = _mm512_loadu_ps(i3);
188*4bdc9457SAndroid Build Coastguard Worker i3 += 16;
189*4bdc9457SAndroid Build Coastguard Worker
190*4bdc9457SAndroid Build Coastguard Worker const __m512 vk3x0123456789ABCDEF = _mm512_load_ps(w + 64);
191*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF, vacc0123456789ABCDEFp1);
192*4bdc9457SAndroid Build Coastguard Worker
193*4bdc9457SAndroid Build Coastguard Worker const __m512 vi4x0123456789ABCDEF = _mm512_loadu_ps(i4);
194*4bdc9457SAndroid Build Coastguard Worker i4 += 16;
195*4bdc9457SAndroid Build Coastguard Worker
196*4bdc9457SAndroid Build Coastguard Worker const __m512 vk4x0123456789ABCDEF = _mm512_load_ps(w + 80);
197*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF, vacc0123456789ABCDEFp0);
198*4bdc9457SAndroid Build Coastguard Worker
199*4bdc9457SAndroid Build Coastguard Worker const __m512 vi5x0123456789ABCDEF = _mm512_loadu_ps(i5);
200*4bdc9457SAndroid Build Coastguard Worker i5 += 16;
201*4bdc9457SAndroid Build Coastguard Worker
202*4bdc9457SAndroid Build Coastguard Worker const __m512 vk5x0123456789ABCDEF = _mm512_load_ps(w + 96);
203*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF, vacc0123456789ABCDEFp1);
204*4bdc9457SAndroid Build Coastguard Worker
205*4bdc9457SAndroid Build Coastguard Worker const __m512 vi6x0123456789ABCDEF = _mm512_loadu_ps(i6);
206*4bdc9457SAndroid Build Coastguard Worker i6 += 16;
207*4bdc9457SAndroid Build Coastguard Worker
208*4bdc9457SAndroid Build Coastguard Worker const __m512 vk6x0123456789ABCDEF = _mm512_load_ps(w + 112);
209*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF, vacc0123456789ABCDEFp0);
210*4bdc9457SAndroid Build Coastguard Worker
211*4bdc9457SAndroid Build Coastguard Worker const __m512 vi7x0123456789ABCDEF = _mm512_loadu_ps(i7);
212*4bdc9457SAndroid Build Coastguard Worker i7 += 16;
213*4bdc9457SAndroid Build Coastguard Worker
214*4bdc9457SAndroid Build Coastguard Worker const __m512 vk7x0123456789ABCDEF = _mm512_load_ps(w + 128);
215*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF, vacc0123456789ABCDEFp1);
216*4bdc9457SAndroid Build Coastguard Worker
217*4bdc9457SAndroid Build Coastguard Worker const __m512 vi8x0123456789ABCDEF = _mm512_loadu_ps(i8);
218*4bdc9457SAndroid Build Coastguard Worker i8 += 16;
219*4bdc9457SAndroid Build Coastguard Worker
220*4bdc9457SAndroid Build Coastguard Worker const __m512 vk8x0123456789ABCDEF = _mm512_load_ps(w + 144);
221*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF, vacc0123456789ABCDEFp0);
222*4bdc9457SAndroid Build Coastguard Worker
223*4bdc9457SAndroid Build Coastguard Worker const __m512 vi9x0123456789ABCDEF = _mm512_loadu_ps(i9);
224*4bdc9457SAndroid Build Coastguard Worker i9 += 16;
225*4bdc9457SAndroid Build Coastguard Worker
226*4bdc9457SAndroid Build Coastguard Worker const __m512 vk9x0123456789ABCDEF = _mm512_load_ps(w + 160);
227*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF, vacc0123456789ABCDEFp1);
228*4bdc9457SAndroid Build Coastguard Worker
229*4bdc9457SAndroid Build Coastguard Worker const __m512 vi10x0123456789ABCDEF = _mm512_loadu_ps(i10);
230*4bdc9457SAndroid Build Coastguard Worker i10 += 16;
231*4bdc9457SAndroid Build Coastguard Worker
232*4bdc9457SAndroid Build Coastguard Worker const __m512 vk10x0123456789ABCDEF = _mm512_load_ps(w + 176);
233*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF, vacc0123456789ABCDEFp0);
234*4bdc9457SAndroid Build Coastguard Worker
235*4bdc9457SAndroid Build Coastguard Worker const __m512 vi11x0123456789ABCDEF = _mm512_loadu_ps(i11);
236*4bdc9457SAndroid Build Coastguard Worker i11 += 16;
237*4bdc9457SAndroid Build Coastguard Worker
238*4bdc9457SAndroid Build Coastguard Worker const __m512 vk11x0123456789ABCDEF = _mm512_load_ps(w + 192);
239*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF, vacc0123456789ABCDEFp1);
240*4bdc9457SAndroid Build Coastguard Worker
241*4bdc9457SAndroid Build Coastguard Worker const __m512 vi12x0123456789ABCDEF = _mm512_loadu_ps(i12);
242*4bdc9457SAndroid Build Coastguard Worker i12 += 16;
243*4bdc9457SAndroid Build Coastguard Worker
244*4bdc9457SAndroid Build Coastguard Worker const __m512 vk12x0123456789ABCDEF = _mm512_load_ps(w + 208);
245*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF, vacc0123456789ABCDEFp0);
246*4bdc9457SAndroid Build Coastguard Worker
247*4bdc9457SAndroid Build Coastguard Worker const __m512 vi13x0123456789ABCDEF = _mm512_loadu_ps(i13);
248*4bdc9457SAndroid Build Coastguard Worker i13 += 16;
249*4bdc9457SAndroid Build Coastguard Worker
250*4bdc9457SAndroid Build Coastguard Worker const __m512 vk13x0123456789ABCDEF = _mm512_load_ps(w + 224);
251*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF, vacc0123456789ABCDEFp1);
252*4bdc9457SAndroid Build Coastguard Worker
253*4bdc9457SAndroid Build Coastguard Worker const __m512 vi14x0123456789ABCDEF = _mm512_loadu_ps(i14);
254*4bdc9457SAndroid Build Coastguard Worker i14 += 16;
255*4bdc9457SAndroid Build Coastguard Worker
256*4bdc9457SAndroid Build Coastguard Worker const __m512 vk14x0123456789ABCDEF = _mm512_load_ps(w + 240);
257*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF, vacc0123456789ABCDEFp0);
258*4bdc9457SAndroid Build Coastguard Worker
259*4bdc9457SAndroid Build Coastguard Worker const __m512 vi15x0123456789ABCDEF = _mm512_loadu_ps(i15);
260*4bdc9457SAndroid Build Coastguard Worker i15 += 16;
261*4bdc9457SAndroid Build Coastguard Worker
262*4bdc9457SAndroid Build Coastguard Worker const __m512 vk15x0123456789ABCDEF = _mm512_load_ps(w + 256);
263*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF, vacc0123456789ABCDEFp1);
264*4bdc9457SAndroid Build Coastguard Worker
265*4bdc9457SAndroid Build Coastguard Worker const __m512 vi16x0123456789ABCDEF = _mm512_loadu_ps(i16);
266*4bdc9457SAndroid Build Coastguard Worker i16 += 16;
267*4bdc9457SAndroid Build Coastguard Worker
268*4bdc9457SAndroid Build Coastguard Worker const __m512 vk16x0123456789ABCDEF = _mm512_load_ps(w + 272);
269*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF, vacc0123456789ABCDEFp0);
270*4bdc9457SAndroid Build Coastguard Worker
271*4bdc9457SAndroid Build Coastguard Worker const __m512 vi17x0123456789ABCDEF = _mm512_loadu_ps(i17);
272*4bdc9457SAndroid Build Coastguard Worker i17 += 16;
273*4bdc9457SAndroid Build Coastguard Worker
274*4bdc9457SAndroid Build Coastguard Worker const __m512 vk17x0123456789ABCDEF = _mm512_load_ps(w + 288);
275*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF, vacc0123456789ABCDEFp1);
276*4bdc9457SAndroid Build Coastguard Worker
277*4bdc9457SAndroid Build Coastguard Worker const __m512 vi18x0123456789ABCDEF = _mm512_loadu_ps(i18);
278*4bdc9457SAndroid Build Coastguard Worker i18 += 16;
279*4bdc9457SAndroid Build Coastguard Worker
280*4bdc9457SAndroid Build Coastguard Worker const __m512 vk18x0123456789ABCDEF = _mm512_load_ps(w + 304);
281*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF, vacc0123456789ABCDEFp0);
282*4bdc9457SAndroid Build Coastguard Worker
283*4bdc9457SAndroid Build Coastguard Worker const __m512 vi19x0123456789ABCDEF = _mm512_loadu_ps(i19);
284*4bdc9457SAndroid Build Coastguard Worker i19 += 16;
285*4bdc9457SAndroid Build Coastguard Worker
286*4bdc9457SAndroid Build Coastguard Worker const __m512 vk19x0123456789ABCDEF = _mm512_load_ps(w + 320);
287*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF, vacc0123456789ABCDEFp1);
288*4bdc9457SAndroid Build Coastguard Worker
289*4bdc9457SAndroid Build Coastguard Worker const __m512 vi20x0123456789ABCDEF = _mm512_loadu_ps(i20);
290*4bdc9457SAndroid Build Coastguard Worker i20 += 16;
291*4bdc9457SAndroid Build Coastguard Worker
292*4bdc9457SAndroid Build Coastguard Worker const __m512 vk20x0123456789ABCDEF = _mm512_load_ps(w + 336);
293*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF, vacc0123456789ABCDEFp0);
294*4bdc9457SAndroid Build Coastguard Worker
295*4bdc9457SAndroid Build Coastguard Worker const __m512 vi21x0123456789ABCDEF = _mm512_loadu_ps(i21);
296*4bdc9457SAndroid Build Coastguard Worker i21 += 16;
297*4bdc9457SAndroid Build Coastguard Worker
298*4bdc9457SAndroid Build Coastguard Worker const __m512 vk21x0123456789ABCDEF = _mm512_load_ps(w + 352);
299*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF, vacc0123456789ABCDEFp1);
300*4bdc9457SAndroid Build Coastguard Worker
301*4bdc9457SAndroid Build Coastguard Worker const __m512 vi22x0123456789ABCDEF = _mm512_loadu_ps(i22);
302*4bdc9457SAndroid Build Coastguard Worker i22 += 16;
303*4bdc9457SAndroid Build Coastguard Worker
304*4bdc9457SAndroid Build Coastguard Worker const __m512 vk22x0123456789ABCDEF = _mm512_load_ps(w + 368);
305*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF, vacc0123456789ABCDEFp0);
306*4bdc9457SAndroid Build Coastguard Worker
307*4bdc9457SAndroid Build Coastguard Worker const __m512 vi23x0123456789ABCDEF = _mm512_loadu_ps(i23);
308*4bdc9457SAndroid Build Coastguard Worker i23 += 16;
309*4bdc9457SAndroid Build Coastguard Worker
310*4bdc9457SAndroid Build Coastguard Worker const __m512 vk23x0123456789ABCDEF = _mm512_load_ps(w + 384);
311*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF, vacc0123456789ABCDEFp1);
312*4bdc9457SAndroid Build Coastguard Worker
313*4bdc9457SAndroid Build Coastguard Worker const __m512 vi24x0123456789ABCDEF = _mm512_loadu_ps(i24);
314*4bdc9457SAndroid Build Coastguard Worker i24 += 16;
315*4bdc9457SAndroid Build Coastguard Worker
316*4bdc9457SAndroid Build Coastguard Worker const __m512 vk24x0123456789ABCDEF = _mm512_load_ps(w + 400);
317*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF, vacc0123456789ABCDEFp0);
318*4bdc9457SAndroid Build Coastguard Worker
319*4bdc9457SAndroid Build Coastguard Worker w += 416;
320*4bdc9457SAndroid Build Coastguard Worker
321*4bdc9457SAndroid Build Coastguard Worker // Add up all accumulators to vacc0123456789ABCDEFp0
322*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_add_ps(vacc0123456789ABCDEFp0, vacc0123456789ABCDEFp1);
323*4bdc9457SAndroid Build Coastguard Worker
324*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEF = _mm512_max_ps(vacc0123456789ABCDEFp0, vmin);
325*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEF = _mm512_min_ps(vacc0123456789ABCDEF, vmax);
326*4bdc9457SAndroid Build Coastguard Worker
327*4bdc9457SAndroid Build Coastguard Worker _mm512_storeu_ps(output, vacc0123456789ABCDEF);
328*4bdc9457SAndroid Build Coastguard Worker output += 16;
329*4bdc9457SAndroid Build Coastguard Worker }
330*4bdc9457SAndroid Build Coastguard Worker if XNN_UNLIKELY(c != 0) {
331*4bdc9457SAndroid Build Coastguard Worker assert(c >= 1);
332*4bdc9457SAndroid Build Coastguard Worker assert(c <= 16);
333*4bdc9457SAndroid Build Coastguard Worker // Prepare mask for valid 32-bit elements (depends on nc).
334*4bdc9457SAndroid Build Coastguard Worker const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << c) - UINT32_C(1)));
335*4bdc9457SAndroid Build Coastguard Worker
336*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEFp0 = _mm512_maskz_loadu_ps(vmask, w);
337*4bdc9457SAndroid Build Coastguard Worker
338*4bdc9457SAndroid Build Coastguard Worker const __m512 vi0x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i0);
339*4bdc9457SAndroid Build Coastguard Worker const __m512 vk0x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 16);
340*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi0x0123456789ABCDEF, vk0x0123456789ABCDEF, vacc0123456789ABCDEFp0);
341*4bdc9457SAndroid Build Coastguard Worker
342*4bdc9457SAndroid Build Coastguard Worker const __m512 vi1x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i1);
343*4bdc9457SAndroid Build Coastguard Worker const __m512 vk1x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 32);
344*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEFp1 = _mm512_mul_ps(vi1x0123456789ABCDEF, vk1x0123456789ABCDEF);
345*4bdc9457SAndroid Build Coastguard Worker
346*4bdc9457SAndroid Build Coastguard Worker const __m512 vi2x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i2);
347*4bdc9457SAndroid Build Coastguard Worker const __m512 vk2x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 48);
348*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi2x0123456789ABCDEF, vk2x0123456789ABCDEF, vacc0123456789ABCDEFp0);
349*4bdc9457SAndroid Build Coastguard Worker
350*4bdc9457SAndroid Build Coastguard Worker const __m512 vi3x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i3);
351*4bdc9457SAndroid Build Coastguard Worker const __m512 vk3x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 64);
352*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi3x0123456789ABCDEF, vk3x0123456789ABCDEF, vacc0123456789ABCDEFp1);
353*4bdc9457SAndroid Build Coastguard Worker
354*4bdc9457SAndroid Build Coastguard Worker const __m512 vi4x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i4);
355*4bdc9457SAndroid Build Coastguard Worker const __m512 vk4x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 80);
356*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi4x0123456789ABCDEF, vk4x0123456789ABCDEF, vacc0123456789ABCDEFp0);
357*4bdc9457SAndroid Build Coastguard Worker
358*4bdc9457SAndroid Build Coastguard Worker const __m512 vi5x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i5);
359*4bdc9457SAndroid Build Coastguard Worker const __m512 vk5x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 96);
360*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi5x0123456789ABCDEF, vk5x0123456789ABCDEF, vacc0123456789ABCDEFp1);
361*4bdc9457SAndroid Build Coastguard Worker
362*4bdc9457SAndroid Build Coastguard Worker const __m512 vi6x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i6);
363*4bdc9457SAndroid Build Coastguard Worker const __m512 vk6x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 112);
364*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi6x0123456789ABCDEF, vk6x0123456789ABCDEF, vacc0123456789ABCDEFp0);
365*4bdc9457SAndroid Build Coastguard Worker
366*4bdc9457SAndroid Build Coastguard Worker const __m512 vi7x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i7);
367*4bdc9457SAndroid Build Coastguard Worker const __m512 vk7x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 128);
368*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi7x0123456789ABCDEF, vk7x0123456789ABCDEF, vacc0123456789ABCDEFp1);
369*4bdc9457SAndroid Build Coastguard Worker
370*4bdc9457SAndroid Build Coastguard Worker const __m512 vi8x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i8);
371*4bdc9457SAndroid Build Coastguard Worker const __m512 vk8x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 144);
372*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi8x0123456789ABCDEF, vk8x0123456789ABCDEF, vacc0123456789ABCDEFp0);
373*4bdc9457SAndroid Build Coastguard Worker
374*4bdc9457SAndroid Build Coastguard Worker const __m512 vi9x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i9);
375*4bdc9457SAndroid Build Coastguard Worker const __m512 vk9x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 160);
376*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi9x0123456789ABCDEF, vk9x0123456789ABCDEF, vacc0123456789ABCDEFp1);
377*4bdc9457SAndroid Build Coastguard Worker
378*4bdc9457SAndroid Build Coastguard Worker const __m512 vi10x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i10);
379*4bdc9457SAndroid Build Coastguard Worker const __m512 vk10x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 176);
380*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi10x0123456789ABCDEF, vk10x0123456789ABCDEF, vacc0123456789ABCDEFp0);
381*4bdc9457SAndroid Build Coastguard Worker
382*4bdc9457SAndroid Build Coastguard Worker const __m512 vi11x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i11);
383*4bdc9457SAndroid Build Coastguard Worker const __m512 vk11x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 192);
384*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi11x0123456789ABCDEF, vk11x0123456789ABCDEF, vacc0123456789ABCDEFp1);
385*4bdc9457SAndroid Build Coastguard Worker
386*4bdc9457SAndroid Build Coastguard Worker const __m512 vi12x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i12);
387*4bdc9457SAndroid Build Coastguard Worker const __m512 vk12x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 208);
388*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi12x0123456789ABCDEF, vk12x0123456789ABCDEF, vacc0123456789ABCDEFp0);
389*4bdc9457SAndroid Build Coastguard Worker
390*4bdc9457SAndroid Build Coastguard Worker const __m512 vi13x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i13);
391*4bdc9457SAndroid Build Coastguard Worker const __m512 vk13x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 224);
392*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi13x0123456789ABCDEF, vk13x0123456789ABCDEF, vacc0123456789ABCDEFp1);
393*4bdc9457SAndroid Build Coastguard Worker
394*4bdc9457SAndroid Build Coastguard Worker const __m512 vi14x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i14);
395*4bdc9457SAndroid Build Coastguard Worker const __m512 vk14x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 240);
396*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi14x0123456789ABCDEF, vk14x0123456789ABCDEF, vacc0123456789ABCDEFp0);
397*4bdc9457SAndroid Build Coastguard Worker
398*4bdc9457SAndroid Build Coastguard Worker const __m512 vi15x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i15);
399*4bdc9457SAndroid Build Coastguard Worker const __m512 vk15x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 256);
400*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi15x0123456789ABCDEF, vk15x0123456789ABCDEF, vacc0123456789ABCDEFp1);
401*4bdc9457SAndroid Build Coastguard Worker
402*4bdc9457SAndroid Build Coastguard Worker const __m512 vi16x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i16);
403*4bdc9457SAndroid Build Coastguard Worker const __m512 vk16x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 272);
404*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi16x0123456789ABCDEF, vk16x0123456789ABCDEF, vacc0123456789ABCDEFp0);
405*4bdc9457SAndroid Build Coastguard Worker
406*4bdc9457SAndroid Build Coastguard Worker const __m512 vi17x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i17);
407*4bdc9457SAndroid Build Coastguard Worker const __m512 vk17x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 288);
408*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi17x0123456789ABCDEF, vk17x0123456789ABCDEF, vacc0123456789ABCDEFp1);
409*4bdc9457SAndroid Build Coastguard Worker
410*4bdc9457SAndroid Build Coastguard Worker const __m512 vi18x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i18);
411*4bdc9457SAndroid Build Coastguard Worker const __m512 vk18x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 304);
412*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi18x0123456789ABCDEF, vk18x0123456789ABCDEF, vacc0123456789ABCDEFp0);
413*4bdc9457SAndroid Build Coastguard Worker
414*4bdc9457SAndroid Build Coastguard Worker const __m512 vi19x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i19);
415*4bdc9457SAndroid Build Coastguard Worker const __m512 vk19x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 320);
416*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi19x0123456789ABCDEF, vk19x0123456789ABCDEF, vacc0123456789ABCDEFp1);
417*4bdc9457SAndroid Build Coastguard Worker
418*4bdc9457SAndroid Build Coastguard Worker const __m512 vi20x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i20);
419*4bdc9457SAndroid Build Coastguard Worker const __m512 vk20x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 336);
420*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi20x0123456789ABCDEF, vk20x0123456789ABCDEF, vacc0123456789ABCDEFp0);
421*4bdc9457SAndroid Build Coastguard Worker
422*4bdc9457SAndroid Build Coastguard Worker const __m512 vi21x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i21);
423*4bdc9457SAndroid Build Coastguard Worker const __m512 vk21x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 352);
424*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi21x0123456789ABCDEF, vk21x0123456789ABCDEF, vacc0123456789ABCDEFp1);
425*4bdc9457SAndroid Build Coastguard Worker
426*4bdc9457SAndroid Build Coastguard Worker const __m512 vi22x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i22);
427*4bdc9457SAndroid Build Coastguard Worker const __m512 vk22x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 368);
428*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi22x0123456789ABCDEF, vk22x0123456789ABCDEF, vacc0123456789ABCDEFp0);
429*4bdc9457SAndroid Build Coastguard Worker
430*4bdc9457SAndroid Build Coastguard Worker const __m512 vi23x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i23);
431*4bdc9457SAndroid Build Coastguard Worker const __m512 vk23x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 384);
432*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp1 = _mm512_fmadd_ps(vi23x0123456789ABCDEF, vk23x0123456789ABCDEF, vacc0123456789ABCDEFp1);
433*4bdc9457SAndroid Build Coastguard Worker
434*4bdc9457SAndroid Build Coastguard Worker const __m512 vi24x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, i24);
435*4bdc9457SAndroid Build Coastguard Worker const __m512 vk24x0123456789ABCDEF = _mm512_maskz_loadu_ps(vmask, w + 400);
436*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_fmadd_ps(vi24x0123456789ABCDEF, vk24x0123456789ABCDEF, vacc0123456789ABCDEFp0);
437*4bdc9457SAndroid Build Coastguard Worker
438*4bdc9457SAndroid Build Coastguard Worker // Add up all accumulators to vacc0123456789ABCDEFp0
439*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEFp0 = _mm512_add_ps(vacc0123456789ABCDEFp0, vacc0123456789ABCDEFp1);
440*4bdc9457SAndroid Build Coastguard Worker
441*4bdc9457SAndroid Build Coastguard Worker __m512 vacc0123456789ABCDEF = _mm512_max_ps(vacc0123456789ABCDEFp0, vmin);
442*4bdc9457SAndroid Build Coastguard Worker vacc0123456789ABCDEF = _mm512_min_ps(vacc0123456789ABCDEF, vmax);
443*4bdc9457SAndroid Build Coastguard Worker
444*4bdc9457SAndroid Build Coastguard Worker _mm512_mask_storeu_ps(output, vmask, vacc0123456789ABCDEF);
445*4bdc9457SAndroid Build Coastguard Worker output += c;
446*4bdc9457SAndroid Build Coastguard Worker }
447*4bdc9457SAndroid Build Coastguard Worker
448*4bdc9457SAndroid Build Coastguard Worker output = (float*) ((uintptr_t) output + output_increment);
449*4bdc9457SAndroid Build Coastguard Worker } while (--output_width != 0);
450*4bdc9457SAndroid Build Coastguard Worker }
451