1 // Auto-generated file. Do not edit!
2 // Template: src/qs8-gavgpool/multipass-scalar.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2021 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <math.h>
12
13 #include <xnnpack/gavgpool.h>
14 #include <xnnpack/math.h>
15
16
xnn_qs8_gavgpool_minmax_fp32_ukernel_7p7x__scalar_lrintf_c4(size_t rows,size_t channels,const int8_t * input,size_t input_stride,const int8_t * zero,int32_t * buffer,int8_t * output,const union xnn_qs8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])17 void xnn_qs8_gavgpool_minmax_fp32_ukernel_7p7x__scalar_lrintf_c4(
18 size_t rows,
19 size_t channels,
20 const int8_t* input,
21 size_t input_stride,
22 const int8_t* zero,
23 int32_t* buffer,
24 int8_t* output,
25 const union xnn_qs8_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)])
26 {
27 assert(rows > 7);
28 assert(channels != 0);
29
30 const int8_t* i0 = input;
31 const int8_t* i1 = (const int8_t*) ((uintptr_t) i0 + input_stride);
32 const int8_t* i2 = (const int8_t*) ((uintptr_t) i1 + input_stride);
33 const int8_t* i3 = (const int8_t*) ((uintptr_t) i2 + input_stride);
34 const int8_t* i4 = (const int8_t*) ((uintptr_t) i3 + input_stride);
35 const int8_t* i5 = (const int8_t*) ((uintptr_t) i4 + input_stride);
36 const int8_t* i6 = (const int8_t*) ((uintptr_t) i5 + input_stride);
37 const size_t input_increment = 7 * input_stride - round_up_po2(channels, 4) * sizeof(int8_t);
38
39 const int32_t vinit_bias = params->fp32_scalar_lrintf.init_bias;
40 int32_t* b = buffer;
41 for (ptrdiff_t c = (ptrdiff_t) channels; c > 0; c -= 4) {
42 const int32_t vi0x0 = (int32_t) i0[0];
43 const int32_t vi0x1 = (int32_t) i0[1];
44 const int32_t vi0x2 = (int32_t) i0[2];
45 const int32_t vi0x3 = (int32_t) i0[3];
46 i0 += 4;
47
48 int32_t vacc0 = vi0x0 + vinit_bias;
49 const int32_t vi1x0 = (int32_t) i1[0];
50 int32_t vacc1 = vi0x1 + vinit_bias;
51 const int32_t vi1x1 = (int32_t) i1[1];
52 int32_t vacc2 = vi0x2 + vinit_bias;
53 const int32_t vi1x2 = (int32_t) i1[2];
54 int32_t vacc3 = vi0x3 + vinit_bias;
55 const int32_t vi1x3 = (int32_t) i1[3];
56 i1 += 4;
57
58 vacc0 += vi1x0;
59 const int32_t vi2x0 = (int32_t) i2[0];
60 vacc1 += vi1x1;
61 const int32_t vi2x1 = (int32_t) i2[1];
62 vacc2 += vi1x2;
63 const int32_t vi2x2 = (int32_t) i2[2];
64 vacc3 += vi1x3;
65 const int32_t vi2x3 = (int32_t) i2[3];
66 i2 += 4;
67 vacc0 += vi2x0;
68 const int32_t vi3x0 = (int32_t) i3[0];
69 vacc1 += vi2x1;
70 const int32_t vi3x1 = (int32_t) i3[1];
71 vacc2 += vi2x2;
72 const int32_t vi3x2 = (int32_t) i3[2];
73 vacc3 += vi2x3;
74 const int32_t vi3x3 = (int32_t) i3[3];
75 i3 += 4;
76 vacc0 += vi3x0;
77 const int32_t vi4x0 = (int32_t) i4[0];
78 vacc1 += vi3x1;
79 const int32_t vi4x1 = (int32_t) i4[1];
80 vacc2 += vi3x2;
81 const int32_t vi4x2 = (int32_t) i4[2];
82 vacc3 += vi3x3;
83 const int32_t vi4x3 = (int32_t) i4[3];
84 i4 += 4;
85 vacc0 += vi4x0;
86 const int32_t vi5x0 = (int32_t) i5[0];
87 vacc1 += vi4x1;
88 const int32_t vi5x1 = (int32_t) i5[1];
89 vacc2 += vi4x2;
90 const int32_t vi5x2 = (int32_t) i5[2];
91 vacc3 += vi4x3;
92 const int32_t vi5x3 = (int32_t) i5[3];
93 i5 += 4;
94 vacc0 += vi5x0;
95 const int32_t vi6x0 = (int32_t) i6[0];
96 vacc1 += vi5x1;
97 const int32_t vi6x1 = (int32_t) i6[1];
98 vacc2 += vi5x2;
99 const int32_t vi6x2 = (int32_t) i6[2];
100 vacc3 += vi5x3;
101 const int32_t vi6x3 = (int32_t) i6[3];
102 i6 += 4;
103
104 vacc0 += vi6x0;
105 vacc1 += vi6x1;
106 vacc2 += vi6x2;
107 vacc3 += vi6x3;
108
109 b[0] = vacc0;
110 b[1] = vacc1;
111 b[2] = vacc2;
112 b[3] = vacc3;
113 b += 4;
114 }
115
116 for (rows -= 7; rows > 7; rows -= 7) {
117 i0 = (const int8_t*) ((uintptr_t) i0 + input_increment);
118 i1 = (const int8_t*) ((uintptr_t) i1 + input_increment);
119 i2 = (const int8_t*) ((uintptr_t) i2 + input_increment);
120 i3 = (const int8_t*) ((uintptr_t) i3 + input_increment);
121 i4 = (const int8_t*) ((uintptr_t) i4 + input_increment);
122 i5 = (const int8_t*) ((uintptr_t) i5 + input_increment);
123 i6 = (const int8_t*) ((uintptr_t) i6 + input_increment);
124
125 int32_t* b = buffer;
126 for (ptrdiff_t c = (ptrdiff_t) channels; c > 0; c -= 4) {
127 int32_t vacc0 = b[0];
128 const int32_t vi0x0 = (int32_t) i0[0];
129 int32_t vacc1 = b[1];
130 const int32_t vi0x1 = (int32_t) i0[1];
131 int32_t vacc2 = b[2];
132 const int32_t vi0x2 = (int32_t) i0[2];
133 int32_t vacc3 = b[3];
134 const int32_t vi0x3 = (int32_t) i0[3];
135 i0 += 4;
136
137 vacc0 += vi0x0;
138 const int32_t vi1x0 = (int32_t) i1[0];
139 vacc1 += vi0x1;
140 const int32_t vi1x1 = (int32_t) i1[1];
141 vacc2 += vi0x2;
142 const int32_t vi1x2 = (int32_t) i1[2];
143 vacc3 += vi0x3;
144 const int32_t vi1x3 = (int32_t) i1[3];
145 i1 += 4;
146 vacc0 += vi1x0;
147 const int32_t vi2x0 = (int32_t) i2[0];
148 vacc1 += vi1x1;
149 const int32_t vi2x1 = (int32_t) i2[1];
150 vacc2 += vi1x2;
151 const int32_t vi2x2 = (int32_t) i2[2];
152 vacc3 += vi1x3;
153 const int32_t vi2x3 = (int32_t) i2[3];
154 i2 += 4;
155 vacc0 += vi2x0;
156 const int32_t vi3x0 = (int32_t) i3[0];
157 vacc1 += vi2x1;
158 const int32_t vi3x1 = (int32_t) i3[1];
159 vacc2 += vi2x2;
160 const int32_t vi3x2 = (int32_t) i3[2];
161 vacc3 += vi2x3;
162 const int32_t vi3x3 = (int32_t) i3[3];
163 i3 += 4;
164 vacc0 += vi3x0;
165 const int32_t vi4x0 = (int32_t) i4[0];
166 vacc1 += vi3x1;
167 const int32_t vi4x1 = (int32_t) i4[1];
168 vacc2 += vi3x2;
169 const int32_t vi4x2 = (int32_t) i4[2];
170 vacc3 += vi3x3;
171 const int32_t vi4x3 = (int32_t) i4[3];
172 i4 += 4;
173 vacc0 += vi4x0;
174 const int32_t vi5x0 = (int32_t) i5[0];
175 vacc1 += vi4x1;
176 const int32_t vi5x1 = (int32_t) i5[1];
177 vacc2 += vi4x2;
178 const int32_t vi5x2 = (int32_t) i5[2];
179 vacc3 += vi4x3;
180 const int32_t vi5x3 = (int32_t) i5[3];
181 i5 += 4;
182 vacc0 += vi5x0;
183 const int32_t vi6x0 = (int32_t) i6[0];
184 vacc1 += vi5x1;
185 const int32_t vi6x1 = (int32_t) i6[1];
186 vacc2 += vi5x2;
187 const int32_t vi6x2 = (int32_t) i6[2];
188 vacc3 += vi5x3;
189 const int32_t vi6x3 = (int32_t) i6[3];
190 i6 += 4;
191
192 vacc0 += vi6x0;
193 vacc1 += vi6x1;
194 vacc2 += vi6x2;
195 vacc3 += vi6x3;
196
197 b[0] = vacc0;
198 b[1] = vacc1;
199 b[2] = vacc2;
200 b[3] = vacc3;
201 b += 4;
202 }
203 }
204
205 i0 = (const int8_t*) ((uintptr_t) i0 + input_increment);
206 i1 = (const int8_t*) ((uintptr_t) i1 + input_increment);
207 if XNN_UNPREDICTABLE(rows < 2) {
208 i1 = zero;
209 }
210 i2 = (const int8_t*) ((uintptr_t) i2 + input_increment);
211 if XNN_UNPREDICTABLE(rows <= 2) {
212 i2 = zero;
213 }
214 i3 = (const int8_t*) ((uintptr_t) i3 + input_increment);
215 if XNN_UNPREDICTABLE(rows < 4) {
216 i3 = zero;
217 }
218 i4 = (const int8_t*) ((uintptr_t) i4 + input_increment);
219 if XNN_UNPREDICTABLE(rows <= 4) {
220 i4 = zero;
221 }
222 i5 = (const int8_t*) ((uintptr_t) i5 + input_increment);
223 if XNN_UNPREDICTABLE(rows < 6) {
224 i5 = zero;
225 }
226 i6 = (const int8_t*) ((uintptr_t) i6 + input_increment);
227 if XNN_UNPREDICTABLE(rows <= 6) {
228 i6 = zero;
229 }
230
231 const float vscale = params->fp32_scalar_lrintf.scale;
232 const float voutput_min_less_zero_point = params->fp32_scalar_lrintf.output_min_less_zero_point;
233 const float voutput_max_less_zero_point = params->fp32_scalar_lrintf.output_max_less_zero_point;
234 const int32_t voutput_zero_point = params->fp32_scalar_lrintf.output_zero_point;
235 for (; channels >= 4; channels -= 4) {
236 int32_t vacc0 = buffer[0];
237 const int32_t vi0x0 = (int32_t) i0[0];
238 int32_t vacc1 = buffer[1];
239 const int32_t vi0x1 = (int32_t) i0[1];
240 int32_t vacc2 = buffer[2];
241 const int32_t vi0x2 = (int32_t) i0[2];
242 int32_t vacc3 = buffer[3];
243 const int32_t vi0x3 = (int32_t) i0[3];
244 buffer += 4;
245 i0 += 4;
246
247 vacc0 += vi0x0;
248 const int32_t vi1x0 = (int32_t) i1[0];
249 vacc1 += vi0x1;
250 const int32_t vi1x1 = (int32_t) i1[1];
251 vacc2 += vi0x2;
252 const int32_t vi1x2 = (int32_t) i1[2];
253 vacc3 += vi0x3;
254 const int32_t vi1x3 = (int32_t) i1[3];
255 i1 += 4;
256 vacc0 += vi1x0;
257 const int32_t vi2x0 = (int32_t) i2[0];
258 vacc1 += vi1x1;
259 const int32_t vi2x1 = (int32_t) i2[1];
260 vacc2 += vi1x2;
261 const int32_t vi2x2 = (int32_t) i2[2];
262 vacc3 += vi1x3;
263 const int32_t vi2x3 = (int32_t) i2[3];
264 i2 += 4;
265 vacc0 += vi2x0;
266 const int32_t vi3x0 = (int32_t) i3[0];
267 vacc1 += vi2x1;
268 const int32_t vi3x1 = (int32_t) i3[1];
269 vacc2 += vi2x2;
270 const int32_t vi3x2 = (int32_t) i3[2];
271 vacc3 += vi2x3;
272 const int32_t vi3x3 = (int32_t) i3[3];
273 i3 += 4;
274 vacc0 += vi3x0;
275 const int32_t vi4x0 = (int32_t) i4[0];
276 vacc1 += vi3x1;
277 const int32_t vi4x1 = (int32_t) i4[1];
278 vacc2 += vi3x2;
279 const int32_t vi4x2 = (int32_t) i4[2];
280 vacc3 += vi3x3;
281 const int32_t vi4x3 = (int32_t) i4[3];
282 i4 += 4;
283 vacc0 += vi4x0;
284 const int32_t vi5x0 = (int32_t) i5[0];
285 vacc1 += vi4x1;
286 const int32_t vi5x1 = (int32_t) i5[1];
287 vacc2 += vi4x2;
288 const int32_t vi5x2 = (int32_t) i5[2];
289 vacc3 += vi4x3;
290 const int32_t vi5x3 = (int32_t) i5[3];
291 i5 += 4;
292 vacc0 += vi5x0;
293 const int32_t vi6x0 = (int32_t) i6[0];
294 vacc1 += vi5x1;
295 const int32_t vi6x1 = (int32_t) i6[1];
296 vacc2 += vi5x2;
297 const int32_t vi6x2 = (int32_t) i6[2];
298 vacc3 += vi5x3;
299 const int32_t vi6x3 = (int32_t) i6[3];
300 i6 += 4;
301
302 vacc0 += vi6x0;
303 vacc1 += vi6x1;
304 vacc2 += vi6x2;
305 vacc3 += vi6x3;
306
307 float vfpacc0 = (float) vacc0 * vscale;
308 float vfpacc1 = (float) vacc1 * vscale;
309 float vfpacc2 = (float) vacc2 * vscale;
310 float vfpacc3 = (float) vacc3 * vscale;
311
312 vfpacc0 = math_max_f32(vfpacc0, voutput_min_less_zero_point);
313 vfpacc1 = math_max_f32(vfpacc1, voutput_min_less_zero_point);
314 vfpacc2 = math_max_f32(vfpacc2, voutput_min_less_zero_point);
315 vfpacc3 = math_max_f32(vfpacc3, voutput_min_less_zero_point);
316
317 vfpacc0 = math_min_f32(vfpacc0, voutput_max_less_zero_point);
318 vfpacc1 = math_min_f32(vfpacc1, voutput_max_less_zero_point);
319 vfpacc2 = math_min_f32(vfpacc2, voutput_max_less_zero_point);
320 vfpacc3 = math_min_f32(vfpacc3, voutput_max_less_zero_point);
321
322 const int32_t vrndacc0 = (int32_t) lrintf(vfpacc0);
323 const int32_t vrndacc1 = (int32_t) lrintf(vfpacc1);
324 const int32_t vrndacc2 = (int32_t) lrintf(vfpacc2);
325 const int32_t vrndacc3 = (int32_t) lrintf(vfpacc3);
326
327 int32_t vout0 = vrndacc0 + voutput_zero_point;
328 int32_t vout1 = vrndacc1 + voutput_zero_point;
329 int32_t vout2 = vrndacc2 + voutput_zero_point;
330 int32_t vout3 = vrndacc3 + voutput_zero_point;
331
332 output[0] = (int8_t) vout0;
333 output[1] = (int8_t) vout1;
334 output[2] = (int8_t) vout2;
335 output[3] = (int8_t) vout3;
336 output += 4;
337 }
338 if XNN_UNLIKELY(channels != 0) {
339 do {
340 int32_t vacc = *buffer++;
341 const int32_t vi0 = (int32_t) *i0++;
342 const int32_t vi1 = (int32_t) *i1++;
343
344 vacc += vi0;
345 const int32_t vi2 = (int32_t) *i2++;
346 vacc += vi1;
347 const int32_t vi3 = (int32_t) *i3++;
348 vacc += vi2;
349 const int32_t vi4 = (int32_t) *i4++;
350 vacc += vi3;
351 const int32_t vi5 = (int32_t) *i5++;
352 vacc += vi4;
353 const int32_t vi6 = (int32_t) *i6++;
354
355 vacc += vi5;
356 vacc += vi6;
357
358 float vfpacc = (float) vacc * vscale;
359 vfpacc = math_max_f32(vfpacc, voutput_min_less_zero_point);
360 vfpacc = math_min_f32(vfpacc, voutput_max_less_zero_point);
361 const int32_t vrndacc = (int32_t) lrintf(vfpacc);
362 int32_t vout = vrndacc + voutput_zero_point;
363
364 *output++ = (int8_t) vout;
365 } while (--channels != 0);
366 }
367 }
368