1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <arm_neon.h>
9
10 #include <xnnpack/gavgpool.h>
11 #include <xnnpack/math.h>
12
13
xnn_f16_gavgpool_cw_ukernel__neonfp16arith_x4(size_t elements,size_t channels,const void * input,void * output,const union xnn_f16_gavgpool_params params[restrict XNN_MIN_ELEMENTS (1)])14 void xnn_f16_gavgpool_cw_ukernel__neonfp16arith_x4(
15 size_t elements,
16 size_t channels,
17 const void* input,
18 void* output,
19 const union xnn_f16_gavgpool_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
20 {
21 assert(elements != 0);
22 assert(elements % sizeof(__fp16) == 0);
23 assert(channels != 0);
24
25 __fp16* o = (__fp16*) output;
26 const __fp16* i0 = input;
27 const __fp16* i1 = (const __fp16*) ((uintptr_t) i0 + elements);
28 const __fp16* i2 = (const __fp16*) ((uintptr_t) i1 + elements);
29 const __fp16* i3 = (const __fp16*) ((uintptr_t) i2 + elements);
30
31 const uint16x4_t vmask = vld1_u16(params->neonfp16arith.mask);
32 const float16x4_t vmultiplier = vreinterpret_f16_u16(vld1_dup_u16(¶ms->neonfp16arith.multiplier));
33 const float16x4_t voutput_min = vreinterpret_f16_u16(vld1_dup_u16(¶ms->neonfp16arith.output_min));
34 const float16x4_t voutput_max = vreinterpret_f16_u16(vld1_dup_u16(¶ms->neonfp16arith.output_max));
35
36 while (channels >= 4) {
37 float16x4_t vsum0 = vmov_n_f16(0);
38 float16x4_t vsum1 = vmov_n_f16(0);
39 float16x4_t vsum2 = vmov_n_f16(0);
40 float16x4_t vsum3 = vmov_n_f16(0);
41 size_t n = elements;
42 while (n >= 4 * sizeof(__fp16)) {
43 const float16x4_t vi0 = vld1_f16(i0); i0 += 4;
44 const float16x4_t vi1 = vld1_f16(i1); i1 += 4;
45 const float16x4_t vi2 = vld1_f16(i2); i2 += 4;
46 const float16x4_t vi3 = vld1_f16(i3); i3 += 4;
47
48 vsum0 = vadd_f16(vsum0, vi0);
49 vsum1 = vadd_f16(vsum1, vi1);
50 vsum2 = vadd_f16(vsum2, vi2);
51 vsum3 = vadd_f16(vsum3, vi3);
52 n -= 4 * sizeof(__fp16);
53 }
54
55 if XNN_UNLIKELY(n != 0) {
56 float16x4_t vi0 = vld1_f16(i0); i0 = (const __fp16*) ((uintptr_t) i0 + n);
57 float16x4_t vi1 = vld1_f16(i1); i1 = (const __fp16*) ((uintptr_t) i1 + n);
58 float16x4_t vi2 = vld1_f16(i2); i2 = (const __fp16*) ((uintptr_t) i2 + n);
59 float16x4_t vi3 = vld1_f16(i3); i3 = (const __fp16*) ((uintptr_t) i3 + n);
60
61 vi0 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi0)));
62 vi1 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi1)));
63 vi2 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi2)));
64 vi3 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi3)));
65
66 vsum0 = vadd_f16(vsum0, vi0);
67 vsum1 = vadd_f16(vsum1, vi1);
68 vsum2 = vadd_f16(vsum2, vi2);
69 vsum3 = vadd_f16(vsum3, vi3);
70 }
71
72 // Having exactly 4 rows makes this work out nicely as we end up with
73 // the 4 totals in 4 different lanes of the same vector.
74 const float16x4_t vsum01 = vpadd_f16(vsum0, vsum1);
75 const float16x4_t vsum23 = vpadd_f16(vsum2, vsum3);
76 const float16x4_t vsum = vpadd_f16(vsum01, vsum23);
77
78 float16x4_t vout = vmul_f16(vsum, vmultiplier);
79
80 vout = vmax_f16(vout, voutput_min);
81 vout = vmin_f16(vout, voutput_max);
82
83 vst1_f16(o, vout); o += 4;
84
85 i0 = i3;
86 i1 = (const __fp16*) ((uintptr_t) i0 + elements);
87 i2 = (const __fp16*) ((uintptr_t) i1 + elements);
88 i3 = (const __fp16*) ((uintptr_t) i2 + elements);
89 channels -= 4;
90 }
91
92 while (channels != 0) {
93 float16x4_t vsum0 = vmov_n_f16(0);
94 size_t n = elements;
95 while (n >= 4 * sizeof(__fp16)) {
96 const float16x4_t vi0 = vld1_f16(i0); i0 += 4;
97
98 vsum0 = vadd_f16(vsum0, vi0);
99 n -= 4 * sizeof(__fp16);
100 }
101
102 if XNN_UNLIKELY(n != 0) {
103 float16x4_t vi0 = vld1_f16(i0); i0 = (const __fp16*) ((uintptr_t) i0 + n);
104
105 vi0 = vreinterpret_f16_u16(vand_u16(vmask, vreinterpret_u16_f16(vi0)));
106
107 vsum0 = vadd_f16(vsum0, vi0);
108 }
109
110 const float16x4_t vsum01 = vpadd_f16(vsum0, vsum0);
111 const float16x4_t vsum = vpadd_f16(vsum01, vsum01);
112
113 float16x4_t vout = vmul_f16(vsum, vmultiplier);
114
115 vout = vmax_f16(vout, voutput_min);
116 vout = vmin_f16(vout, voutput_max);
117
118 vst1_lane_f16(o, vout, 0); o += 1;
119 channels -= 1;
120 }
121 }
122