1 // Auto-generated file. Do not edit!
2 // Template: src/f16-ibilinear-chw/neonfp16arith.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2022 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11
12 #include <arm_neon.h>
13
14 #include <xnnpack/ibilinear.h>
15
16
xnn_f16_ibilinear_chw_ukernel__neonfp16arith_p4(size_t output_pixels,size_t channels,const void ** restrict input,size_t input_offset,const void * restrict weights,void * restrict output,size_t input_increment)17 void xnn_f16_ibilinear_chw_ukernel__neonfp16arith_p4(
18 size_t output_pixels,
19 size_t channels,
20 const void**restrict input,
21 size_t input_offset,
22 const void*restrict weights,
23 void*restrict output,
24 size_t input_increment) XNN_OOB_READS
25 {
26 assert(output_pixels != 0);
27 assert(channels != 0);
28 assert(input_increment % sizeof(__fp16) == 0);
29
30 __fp16* o = (__fp16*) output;
31 do {
32 const __fp16** i = (const __fp16**)input;
33 const __fp16* w = weights;
34 size_t p = output_pixels;
35
36
37 for (; p >= 4; p -= 4) {
38 const __fp16* itl0 = (const __fp16*) ((uintptr_t) i[0] + input_offset);
39 const __fp16* ibl0 = (const __fp16*) ((uintptr_t) i[1] + input_offset);
40 const __fp16* itl1 = (const __fp16*) ((uintptr_t) i[2] + input_offset);
41 const __fp16* ibl1 = (const __fp16*) ((uintptr_t) i[3] + input_offset);
42 const __fp16* itl2 = (const __fp16*) ((uintptr_t) i[4] + input_offset);
43 const __fp16* ibl2 = (const __fp16*) ((uintptr_t) i[5] + input_offset);
44 const __fp16* itl3 = (const __fp16*) ((uintptr_t) i[6] + input_offset);
45 const __fp16* ibl3 = (const __fp16*) ((uintptr_t) i[7] + input_offset);
46 i += 8;
47
48 const float16x4x2_t vw = vld2_f16(w);
49 w += 8;
50
51 float16x8_t vtltr = vmovq_n_f16(0); // vmov for uninitialized var warning
52 float16x8_t vblbr = vmovq_n_f16(0);
53 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl0, vreinterpretq_u32_f16(vtltr), 0));
54 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl0, vreinterpretq_u32_f16(vblbr), 0));
55 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl1, vreinterpretq_u32_f16(vtltr), 1));
56 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl1, vreinterpretq_u32_f16(vblbr), 1));
57 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl2, vreinterpretq_u32_f16(vtltr), 2));
58 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl2, vreinterpretq_u32_f16(vblbr), 2));
59 vtltr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) itl3, vreinterpretq_u32_f16(vtltr), 3));
60 vblbr = vreinterpretq_f16_u32(vld1q_lane_u32((const void*) ibl3, vreinterpretq_u32_f16(vblbr), 3));
61
62 const float16x4_t valphah = vw.val[0];
63 const float16x4_t valphav = vw.val[1];
64
65 const float16x8_t vldrd = vsubq_f16(vblbr, vtltr);
66
67 const float16x4x2_t vld_t = vuzp_f16(vget_low_f16(vldrd), vget_high_f16(vldrd));
68 const float16x4_t vld = vld_t.val[0];
69 const float16x4_t vrd = vld_t.val[1];
70
71 const float16x4x2_t vtl_t = vuzp_f16(vget_low_f16(vtltr), vget_high_f16(vtltr));
72 const float16x4_t vtl = vtl_t.val[0];
73 const float16x4_t vtr = vtl_t.val[1];
74
75 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
76 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
77
78 const float16x4_t vd = vsub_f16(vr, vl);
79 const float16x4_t vo = vfma_f16(vl, vd, valphah);
80
81 vst1_f16(o, vo);
82 o += 4;
83 }
84
85 if XNN_UNLIKELY(p != 0) {
86 if (p & 2) {
87 const __fp16* itl0 = (const __fp16*) ((uintptr_t) i[0] + input_offset);
88 const __fp16* ibl0 = (const __fp16*) ((uintptr_t) i[1] + input_offset);
89 const __fp16* itl1 = (const __fp16*) ((uintptr_t) i[2] + input_offset);
90 const __fp16* ibl1 = (const __fp16*) ((uintptr_t) i[3] + input_offset);
91 i += 4;
92
93 const float16x4_t vw = vld1_f16(w);
94 w += 4;
95
96 const float16x4x2_t vwhv = vuzp_f16(vw, vw);
97 const float16x4_t valphah = vwhv.val[0];
98 const float16x4_t valphav = vwhv.val[1];
99
100 float16x4_t vtltr = vmov_n_f16(0); // vmov for uninitialized var warning
101 float16x4_t vblbr = vmov_n_f16(0);
102
103 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl0, vreinterpret_u32_f16(vtltr), 0));
104 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl0, vreinterpret_u32_f16(vblbr), 0));
105 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl1, vreinterpret_u32_f16(vtltr), 1));
106 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl1, vreinterpret_u32_f16(vblbr), 1));
107
108 const float16x4_t vldrd = vsub_f16(vblbr, vtltr);
109
110 const float16x4x2_t vld_t = vuzp_f16(vldrd, vldrd);
111 const float16x4_t vld = vld_t.val[0];
112 const float16x4_t vrd = vld_t.val[1];
113
114 const float16x4x2_t vtl_t = vuzp_f16(vtltr, vtltr);
115 const float16x4_t vtl = vtl_t.val[0];
116 const float16x4_t vtr = vtl_t.val[1];
117
118 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
119 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
120
121 const float16x4_t vd = vsub_f16(vr, vl);
122 const float16x4_t vo = vfma_f16(vl, vd, valphah);
123
124 vst1_lane_u32((void*) o, vreinterpret_u32_f16(vo), 0);
125 o += 2;
126 }
127
128 if (p & 1) {
129 // We are computing the following formula:
130 // result = (1 - alpha_h) * (1 - alpha_v) * top_left +
131 // alpha_h * (1 - alpha_v) * top_right +
132 // (1 - alpha_h) * alpha_v * bottom_left +
133 // alpha_h * alpha_v * bottom_right.
134 //
135 // Rearranging gives
136 // result = left + alpha_h * (right - left),
137 // where
138 // left = top_left + alpha_v * (bottom_left - top_left),
139 // right = top_right + alpha_v * (bottom_right - top_right).
140
141 const __fp16* itl = (const __fp16*) ((uintptr_t) i[0] + input_offset);
142 const __fp16* ibl = (const __fp16*) ((uintptr_t) i[1] + input_offset);
143 i += 2;
144
145 float16x4_t vw = vmov_n_f16(0);
146 vw = vreinterpret_f16_u32(vld1_lane_u32((const void*) w, vreinterpret_u32_f16(vw), 0));
147 w += 2;
148
149 const float16x4x2_t vwhv = vuzp_f16(vw, vw);
150 const float16x4_t valphah = vwhv.val[0];
151 const float16x4_t valphav = vwhv.val[1];
152
153 float16x4_t vtltr = vmov_n_f16(0); // vmov for uninitialized var warning
154 float16x4_t vblbr = vmov_n_f16(0);
155
156 vtltr = vreinterpret_f16_u32(vld1_lane_u32((const void*) itl, vreinterpret_u32_f16(vtltr), 0));
157 vblbr = vreinterpret_f16_u32(vld1_lane_u32((const void*) ibl, vreinterpret_u32_f16(vblbr), 0));
158
159 const float16x4_t vldrd = vsub_f16(vblbr, vtltr);
160
161 const float16x4x2_t vld_t = vuzp_f16(vldrd, vldrd);
162 const float16x4_t vld = vld_t.val[0];
163 const float16x4_t vrd = vld_t.val[1];
164
165 const float16x4x2_t vtl_t = vuzp_f16(vtltr, vtltr);
166 const float16x4_t vtl = vtl_t.val[0];
167 const float16x4_t vtr = vtl_t.val[1];
168
169 const float16x4_t vl = vfma_f16(vtl, vld, valphav);
170 const float16x4_t vr = vfma_f16(vtr, vrd, valphav);
171
172 const float16x4_t vd = vsub_f16(vr, vl);
173 const float16x4_t vo = vfma_f16(vl, vd, valphah);
174
175 vst1_lane_f16(o, vo, 0);
176 o += 1;
177 }
178 }
179
180 input_offset += input_increment;
181 } while (--channels != 0);
182 }
183