1 /*
2 * Copyright (c) 2021-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/helpers/ScaleHelpers.h"
25 #include "src/cpu/kernels/scale/neon/list.h"
26
27 namespace arm_compute
28 {
29 namespace
30 {
qasymm8_neon_scale_bilinear(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)31 void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
32 BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
33 bool align_corners, const Window &window)
34 {
35 // Data layout is NHWC
36 const int32_t input_width = src->info()->dimension(1);
37 const int32_t input_height = src->info()->dimension(2);
38
39 const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
40 const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
41
42 // Compute the ratio between source and destination dimensions
43 const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
44 const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
45
46 if(border_mode == BorderMode::CONSTANT)
47 {
48 const int32_t in_stride_y = src->info()->strides_in_bytes()[1];
49 const int32_t in_stride_z = src->info()->strides_in_bytes()[2];
50
51 // Compute the ratio between source height and destination height
52 Window win_off;
53 win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
54 win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
55
56 // Don't increment in X and Y direction for the input tensor
57 // A pointer to the start of this plane is needed as base for the precomputed offsets
58 Window win_in(window);
59 win_in.set(1, Window::Dimension(0, 0, 0));
60 win_in.set(2, Window::Dimension(0, 0, 0));
61
62 for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
63 {
64 win_off.set(d, Window::Dimension(0, 0, 0));
65 }
66
67 Iterator in(src, win_in);
68 Iterator out(dst, window);
69
70 const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
71 execute_window_loop(window, [&](const Coordinates & id)
72 {
73 const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset);
74 const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
75 const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
76 const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
77 const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr());
78
79 const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ?
80 (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) :
81 const_border_value;
82 const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ?
83 (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) :
84 const_border_value;
85 const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ?
86 (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) :
87 const_border_value;
88 const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ?
89 (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) :
90 const_border_value;
91
92 const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info);
93 const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info);
94 const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info);
95 const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info);
96 *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
97 },
98 in, out);
99 }
100 else if(border_mode == BorderMode::REPLICATE)
101 {
102 using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
103 using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>;
104
105 const int in_stride_x = src->info()->strides_in_bytes()[1];
106 const int in_stride_y = src->info()->strides_in_bytes()[2];
107 const int in_stride_b = src->info()->strides_in_bytes()[3];
108 const int out_stride_x = dst->info()->strides_in_bytes()[1];
109 const int out_stride_y = dst->info()->strides_in_bytes()[2];
110 const int out_stride_b = dst->info()->strides_in_bytes()[3];
111 const int out_dim_ch = dst->info()->dimension(0);
112 constexpr int step_cout = 16;
113
114 Window window_execution = window;
115 window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
116 Window win_in_out(window);
117 win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
118 win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
119 Iterator in(src, win_in_out);
120 Iterator out(dst, win_in_out);
121
122 const int xo_start = window_execution[1].start();
123 const int xo_end = window_execution[1].end();
124 const int xo_step = window_execution[1].step();
125 const int yo_start = window_execution[2].start();
126 const int yo_end = window_execution[2].end();
127 const int yo_step = window_execution[2].step();
128 const int bo_start = window_execution[3].start();
129 const int bo_end = window_execution[3].end();
130 const int bo_step = window_execution[3].step();
131
132 const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
133 const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
134
135 const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{});
136 const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32
137
138 const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{});
139 const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset);
140
141 const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
142 const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
143
144 for(int bo = bo_start; bo < bo_end; bo += bo_step)
145 {
146 const uint8_t *in_ptr = in.ptr() + bo * in_stride_b;
147 uint8_t *out_ptr = out.ptr() + bo * out_stride_b;
148
149 for(int yo = yo_start; yo < yo_end; yo += yo_step)
150 {
151 // Floating-point coordinate
152 const float yi_f = yo * scale_y + fp_coord_offset_y;
153 // Integer coordinate
154 const int yi = static_cast<int>(std::floor(yi_f));
155 // Weight for the y coordinate
156 const float a1 = (yi_f - static_cast<float>(yi));
157 const float b1 = (1.f - a1);
158
159 const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
160 const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
161
162 const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
163 const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
164
165 uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
166 for(int xo = xo_start; xo < xo_end; xo += xo_step)
167 {
168 // Floating-point coordinate
169 const float xi_f = xo * scale_x + fp_coord_offset_x;
170 // Integer coordinate
171 const int xi = static_cast<int>(std::floor(xi_f));
172 // Weight for the x coordinate
173 const float a = (xi_f - static_cast<float>(xi));
174 const float b = (1.f - a);
175
176 const float s00_s = b * b1;
177 const float s01_s = a * b1;
178 const float s10_s = b * a1;
179 const float s11_s = a * a1;
180
181 const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{});
182 const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{});
183 const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{});
184 const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{});
185
186 const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
187 const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
188
189 const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
190 const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
191 const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
192 const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
193
194 uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
195
196 int cout = 0;
197 for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
198 {
199 const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
200 const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
201 const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
202 const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
203
204 const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
205 const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
206
207 const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), vscale_in);
208 const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), vscale_in);
209 const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), vscale_in);
210 const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), vscale_in);
211
212 const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
213 const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
214
215 const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), vscale_in);
216 const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), vscale_in);
217 const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), vscale_in);
218 const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), vscale_in);
219
220 const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
221 const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
222
223 const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), vscale_in);
224 const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), vscale_in);
225 const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), vscale_in);
226 const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), vscale_in);
227
228 const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
229 const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
230
231 const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), vscale_in);
232 const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), vscale_in);
233 const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), vscale_in);
234 const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), vscale_in);
235
236 auto out_0 = wrapper::vmul(in00_0, s00);
237 out_0 = wrapper::vmla(out_0, in01_0, s01);
238 out_0 = wrapper::vmla(out_0, in10_0, s10);
239 out_0 = wrapper::vmla(out_0, in11_0, s11);
240
241 auto out_1 = wrapper::vmul(in00_1, s00);
242 out_1 = wrapper::vmla(out_1, in01_1, s01);
243 out_1 = wrapper::vmla(out_1, in10_1, s10);
244 out_1 = wrapper::vmla(out_1, in11_1, s11);
245
246 auto out_2 = wrapper::vmul(in00_2, s00);
247 out_2 = wrapper::vmla(out_2, in01_2, s01);
248 out_2 = wrapper::vmla(out_2, in10_2, s10);
249 out_2 = wrapper::vmla(out_2, in11_2, s11);
250
251 auto out_3 = wrapper::vmul(in00_3, s00);
252 out_3 = wrapper::vmla(out_3, in01_3, s01);
253 out_3 = wrapper::vmla(out_3, in10_3, s10);
254 out_3 = wrapper::vmla(out_3, in11_3, s11);
255
256 #if defined(__aarch64__) && !defined(BARE_METAL)
257 const auto out_0_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
258 const auto out_1_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
259 const auto out_2_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
260 const auto out_3_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
261 #else // defined(__aarch64__) && !defined(BARE_METAL)
262 const auto out_0_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
263 const auto out_1_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
264 const auto out_2_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
265 const auto out_3_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
266 #endif // defined(__aarch64__) && !defined(BARE_METAL)
267 const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
268 const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
269 const auto out = wrapper::vcombine(low_part, high_part);
270
271 wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out);
272 }
273
274 for(; cout < out_dim_ch; ++cout)
275 {
276 const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
277 const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
278 const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
279 const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
280
281 const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale;
282 const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale;
283 const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale;
284 const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale;
285
286 float out = in00_f * s00_s;
287 out += in01_f * s01_s;
288 out += in10_f * s10_s;
289 out += in11_f * s11_s;
290
291 // Rounding modes of vector and scalar loops should match
292 #if defined(__aarch64__) && !defined(BARE_METAL)
293 *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info);
294 #else // defined(__aarch64__) && !defined(BARE_METAL)
295 *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO);
296 #endif // defined(__aarch64__) && !defined(BARE_METAL)
297 }
298 }
299 }
300 }
301 }
302 else
303 {
304 ARM_COMPUTE_ERROR("Not implemented");
305 }
306 }
307 }
308 namespace cpu
309 {
qasymm8_neon_scale(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,InterpolationPolicy policy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)310 void qasymm8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
311 InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
312 bool align_corners, const Window &window)
313 {
314 if(policy == InterpolationPolicy::BILINEAR)
315 {
316 if(src->info()->quantization_info() == dst->info()->quantization_info())
317 {
318 u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window);
319 }
320 else
321 {
322 qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
323 }
324 }
325 else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
326 {
327 nearest_neon_scale<uint8_t>(src, dst, offsets, sampling_offset, align_corners, window);
328 }
329 }
330 } // namespace cpu
331 } // namespace arm_compute