1 /*
2 * Copyright (c) 2021-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/core/Helpers.h"
25 #include "src/core/NEON/wrapper/wrapper.h"
26 #include "src/core/helpers/ScaleHelpers.h"
27 #include "src/core/utils/ScaleUtils.h"
28 #include "support/Rounding.h"
29
30 #include <arm_neon.h>
31
32 namespace arm_compute
33 {
34 namespace
35 {
u8_neon_scale_nearest(const ITensor * src,ITensor * dst,const ITensor * offsets,float sampling_offset,bool align_corners,const Window & window)36 void u8_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets,
37 float sampling_offset, bool align_corners, const Window &window)
38 {
39 const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
40 const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom;
41 const size_t in_stride_wc = in_stride_w * in_stride_c;
42 const size_t in_dim_h = src->info()->dimension(2);
43
44 // Compute the ratio between source height and destination height
45 const auto hr = scale_utils::calculate_resize_ratio(in_dim_h, dst->info()->dimension(2), align_corners);
46 const auto window_start_x = static_cast<int32_t>(window.x().start());
47 const auto window_end_x = static_cast<int32_t>(window.x().end());
48 const int window_step_x = 16;
49
50 Window win(window);
51 win.set(Window::DimX, Window::Dimension(0, 1, 1));
52 Iterator out(dst, win);
53
54 const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes();
55 const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3];
56
57 execute_window_loop(win, [&](const Coordinates & id)
58 {
59 const int32_t offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c;
60 const auto in_hi = static_cast<int>(align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) : std::floor((id.z() + sampling_offset) * hr));
61 const int offset_row = in_hi * in_stride_wc;
62 int32_t x = window_start_x;
63 const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]);
64
65 for(; x <= window_end_x - window_step_x; x += window_step_x)
66 {
67 wrapper::vstore(reinterpret_cast<uint8_t *>(out.ptr()) + x,
68 wrapper::vloadq(in_ptr + offset + offset_row + x));
69 }
70 for(; x < window_end_x; ++x)
71 {
72 *(reinterpret_cast<uint8_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x);
73 }
74 },
75 out);
76 }
77
u8_neon_scale_bilinear(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)78 void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
79 BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
80 bool align_corners, const Window &window)
81 {
82 // Compute the ratio between source and destination dimensions
83 const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
84 const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
85
86 const int input_width = src->info()->dimension(1);
87 const int input_height = src->info()->dimension(2);
88
89 if(border_mode == BorderMode::CONSTANT)
90 {
91 Iterator out(dst, window);
92 const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
93 const int in_stride_wc = in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom);
94
95 // Don't increment in Y and Z direction for the input tensor
96 // A pointer to the start of this plane is needed as base for the precomputed offsets
97 Window win_in(window);
98 win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
99 win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
100 Iterator in(src, win_in);
101
102 const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
103 execute_window_loop(window, [&](const Coordinates & id)
104 {
105 const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
106 const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
107 const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
108 const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset);
109 const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc;
110
111 const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) ? *in_ptr : const_border_value;
112 const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) ? *(in_ptr + in_stride_c) : const_border_value;
113 const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
114 const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
115
116 *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
117 },
118 in, out);
119 }
120 else if(border_mode == BorderMode::REPLICATE)
121 {
122 using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
123
124 const int in_stride_x = src->info()->strides_in_bytes()[1];
125 const int in_stride_y = src->info()->strides_in_bytes()[2];
126 const int in_stride_b = src->info()->strides_in_bytes()[3];
127 const int out_stride_x = dst->info()->strides_in_bytes()[1];
128 const int out_stride_y = dst->info()->strides_in_bytes()[2];
129 const int out_stride_b = dst->info()->strides_in_bytes()[3];
130
131 const int out_dim_ch = dst->info()->dimension(0);
132 constexpr int step_cout = 16;
133
134 Window window_execution = window;
135 window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
136 Window win_in_out(window);
137 win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
138 win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
139 Iterator in(src, win_in_out);
140 Iterator out(dst, win_in_out);
141
142 const int xo_start = window_execution[1].start();
143 const int xo_end = window_execution[1].end();
144 const int xo_step = window_execution[1].step();
145 const int yo_start = window_execution[2].start();
146 const int yo_end = window_execution[2].end();
147 const int yo_step = window_execution[2].step();
148 const int bo_start = window_execution[3].start();
149 const int bo_end = window_execution[3].end();
150 const int bo_step = window_execution[3].step();
151
152 const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
153 const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
154
155 for(int bo = bo_start; bo < bo_end; bo += bo_step)
156 {
157 const uint8_t *in_ptr = in.ptr() + bo * in_stride_b;
158 uint8_t *out_ptr = out.ptr() + bo * out_stride_b;
159
160 for(int yo = yo_start; yo < yo_end; yo += yo_step)
161 {
162 // Floating-point coordinate
163 const float yi_f = yo * scale_y + fp_coord_offset_y;
164 // Integer coordinate
165 const int yi = static_cast<int>(std::floor(yi_f));
166 // Weight for the y coordinate
167 const float a1 = (yi_f - static_cast<float>(yi));
168 const float b1 = (1.f - a1);
169
170 const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
171 const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
172
173 const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
174 const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
175
176 uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
177 for(int xo = xo_start; xo < xo_end; xo += xo_step)
178 {
179 // Floating-point coordinate
180 const float xi_f = xo * scale_x + fp_coord_offset_x;
181 // Integer coordinate
182 const int xi = static_cast<int>(std::floor(xi_f));
183 // Weight for the x coordinate
184 const float a = (xi_f - static_cast<float>(xi));
185 const float b = (1.f - a);
186
187 const float s00_s = b * b1;
188 const float s01_s = a * b1;
189 const float s10_s = b * a1;
190 const float s11_s = a * a1;
191
192 const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{});
193 const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{});
194 const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{});
195 const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{});
196
197 const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
198 const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
199
200 const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
201 const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
202 const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
203 const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
204
205 uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
206
207 int cout = 0;
208 for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
209 {
210 const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
211 const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
212 const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
213 const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
214
215 const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
216 const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
217
218 const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low)));
219 const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low)));
220 const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high)));
221 const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high)));
222
223 const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
224 const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
225
226 const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low)));
227 const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low)));
228 const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high)));
229 const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high)));
230
231 const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
232 const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
233
234 const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low)));
235 const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low)));
236 const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high)));
237 const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high)));
238
239 const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
240 const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
241
242 const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low)));
243 const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low)));
244 const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high)));
245 const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high)));
246
247 auto out_0 = wrapper::vmul(in00_0, s00);
248 out_0 = wrapper::vmla(out_0, in01_0, s01);
249 out_0 = wrapper::vmla(out_0, in10_0, s10);
250 out_0 = wrapper::vmla(out_0, in11_0, s11);
251
252 auto out_1 = wrapper::vmul(in00_1, s00);
253 out_1 = wrapper::vmla(out_1, in01_1, s01);
254 out_1 = wrapper::vmla(out_1, in10_1, s10);
255 out_1 = wrapper::vmla(out_1, in11_1, s11);
256
257 auto out_2 = wrapper::vmul(in00_2, s00);
258 out_2 = wrapper::vmla(out_2, in01_2, s01);
259 out_2 = wrapper::vmla(out_2, in10_2, s10);
260 out_2 = wrapper::vmla(out_2, in11_2, s11);
261
262 auto out_3 = wrapper::vmul(in00_3, s00);
263 out_3 = wrapper::vmla(out_3, in01_3, s01);
264 out_3 = wrapper::vmla(out_3, in10_3, s10);
265 out_3 = wrapper::vmla(out_3, in11_3, s11);
266
267 #if defined(__aarch64__) && !defined(BARE_METAL)
268 const auto out_0_int = wrapper::vcvta<uint32_t>(out_0);
269 const auto out_1_int = wrapper::vcvta<uint32_t>(out_1);
270 const auto out_2_int = wrapper::vcvta<uint32_t>(out_2);
271 const auto out_3_int = wrapper::vcvta<uint32_t>(out_3);
272 #else // defined(__aarch64__) && !defined(BARE_METAL)
273 const auto out_0_int = wrapper::vcvt<uint32_t>(out_0);
274 const auto out_1_int = wrapper::vcvt<uint32_t>(out_1);
275 const auto out_2_int = wrapper::vcvt<uint32_t>(out_2);
276 const auto out_3_int = wrapper::vcvt<uint32_t>(out_3);
277 #endif // defined(__aarch64__) && !defined(BARE_METAL)
278 const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
279 const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
280 const auto out = wrapper::vcombine(low_part, high_part);
281
282 wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out);
283 }
284
285 for(; cout < out_dim_ch; ++cout)
286 {
287 const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
288 const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
289 const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
290 const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
291
292 float out0 = in00 * s00_s;
293 out0 += in01 * s01_s;
294 out0 += in10 * s10_s;
295 out0 += in11 * s11_s;
296
297 // Rounding modes of vector and scalar loops should match
298 #if defined(__aarch64__) && !defined(BARE_METAL)
299 *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(std::round(out0));
300 #else // defined(__aarch64__) && !defined(BARE_METAL)
301 *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(out0);
302 #endif // defined(__aarch64__) && !defined(BARE_METAL)
303 }
304 }
305 }
306 }
307 }
308 else
309 {
310 ARM_COMPUTE_ERROR("Not implemented");
311 }
312 }
313
s8_neon_scale_bilinear(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)314 void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
315 BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
316 bool align_corners, const Window &window)
317 {
318 ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value);
319 if(border_mode == BorderMode::REPLICATE)
320 {
321 using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
322
323 // Compute the ratio between source and destination dimensions
324 const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
325 const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
326
327 const int in_stride_x = src->info()->strides_in_bytes()[1];
328 const int in_stride_y = src->info()->strides_in_bytes()[2];
329 const int in_stride_b = src->info()->strides_in_bytes()[3];
330 const int out_stride_x = dst->info()->strides_in_bytes()[1];
331 const int out_stride_y = dst->info()->strides_in_bytes()[2];
332 const int out_stride_b = dst->info()->strides_in_bytes()[3];
333 const int input_width = src->info()->dimension(1);
334 const int input_height = src->info()->dimension(2);
335 const int out_dim_ch = dst->info()->dimension(0);
336 constexpr int step_cout = 16;
337
338 Window window_execution = window;
339 window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
340 Window win_in_out(window);
341 win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
342 win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
343 Iterator in(src, win_in_out);
344 Iterator out(dst, win_in_out);
345
346 const int xo_start = window_execution[1].start();
347 const int xo_end = window_execution[1].end();
348 const int xo_step = window_execution[1].step();
349 const int yo_start = window_execution[2].start();
350 const int yo_end = window_execution[2].end();
351 const int yo_step = window_execution[2].step();
352 const int bo_start = window_execution[3].start();
353 const int bo_end = window_execution[3].end();
354 const int bo_step = window_execution[3].step();
355
356 const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
357 const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
358
359 for(int bo = bo_start; bo < bo_end; bo += bo_step)
360 {
361 const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b);
362 int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b);
363
364 for(int yo = yo_start; yo < yo_end; yo += yo_step)
365 {
366 // Floating-point coordinate
367 const float yi_f = yo * scale_y + fp_coord_offset_y;
368 // Integer coordinate
369 const int yi = static_cast<int>(std::floor(yi_f));
370 // Weight for the y coordinate
371 const float a1 = (yi_f - static_cast<float>(yi));
372 const float b1 = (1.f - a1);
373
374 const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
375 const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
376
377 const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
378 const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
379
380 int8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
381 for(int xo = xo_start; xo < xo_end; xo += xo_step)
382 {
383 // Floating-point coordinate
384 const float xi_f = xo * scale_x + fp_coord_offset_x;
385 // Integer coordinate
386 const int xi = static_cast<int>(std::floor(xi_f));
387 // Weight for the x coordinate
388 const float a = (xi_f - static_cast<float>(xi));
389 const float b = (1.f - a);
390
391 const float s00_s = b * b1;
392 const float s01_s = a * b1;
393 const float s10_s = b * a1;
394 const float s11_s = a * a1;
395
396 const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{});
397 const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{});
398 const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{});
399 const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{});
400
401 const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
402 const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
403
404 const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
405 const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
406 const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
407 const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
408
409 int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
410
411 int cout = 0;
412 for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
413 {
414 const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
415 const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
416 const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
417 const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
418
419 const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
420 const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
421
422 const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low)));
423 const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low)));
424 const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high)));
425 const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high)));
426
427 const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
428 const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
429
430 const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low)));
431 const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low)));
432 const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high)));
433 const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high)));
434
435 const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
436 const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
437
438 const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low)));
439 const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low)));
440 const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high)));
441 const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high)));
442
443 const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
444 const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
445
446 const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low)));
447 const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low)));
448 const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high)));
449 const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high)));
450
451 auto out_0 = wrapper::vmul(in00_0, s00);
452 out_0 = wrapper::vmla(out_0, in01_0, s01);
453 out_0 = wrapper::vmla(out_0, in10_0, s10);
454 out_0 = wrapper::vmla(out_0, in11_0, s11);
455
456 auto out_1 = wrapper::vmul(in00_1, s00);
457 out_1 = wrapper::vmla(out_1, in01_1, s01);
458 out_1 = wrapper::vmla(out_1, in10_1, s10);
459 out_1 = wrapper::vmla(out_1, in11_1, s11);
460
461 auto out_2 = wrapper::vmul(in00_2, s00);
462 out_2 = wrapper::vmla(out_2, in01_2, s01);
463 out_2 = wrapper::vmla(out_2, in10_2, s10);
464 out_2 = wrapper::vmla(out_2, in11_2, s11);
465
466 auto out_3 = wrapper::vmul(in00_3, s00);
467 out_3 = wrapper::vmla(out_3, in01_3, s01);
468 out_3 = wrapper::vmla(out_3, in10_3, s10);
469 out_3 = wrapper::vmla(out_3, in11_3, s11);
470
471 #if defined(__aarch64__) && !defined(BARE_METAL)
472 const auto out_0_int = wrapper::vcvta<int32_t>(out_0);
473 const auto out_1_int = wrapper::vcvta<int32_t>(out_1);
474 const auto out_2_int = wrapper::vcvta<int32_t>(out_2);
475 const auto out_3_int = wrapper::vcvta<int32_t>(out_3);
476 #else // defined(__aarch64__) && !defined(BARE_METAL)
477 const auto out_0_int = wrapper::vcvt<int32_t>(out_0);
478 const auto out_1_int = wrapper::vcvt<int32_t>(out_1);
479 const auto out_2_int = wrapper::vcvt<int32_t>(out_2);
480 const auto out_3_int = wrapper::vcvt<int32_t>(out_3);
481 #endif // defined(__aarch64__) && !defined(BARE_METAL)
482 const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
483 const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
484 const auto out = wrapper::vcombine(low_part, high_part);
485
486 wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out);
487 }
488
489 for(; cout < out_dim_ch; ++cout)
490 {
491 const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
492 const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
493 const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
494 const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
495
496 float out0 = in00 * s00_s;
497 out0 += in01 * s01_s;
498 out0 += in10 * s10_s;
499 out0 += in11 * s11_s;
500
501 // Rounding modes of vector and scalar loops should match
502 #if defined(__aarch64__) && !defined(BARE_METAL)
503 *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(std::round(out0));
504 #else // defined(__aarch64__) && !defined(BARE_METAL)
505 *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(out0);
506 #endif // defined(__aarch64__) && !defined(BARE_METAL)
507 }
508 }
509 }
510 }
511 }
512 else
513 {
514 ARM_COMPUTE_ERROR("Not implemented");
515 }
516 }
517
s16_neon_scale_nearest(const ITensor * src,ITensor * dst,const ITensor * offsets,float sampling_offset,bool align_corners,const Window & window)518 void s16_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets,
519 float sampling_offset, bool align_corners, const Window &window)
520 {
521 const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
522 const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom;
523 const size_t in_stride_wc = in_stride_w * in_stride_c;
524 const size_t in_dim_h = src->info()->dimension(2);
525
526 // Compute the ratio between source height and destination height
527 const auto hr = scale_utils::calculate_resize_ratio(in_dim_h, dst->info()->dimension(2), align_corners);
528 const auto window_start_x = static_cast<int32_t>(window.x().start());
529 const auto window_end_x = static_cast<int32_t>(window.x().end());
530 const int window_step_x = 8;
531
532 Window win(window);
533 win.set(Window::DimX, Window::Dimension(0, 1, 1));
534 Iterator out(dst, win);
535
536 const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes();
537 const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3];
538
539 execute_window_loop(win, [&](const Coordinates & id)
540 {
541 const int32_t offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c;
542 const auto in_hi = static_cast<int>(align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) : std::floor((id.z() + sampling_offset) * hr));
543 const int offset_row = in_hi * in_stride_wc;
544 int32_t x = window_start_x;
545 const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]);
546
547 for(; x <= window_end_x - window_step_x; x += window_step_x)
548 {
549 wrapper::vstore(reinterpret_cast<int16_t *>(out.ptr()) + x,
550 wrapper::vloadq(in_ptr + offset + offset_row + x));
551 }
552 for(; x < window_end_x; ++x)
553 {
554 *(reinterpret_cast<int16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x);
555 }
556 },
557 out);
558 }
559
s16_neon_scale_bilinear(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)560 void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
561 BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
562 bool align_corners, const Window &window)
563 {
564 // Compute the ratio between source height and destination height
565 const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
566
567 Iterator out(dst, window);
568 const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
569 const int in_dim_w = src->info()->dimension(1);
570 const int in_dim_h = src->info()->dimension(2);
571 const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom);
572
573 // Don't increment in Y and Z direction for the input tensor
574 // A pointer to the start of this plane is needed as base for the precomputed offsets
575 Window win_in(window);
576 win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
577 win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
578 Iterator in(src, win_in);
579
580 if(border_mode == BorderMode::CONSTANT)
581 {
582 const int16_t const_border_value = static_cast<int16_t>(constant_border_value.get<int16_t>());
583 execute_window_loop(window, [&](const Coordinates & id)
584 {
585 const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
586 const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
587 const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
588 const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
589 const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc;
590
591 const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value;
592 const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value;
593 const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
594 const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
595
596 *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
597 },
598 in, out);
599 }
600 else if(border_mode == BorderMode::REPLICATE)
601 {
602 execute_window_loop(window, [&](const Coordinates & id)
603 {
604 const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
605 const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
606 const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
607 const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
608
609 const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
610 const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
611 const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
612 const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
613
614 const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc);
615 const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc);
616 const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc);
617 const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc);
618
619 *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
620 },
621 in, out);
622 }
623 else
624 {
625 ARM_COMPUTE_ERROR("Not implemented");
626 }
627 }
628 }
629 namespace cpu
630 {
s8_neon_scale(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,InterpolationPolicy policy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)631 void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
632 InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
633 bool align_corners, const Window &window)
634 {
635 if(policy == InterpolationPolicy::BILINEAR)
636 {
637 s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
638 }
639 else
640 {
641 ARM_COMPUTE_ERROR("Not implemented");
642 }
643 }
644
u8_neon_scale(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,InterpolationPolicy policy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)645 void u8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
646 InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
647 bool align_corners, const Window &window)
648 {
649 if(policy == InterpolationPolicy::BILINEAR)
650 {
651 u8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
652 }
653 else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
654 {
655 u8_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
656 }
657 }
658
s16_neon_scale(const ITensor * src,ITensor * dst,const ITensor * offsets,const ITensor * dx,const ITensor * dy,InterpolationPolicy policy,BorderMode border_mode,PixelValue constant_border_value,float sampling_offset,bool align_corners,const Window & window)659 void s16_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
660 InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
661 bool align_corners, const Window &window)
662 {
663 if(policy == InterpolationPolicy::BILINEAR)
664 {
665 s16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
666 }
667 else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
668 {
669 s16_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
670 }
671 }
672 } // namespace cpu
673 } // namespace arm_compute