1 /*
2 * Copyright (c) 2021-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/core/Helpers.h"
25 #include "arm_compute/core/ITensor.h"
26 #include "arm_compute/core/Types.h"
27 #include "arm_compute/core/utils/misc/Traits.h"
28 #include "src/core/NEON/wrapper/intrinsics/intrinsics.h"
29 #include "src/core/helpers/WindowHelpers.h"
30 #include "src/cpu/kernels/pool2d/neon/list.h"
31
32 namespace arm_compute
33 {
34 namespace cpu
35 {
36 namespace
37 {
pooling2_f32_maxpool_indices(const ITensor * src,ITensor * dst0,ITensor * dst1,PoolingLayerInfo & pool_info,const Window & window_src,const Window & window)38 void pooling2_f32_maxpool_indices(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
39 {
40 const int window_start_x = window.x().start();
41 const int window_end_x = window.x().end();
42 const int window_step_x = 4;
43
44 Window window_out = window;
45 window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
46
47 Iterator in(src, window_src);
48 Iterator out(dst0, window_out);
49 Iterator indices(dst1, window_out);
50
51 const int pool_pad_top = pool_info.pad_stride_info.pad_top();
52 const int pool_pad_left = pool_info.pad_stride_info.pad_left();
53
54 int pool_stride_x = 0;
55 int pool_stride_y = 0;
56 std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
57
58 float32x4_t vres;
59 float res;
60
61 const int pad_right = src->info()->padding().right;
62 const int pad_left = src->info()->padding().left;
63 const int pad_horizontal = pad_right + pad_left;
64 const int in_stride_y = static_cast<int>(src->info()->strides_in_bytes().y());
65 const int in_stride_z = static_cast<int>(src->info()->strides_in_bytes().z());
66
67 execute_window_loop(window_out, [&](const Coordinates & id)
68 {
69 const int idx_width = id.y() * pool_stride_x;
70 const int idx_height = id.z() * pool_stride_y;
71 const int pool_limit_y = pool_pad_top - idx_height;
72 const int pool_limit_x = pool_pad_left - idx_width;
73
74 const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
75 const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
76
77 const int in_x0_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>(src->info()->strides_in_bytes().z());
78 const int in_x1_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y - pool_pad_top) * static_cast<int>
79 (src->info()->strides_in_bytes().z());
80 const int in_x2_offset = (pool_start_x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
81 (src->info()->strides_in_bytes().z());
82 const int in_x3_offset = (pool_start_x + 1 - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (pool_start_y + 1 - pool_pad_top) * static_cast<int>
83 (src->info()->strides_in_bytes().z());
84
85 int x_off = window_start_x;
86 for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
87 {
88 const auto in_x0_ptr = reinterpret_cast<const float *>(in.ptr() + in_x0_offset);
89 const auto in_x1_ptr = reinterpret_cast<const float *>(in.ptr() + in_x1_offset);
90 const auto in_x2_ptr = reinterpret_cast<const float *>(in.ptr() + in_x2_offset);
91 const auto in_x3_ptr = reinterpret_cast<const float *>(in.ptr() + in_x3_offset);
92 const auto v_x0 = vld1q_f32(in_x0_ptr + x_off);
93 const auto v_x1 = vld1q_f32(in_x1_ptr + x_off);
94 const auto v_x2 = vld1q_f32(in_x2_ptr + x_off);
95 const auto v_x3 = vld1q_f32(in_x3_ptr + x_off);
96 vres = vmaxq_f32(vmaxq_f32(v_x2, v_x3), vmaxq_f32(v_x0, v_x1));
97 // Store result
98 vst1q_f32(reinterpret_cast<float *>(out.ptr()) + x_off, vres);
99
100 const uint32_t offset_base = offset_no_padding<float>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
101 const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float) + x_off;
102 const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float) - pad_horizontal;
103 const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float) - pad_horizontal * src->info()->tensor_shape()[1];
104 const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float) - pad_horizontal;
105 const uint32x4_t voffset_x0 = { offset_x0, offset_x0 + 1, offset_x0 + 2, offset_x0 + 3 };
106 const uint32x4_t voffset_x1 = { offset_x1, offset_x1 + 1, offset_x1 + 2, offset_x1 + 3 };
107 const uint32x4_t voffset_x2 = { offset_x2, offset_x2 + 1, offset_x2 + 2, offset_x2 + 3 };
108 const uint32x4_t voffset_x3 = { offset_x3, offset_x3 + 1, offset_x3 + 2, offset_x3 + 3 };
109 const uint32x4_t tmp_indices0 = vbslq_u32(vcgeq_f32(v_x0, v_x1), voffset_x0, voffset_x1);
110 const uint32x4_t tmp_indices1 = vbslq_u32(vcgeq_f32(v_x2, v_x3), voffset_x2, voffset_x3);
111 const uint32x4_t tmp_indices2 = vbslq_u32(vcgeq_f32(vmaxq_f32(v_x0, v_x1), vmaxq_f32(v_x2, v_x3)), tmp_indices0, tmp_indices1);
112
113 // Store indices
114 vst1q_u32(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off, tmp_indices2);
115 }
116
117 // Left-overs loop
118 for(; x_off < window_end_x; ++x_off)
119 {
120 const auto x0 = *(reinterpret_cast<const float *>(in.ptr() + in_x0_offset) + x_off);
121 const auto x1 = *(reinterpret_cast<const float *>(in.ptr() + in_x1_offset) + x_off);
122 const auto x2 = *(reinterpret_cast<const float *>(in.ptr() + in_x2_offset) + x_off);
123 const auto x3 = *(reinterpret_cast<const float *>(in.ptr() + in_x3_offset) + x_off);
124 res = std::max(std::max(x2, x3), std::max(x0, x1));
125
126 // Store result
127 *(reinterpret_cast<float *>(out.ptr()) + x_off) = res;
128
129 const uint32_t offset_base = offset_no_padding<float>(in.offset(), id, *src->info(), pool_stride_x, pool_stride_y, DataLayout::NHWC);
130 const uint32_t offset_x0 = (uint32_t)offset_base / sizeof(float) + x_off;
131 const uint32_t offset_x1 = (uint32_t)offset_x0 + in_stride_y / sizeof(float) - pad_horizontal;
132 const uint32_t offset_x2 = (uint32_t)offset_x0 + in_stride_z / sizeof(float) - pad_horizontal * src->info()->tensor_shape()[1];
133 const uint32_t offset_x3 = (uint32_t)offset_x2 + in_stride_y / sizeof(float) - pad_horizontal;
134 const uint32_t tmp_idx0 = (x0 >= x1) ? offset_x0 : offset_x1;
135 const uint32_t tmp_idx1 = (x2 >= x3) ? offset_x2 : offset_x3;
136 const uint32_t tmp_idx2 = (std::max(x0, x1) >= std::max(x2, x3)) ? tmp_idx0 : tmp_idx1;
137
138 // Store indices
139 *(reinterpret_cast<uint32_t *>(indices.ptr()) + x_off) = tmp_idx2;
140 }
141 },
142 in, out, indices);
143 }
144 }
145
poolingMxN_fp32_neon_nhwc(const ITensor * src,ITensor * dst0,ITensor * dst1,PoolingLayerInfo & pool_info,const Window & window_src,const Window & window)146 void poolingMxN_fp32_neon_nhwc(const ITensor *src, ITensor *dst0, ITensor *dst1, PoolingLayerInfo &pool_info, const Window &window_src, const Window &window)
147 {
148 if(pool_info.pool_size == Size2D(2, 2) && pool_info.pool_type == PoolingType::MAX && dst1)
149 {
150 pooling2_f32_maxpool_indices(src, dst0, dst1, pool_info, window_src, window);
151 }
152 else
153 {
154 const int window_start_x = window.x().start();
155 const int window_end_x = window.x().end();
156 const int window_step_x = 4;
157
158 Window window_out = window;
159 window_out.set(Window::DimX, Window::Dimension(0, 1, 1));
160
161 Iterator in(src, window_src);
162 Iterator out(dst0, window_out);
163
164 const int pool_size_x = pool_info.is_global_pooling ? src->info()->tensor_shape().y() : pool_info.pool_size.width;
165 const int pool_size_y = pool_info.is_global_pooling ? src->info()->tensor_shape().z() : pool_info.pool_size.height;
166 const int pool_pad_right = pool_info.pad_stride_info.pad_right();
167 const int pool_pad_top = pool_info.pad_stride_info.pad_top();
168 const int pool_pad_left = pool_info.pad_stride_info.pad_left();
169 const int pool_pad_bottom = pool_info.pad_stride_info.pad_bottom();
170 int pool_stride_x = 0;
171 int pool_stride_y = 0;
172 std::tie(pool_stride_x, pool_stride_y) = pool_info.pad_stride_info.stride();
173 const int upper_bound_w = src->info()->dimension(1) + (pool_info.exclude_padding ? 0 : pool_pad_right);
174 const int upper_bound_h = src->info()->dimension(2) + (pool_info.exclude_padding ? 0 : pool_pad_bottom);
175
176 float32x4_t vres;
177
178 execute_window_loop(window_out, [&](const Coordinates & id)
179 {
180 const int idx_width = id.y() * pool_stride_x;
181 const int idx_height = id.z() * pool_stride_y;
182 const int pool_limit_y = pool_pad_top - idx_height;
183 const int pool_limit_x = pool_pad_left - idx_width;
184
185 const int pool_start_y = std::max(0, window_src.z().start() + pool_limit_y);
186 const int pool_end_y = std::min(pool_size_y, window_src.z().end() + pool_limit_y);
187 const int pool_start_x = std::max(0, window_src.y().start() + pool_limit_x);
188 const int pool_end_x = std::min(pool_size_x, window_src.y().end() + pool_limit_x);
189
190 int x_off = window_start_x;
191 for(; x_off <= (window_end_x - window_step_x); x_off += window_step_x)
192 {
193 if(pool_info.pool_type != PoolingType::MAX)
194 {
195 // Calculate scale
196 const float scale = calculate_avg_scale_pool2d(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
197 pool_stride_y);
198 const float32x4_t scale_v = vdupq_n_f32(scale);
199
200 // Perform pooling
201 vres = vdupq_n_f32(0.0f);
202
203 for(int y = pool_start_y; y < pool_end_y; ++y)
204 {
205 for(int x = pool_start_x; x < pool_end_x; ++x)
206 {
207 const float32x4_t data = vld1q_f32(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
208 (src->info()->strides_in_bytes().z())) + x_off);
209
210 // Get power of 2 in case of l2 pooling and accumulate
211 if(pool_info.pool_type == PoolingType::L2)
212 {
213 vres = vmlaq_f32(vres, data, data);
214 }
215 else
216 {
217 vres = vaddq_f32(vres, data);
218 }
219 }
220 }
221 // Divide by scale
222 vres = vmulq_f32(vres, scale_v);
223 }
224 else
225 {
226 vres = vdupq_n_f32(-std::numeric_limits<float>::infinity());
227 for(int y = pool_start_y; y < pool_end_y; ++y)
228 {
229 for(int x = pool_start_x; x < pool_end_x; ++x)
230 {
231 const float32x4_t data = vld1q_f32(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
232 (src->info()->strides_in_bytes().z())) + x_off);
233 vres = vmaxq_f32(vres, data);
234 }
235 }
236 }
237
238 // Calculate square-root in case of l2 pooling
239 if(pool_info.pool_type == PoolingType::L2)
240 {
241 float32x4_t l2_res = { static_cast<float>(sqrt(vgetq_lane_f32(vres, 0))),
242 static_cast<float>(sqrt(vgetq_lane_f32(vres, 1))),
243 static_cast<float>(sqrt(vgetq_lane_f32(vres, 2))),
244 static_cast<float>(sqrt(vgetq_lane_f32(vres, 3)))
245 };
246 vres = l2_res;
247 }
248
249 // Store result
250 vst1q_f32(reinterpret_cast<float *>(out.ptr()) + x_off, vres);
251 }
252
253 // Left-overs loop
254 for(; x_off < window_end_x; ++x_off)
255 {
256 float res = 0.0f;
257
258 if(pool_info.pool_type != PoolingType::MAX)
259 {
260 // Calculate scale
261 const float scale = calculate_avg_scale_pool2d(pool_info.exclude_padding, DataLayout::NHWC, id, pool_size_x, pool_size_y, upper_bound_w, upper_bound_h, pool_pad_left, pool_pad_top, pool_stride_x,
262 pool_stride_y);
263
264 for(int y = pool_start_y; y < pool_end_y; ++y)
265 {
266 for(int x = pool_start_x; x < pool_end_x; ++x)
267 {
268 const float data = *(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
269 (src->info()->strides_in_bytes().z())) + x_off);
270
271 // Get power of 2 in case of l2 pooling and accumulate
272 if(pool_info.pool_type == PoolingType::L2)
273 {
274 res += data * data;
275 }
276 else
277 {
278 res += data;
279 }
280 }
281 }
282
283 // Divide by scale
284 res *= scale;
285 }
286 else
287 {
288 res = -std::numeric_limits<float>::infinity();
289 for(int y = pool_start_y; y < pool_end_y; ++y)
290 {
291 for(int x = pool_start_x; x < pool_end_x; ++x)
292 {
293 const float data = *(reinterpret_cast<const float *>(in.ptr() + (x - pool_pad_left) * static_cast<int>(src->info()->strides_in_bytes().y()) + (y - pool_pad_top) * static_cast<int>
294 (src->info()->strides_in_bytes().z())) + x_off);
295 res = std::max(res, data);
296 }
297 }
298 }
299
300 // Calculate square-root in case of l2 pooling
301 if(pool_info.pool_type == PoolingType::L2)
302 {
303 res = std::sqrt(res);
304 }
305
306 // Store result
307 *(reinterpret_cast<float *>(out.ptr()) + x_off) = res;
308 }
309 },
310 in, out);
311 }
312 }
313 } // namespace cpu
314 } // namespace arm_compute