xref: /aosp_15_r20/external/ComputeLibrary/src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.cpp (revision c217d954acce2dbc11938adb493fc0abd69584f3) 
1 /*
2  * Copyright (c) 2021-2022 Arm Limited.
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to
8  * deal in the Software without restriction, including without limitation the
9  * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10  * sell copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #include "src/cpu/kernels/internal/CpuPool2dAssemblyWrapperKernel.h"
25 #include "arm_compute/core/Utils.h"
26 #include "arm_compute/core/Validate.h"
27 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
28 #include "arm_compute/core/utils/quantization/AsymmHelpers.h"
29 #include "src/core/CPP/Validate.h"
30 #include "src/core/NEON/INEKernel.h"
31 #include "src/core/helpers/AutoConfiguration.h"
32 #include "src/core/helpers/WindowHelpers.h"
33 
34 #include <arm_neon.h>
35 
36 namespace arm_compute
37 {
38 namespace cpu
39 {
40 namespace kernels
41 {
42 using namespace arm_compute::misc::shape_calculator;
43 
configure(const ITensorInfo * src,ITensorInfo * dst,const PoolingLayerInfo & info,const CPUInfo & cpu_info)44 void CpuPool2dAssemblyWrapperKernel::configure(const ITensorInfo *src, ITensorInfo *dst, const PoolingLayerInfo &info, const CPUInfo &cpu_info)
45 {
46     ARM_COMPUTE_UNUSED(cpu_info);
47     ARM_COMPUTE_ERROR_ON_NULLPTR(src, dst);
48 
49     // dst initialization if not yet initialized
50     auto_init_if_empty(*dst, src->clone()->set_tensor_shape(compute_pool_shape(*src, info)));
51 
52 #if defined(__aarch64__)
53     const bool requantize = src->quantization_info() != dst->quantization_info();
54 
55     switch(src->data_type())
56     {
57         case DataType::QASYMM8:
58             if(requantize)
59             {
60                 create_arm_pooling_requant<uint8_t, uint8_t>(src, dst, info, cpu_info);
61             }
62             else
63             {
64                 create_arm_pooling<uint8_t, uint8_t>(src, dst, info, cpu_info);
65             }
66             break;
67         case DataType::QASYMM8_SIGNED:
68             if(requantize)
69             {
70                 create_arm_pooling_requant<int8_t, int8_t>(src, dst, info, cpu_info);
71             }
72             else
73             {
74                 create_arm_pooling<int8_t, int8_t>(src, dst, info, cpu_info);
75             }
76             break;
77 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
78         case DataType::F16:
79             create_arm_pooling<float16_t, float16_t>(src, dst, info, cpu_info);
80             break;
81 #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */
82         case DataType::F32:
83             create_arm_pooling<float, float>(src, dst, info, cpu_info);
84             break;
85         default:
86             break;
87     }
88 #endif // defined(__aarch64__)
89 
90     Window win = calculate_max_window(*dst, Steps());
91     INEKernel::configure(win);
92 }
93 
validate(const ITensorInfo * src,const ITensorInfo * dst,const PoolingLayerInfo & info)94 Status CpuPool2dAssemblyWrapperKernel::validate(const ITensorInfo *src, const ITensorInfo *dst, const PoolingLayerInfo &info)
95 {
96     ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src, dst);
97 
98 #ifndef __aarch64__
99     ARM_COMPUTE_RETURN_ERROR_MSG("32-bit is not supported by assembly kernels");
100 #endif /* __aarch64__ */
101     ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(src);
102     ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32);
103     ARM_COMPUTE_RETURN_ERROR_ON_MSG((src->data_layout() != DataLayout::NHWC) || (info.data_layout != DataLayout::NHWC), "Only NHWC is supported by assembly kernels");
104     ARM_COMPUTE_RETURN_ERROR_ON_MSG((info.pool_type != PoolingType::AVG) && (info.pool_type != PoolingType::MAX),
105                                     "Only AVG and MAX pooling are supported by assembly kernels");
106 
107     ARM_COMPUTE_RETURN_ERROR_ON_MSG(is_pool_region_entirely_outside_input(info), "Pooling region that is entirely outside input tensor is unsupported by assembly kernels");
108 
109     if(dst->total_size() > 0)
110     {
111         ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(src, dst);
112 
113         const auto src_qinfo = src->quantization_info().uniform();
114         const auto dst_qinfo = dst->quantization_info().uniform();
115 
116         if(src_qinfo != dst_qinfo)
117         {
118             const float multiplier = src_qinfo.scale / dst_qinfo.scale;
119             int32_t     dst_multiplier{};
120             int32_t     dst_shift{};
121             ARM_COMPUTE_RETURN_ERROR_ON(quantization::calculate_quantized_multiplier(multiplier, &dst_multiplier, &dst_shift));
122         }
123         else
124         {
125             if(src->data_type() == DataType::QASYMM8)
126             {
127                 const bool has_padding = info.pad_stride_info.has_padding();
128                 ARM_COMPUTE_RETURN_ERROR_ON_MSG(!info.exclude_padding && has_padding, "Assembly kernels do not support padding for QASYMM8 with same src/dst quantization info");
129             }
130         }
131     }
132     else
133     {
134         if(src->data_type() == DataType::QASYMM8)
135         {
136             // If dst is not configured, the quantization info are the same
137             const bool has_padding = info.pad_stride_info.has_padding();
138             ARM_COMPUTE_RETURN_ERROR_ON_MSG(!info.exclude_padding && has_padding, "Assembly kernels do not support padding for QASYMM8 with same src/dst quantization info");
139         }
140     }
141     return Status{};
142 }
143 
run_op(ITensorPack & tensors,const Window & window,const ThreadInfo & info)144 void CpuPool2dAssemblyWrapperKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info)
145 {
146     ARM_COMPUTE_ERROR_ON_NULLPTR(_kernel_asm.get());
147     ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
148     ARM_COMPUTE_UNUSED(window);
149     ARM_COMPUTE_UNUSED(info);
150 
151     ARM_COMPUTE_ERROR_ON(tensors.empty());
152 
153     const ITensor *src       = tensors.get_const_tensor(TensorType::ACL_SRC);
154     ITensor       *dst       = tensors.get_tensor(TensorType::ACL_DST);
155     ITensor       *workspace = tensors.get_tensor(TensorType::ACL_INT_0);
156 
157     const auto in_ptr        = src->buffer() + src->info()->offset_first_element_in_bytes();
158     auto       out_ptr       = dst->buffer() + dst->info()->offset_first_element_in_bytes();
159     auto       working_space = (workspace == nullptr) ? nullptr : workspace->buffer() + workspace->info()->offset_first_element_in_bytes();
160 
161     const auto src_shape   = src->info()->tensor_shape();
162     const auto dst_shape   = dst->info()->tensor_shape();
163     const auto src_padding = src->info()->padding();
164     const auto dst_padding = dst->info()->padding();
165 
166     const size_t ld_src_col   = src_shape[0] + src_padding.left + src_padding.right;
167     const size_t ld_src_row   = ld_src_col * (src_shape[1] + src_padding.top + src_padding.bottom);
168     const size_t ld_src_batch = ld_src_row * src_shape[2];
169     const size_t ld_dst_col   = dst_shape[0] + dst_padding.left + dst_padding.right;
170     const size_t ld_dst_row   = ld_dst_col * (dst_shape[1] + dst_padding.top + dst_padding.bottom);
171     const size_t ld_dst_batch = ld_dst_row * dst_shape[2];
172 
173     _kernel_asm->execute(in_ptr, ld_src_col, ld_src_row, ld_src_batch,
174                          out_ptr, ld_dst_col, ld_dst_row, ld_dst_batch,
175                          working_space, info.thread_id, info.num_threads);
176 }
177 
get_working_size(unsigned int num_threads) const178 size_t CpuPool2dAssemblyWrapperKernel::get_working_size(unsigned int num_threads) const
179 {
180     return _kernel_asm->get_working_size(num_threads);
181 }
182 
is_configured() const183 bool CpuPool2dAssemblyWrapperKernel::is_configured() const
184 {
185     return _kernel_asm != nullptr;
186 }
187 
188 template <typename Typesrc, typename Typedst>
create_arm_pooling(const ITensorInfo * src,ITensorInfo * dst,const PoolingLayerInfo & info,const CPUInfo & cpu_info)189 void CpuPool2dAssemblyWrapperKernel::create_arm_pooling(const ITensorInfo *src, ITensorInfo *dst, const PoolingLayerInfo &info, const CPUInfo &cpu_info)
190 {
191     const arm_conv::pooling::PoolingType pool_type = (info.pool_type == PoolingType::AVG) ? arm_conv::pooling::PoolingType::AVERAGE : arm_conv::pooling::PoolingType::MAX;
192 
193     arm_conv::pooling::PoolingWindow window{};
194     window.cols = static_cast<unsigned int>(info.pool_size.x());
195     window.rows = static_cast<unsigned int>(info.pool_size.y());
196 
197     arm_conv::pooling::PoolingStride stride{};
198     std::tie(stride.cols, stride.rows) = info.pad_stride_info.stride();
199 
200     const arm_conv::pooling::PaddingValues padding{ info.pad_stride_info.pad_left(), info.pad_stride_info.pad_top(), info.pad_stride_info.pad_right(), info.pad_stride_info.pad_bottom() };
201 
202     constexpr unsigned int idx_width    = 1;
203     constexpr unsigned int idx_height   = 2;
204     constexpr unsigned int idx_channels = 0;
205     constexpr unsigned int idx_batches  = 3;
206 
207     const unsigned int n_batches  = src->dimension(idx_batches);
208     const unsigned int src_rows   = src->dimension(idx_height);
209     const unsigned int src_cols   = src->dimension(idx_width);
210     const unsigned int n_channels = src->dimension(idx_channels);
211     const unsigned int dst_rows   = dst->dimension(idx_height);
212     const unsigned int dst_cols   = dst->dimension(idx_width);
213 
214     arm_conv::pooling::PoolingArgs args(&cpu_info, pool_type, window, stride, info.exclude_padding, n_batches, src_rows, src_cols, n_channels, dst_rows, dst_cols, padding, nullptr);
215 
216     // Configure assembly pooling kernel
217     auto pooling_kernel_asm = arm_conv::pooling::pooling<Typesrc, Typedst>(args);
218     if(pooling_kernel_asm == nullptr)
219     {
220         // Configuration not supported: Leave function unconfigured:
221         return;
222     }
223 
224     _kernel_asm = std::move(pooling_kernel_asm);
225 }
226 
227 template <typename Typesrc, typename Typedst>
create_arm_pooling_requant(const ITensorInfo * src,ITensorInfo * dst,const PoolingLayerInfo & info,const CPUInfo & cpu_info)228 void CpuPool2dAssemblyWrapperKernel::create_arm_pooling_requant(const ITensorInfo *src, ITensorInfo *dst, const PoolingLayerInfo &info, const CPUInfo &cpu_info)
229 {
230     const arm_conv::pooling::PoolingType pool_type = (info.pool_type == PoolingType::AVG) ? arm_conv::pooling::PoolingType::AVERAGE : arm_conv::pooling::PoolingType::MAX;
231 
232     arm_conv::pooling::PoolingWindow window{};
233     window.cols = static_cast<unsigned int>(info.pool_size.x());
234     window.rows = static_cast<unsigned int>(info.pool_size.y());
235 
236     arm_conv::pooling::PoolingStride stride{};
237     std::tie(stride.cols, stride.rows) = info.pad_stride_info.stride();
238 
239     const arm_conv::pooling::PaddingValues padding{ info.pad_stride_info.pad_left(), info.pad_stride_info.pad_top(), info.pad_stride_info.pad_right(), info.pad_stride_info.pad_bottom() };
240 
241     constexpr unsigned int idx_width    = 1;
242     constexpr unsigned int idx_height   = 2;
243     constexpr unsigned int idx_channels = 0;
244     constexpr unsigned int idx_batches  = 3;
245 
246     const unsigned int n_batches  = src->dimension(idx_batches);
247     const unsigned int src_rows   = src->dimension(idx_height);
248     const unsigned int src_cols   = src->dimension(idx_width);
249     const unsigned int n_channels = src->dimension(idx_channels);
250     const unsigned int dst_rows   = dst->dimension(idx_height);
251     const unsigned int dst_cols   = dst->dimension(idx_width);
252 
253     arm_conv::pooling::PoolingArgs args(&cpu_info, pool_type, window, stride, info.exclude_padding, n_batches, src_rows, src_cols, n_channels, dst_rows, dst_cols, padding, nullptr);
254 
255     const auto src_qinfo = src->quantization_info().uniform();
256     const auto dst_qinfo = dst->quantization_info().uniform();
257 
258     const float multiplier = src_qinfo.scale / dst_qinfo.scale;
259     int32_t     dst_multiplier{};
260     int32_t     dst_shift{};
261     quantization::calculate_quantized_multiplier(multiplier, &dst_multiplier, &dst_shift);
262 
263     const arm_conv::pooling::Requantize32 requant_args(src_qinfo.offset,
264                                                        dst_qinfo.offset,
265                                                        dst_shift, // left shift
266                                                        0,         // right shift
267                                                        dst_multiplier);
268 
269     // Configure assembly pooling kernel with requantization
270     auto pooling_kernel_asm = arm_conv::pooling::pooling<Typesrc, Typedst, arm_conv::pooling::Requantize32>(args, requant_args);
271     if(pooling_kernel_asm == nullptr)
272     {
273         // Configuration not supported: Leave function unconfigured:
274         return;
275     }
276 
277     _kernel_asm = std::move(pooling_kernel_asm);
278 }
279 
get_mws(const CPUInfo & platform,size_t thread_count) const280 size_t CpuPool2dAssemblyWrapperKernel::get_mws(const CPUInfo &platform, size_t thread_count) const
281 {
282     ARM_COMPUTE_UNUSED(thread_count);
283     ARM_COMPUTE_UNUSED(platform);
284 
285     return ICPPKernel::default_mws;
286 }
287 } // namespace kernels
288 } // namespace cpu
289 } // namespace arm_compute
290