1 /*
2 * Copyright (c) 2017-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/cpu/kernels/CpuQuantizeKernel.h"
25
26 #include "arm_compute/core/Error.h"
27 #include "arm_compute/core/Helpers.h"
28 #include "arm_compute/core/Utils.h"
29 #include "arm_compute/core/Validate.h"
30 #include "arm_compute/core/Window.h"
31 #include "src/core/NEON/NEAsymm.h"
32 #include "src/core/NEON/NEMath.h"
33 #include "src/core/NEON/wrapper/wrapper.h"
34 #include "src/core/helpers/AutoConfiguration.h"
35 #include "src/core/helpers/WindowHelpers.h"
36
37 #include "src/core/CPP/Validate.h"
38
39 #include <arm_neon.h>
40 #include <map>
41
42 namespace arm_compute
43 {
44 namespace cpu
45 {
46 namespace kernels
47 {
48 namespace
49 {
50 constexpr auto window_step = 16;
51
validate_arguments(const ITensorInfo * src,const ITensorInfo * dst)52 Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dst)
53 {
54 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src, dst);
55 ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(src);
56 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32);
57 ARM_COMPUTE_RETURN_ERROR_ON(dst->tensor_shape().total_size() == 0);
58 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(dst, 1, DataType::QSYMM8, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QASYMM16);
59 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(src, dst);
60
61 return Status{};
62 }
63
64 template <typename T>
load_value(const T * input_ptr)65 inline float32x4x4_t load_value(const T *input_ptr)
66 {
67 using Tx16_t = typename wrapper::traits::neon_vector<T, 16>::type;
68 return arm_compute::convert_to_float32x4x4<Tx16_t>(wrapper::vloadq(input_ptr));
69 }
70
71 template <>
load_value(const float * input_ptr)72 inline float32x4x4_t load_value(const float *input_ptr)
73 {
74 return { wrapper::vloadq(input_ptr),
75 wrapper::vloadq(input_ptr + 4),
76 wrapper::vloadq(input_ptr + 8),
77 wrapper::vloadq(input_ptr + 12) };
78 }
79 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
80 template <>
load_value(const float16_t * input_ptr)81 inline float32x4x4_t load_value(const float16_t *input_ptr)
82 {
83 return { vcvt_f32_f16(wrapper::vload(input_ptr)),
84 vcvt_f32_f16(wrapper::vload(input_ptr + 4)),
85 vcvt_f32_f16(wrapper::vload(input_ptr + 8)),
86 vcvt_f32_f16(wrapper::vload(input_ptr + 12)) };
87 }
88
89 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
90
91 template <typename element_type>
92 using vector_type = wrapper::traits::neon_vector_t<element_type, window_step>;
93
94 template <typename quantized_type>
95 vector_type<quantized_type> vquantize_qasymm8(const float32x4x4_t &qv, const UniformQuantizationInfo &qi);
96
97 template <>
vquantize_qasymm8(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)98 vector_type<uint8_t> vquantize_qasymm8<uint8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
99 {
100 return vquantize(qv, qi);
101 }
102
103 template <>
vquantize_qasymm8(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)104 vector_type<int8_t> vquantize_qasymm8<int8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
105 {
106 return vquantize_signed(qv, qi);
107 }
108
109 } // namespace
110
configure(const ITensorInfo * src,ITensorInfo * dst)111 void CpuQuantizeKernel::configure(const ITensorInfo *src, ITensorInfo *dst)
112 {
113 ARM_COMPUTE_ERROR_ON_NULLPTR(src, dst);
114 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src, dst));
115
116 static const std::map<std::string, QuantizeFunctionExecutorPtr> quant_map =
117 {
118 { "op_QASYMM8_QASYMM8", &CpuQuantizeKernel::run_quantize_qasymm8<uint8_t, uint8_t> },
119 { "op_QASYMM8_QASYMM8_SIGNED", &CpuQuantizeKernel::run_quantize_qasymm8<uint8_t, int8_t> },
120 { "op_QASYMM8_QASYMM16", &CpuQuantizeKernel::run_quantize_qasymm16<uint8_t> },
121
122 { "op_QASYMM8_SIGNED_QASYMM8", &CpuQuantizeKernel::run_quantize_qasymm8<int8_t, uint8_t> },
123 { "op_QASYMM8_SIGNED_QASYMM8_SIGNED", &CpuQuantizeKernel::run_quantize_qasymm8<int8_t, int8_t> },
124 { "op_QASYMM8_SIGNED_QASYMM16", &CpuQuantizeKernel::run_quantize_qasymm16<int8_t> },
125
126 { "op_F32_QSYMM8", &CpuQuantizeKernel::run_quantize_qsymm8<float, int8_t> },
127
128 { "op_F32_QASYMM8", &CpuQuantizeKernel::run_quantize_qasymm8<float, uint8_t> },
129 { "op_F32_QASYMM8_SIGNED", &CpuQuantizeKernel::run_quantize_qasymm8<float, int8_t> },
130 { "op_F32_QASYMM16", &CpuQuantizeKernel::run_quantize_qasymm16<float> },
131
132 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
133 { "op_F16_QASYMM8", &CpuQuantizeKernel::run_quantize_qasymm8<float16_t, uint8_t> },
134 { "op_F16_QASYMM8_SIGNED", &CpuQuantizeKernel::run_quantize_qasymm8<float16_t, int8_t> },
135 { "op_F16_QASYMM16", &CpuQuantizeKernel::run_quantize_qasymm16<float16_t> },
136 #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC*/
137 };
138
139 std::string function_to_call("op_");
140 function_to_call += string_from_data_type(src->data_type()) + "_";
141 function_to_call += string_from_data_type(dst->data_type());
142
143 auto it = quant_map.find(function_to_call);
144
145 if(it == quant_map.end())
146 {
147 ARM_COMPUTE_ERROR("Unsupported combination of input and output data types");
148 }
149 _func = it->second;
150
151 // Configure kernel window
152 Window win_config = calculate_max_window(*src, Steps());
153 ICpuKernel::configure(win_config);
154 }
155
validate(const ITensorInfo * src,const ITensorInfo * dst)156 Status CpuQuantizeKernel::validate(const ITensorInfo *src, const ITensorInfo *dst)
157 {
158 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src, dst));
159 return Status{};
160 }
161
162 template <typename TIn, typename TOut>
run_quantize_qsymm8(const ITensor * src,ITensor * dst,const Window & window)163 void CpuQuantizeKernel::run_quantize_qsymm8(const ITensor *src, ITensor *dst, const Window &window)
164 {
165 const auto window_start_x = static_cast<int>(window.x().start());
166 const auto window_end_x = static_cast<int>(window.x().end());
167
168 const UniformQuantizationInfo uqinfo_in = src->info()->quantization_info().uniform();
169 UniformQuantizationInfo uqinfo = dst->info()->quantization_info().uniform();
170 if(is_data_type_quantized_asymmetric(src->info()->data_type()))
171 {
172 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
173 }
174 // Collapse window and reset first dimension to handle tail calculations manually
175 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
176 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
177
178 Iterator input(src, win_collapsed);
179 Iterator output(dst, win_collapsed);
180 execute_window_loop(win_collapsed, [&](const Coordinates &)
181 {
182 auto input_ptr = reinterpret_cast<const TIn *>(input.ptr());
183 auto output_ptr = reinterpret_cast<TOut *>(output.ptr());
184 int x = window_start_x;
185 for(; x <= (window_end_x - window_step); x += window_step)
186 {
187 wrapper::vstore(&output_ptr[x], vquantize_qasymm8<TOut>(load_value(&input_ptr[x]), uqinfo));
188 }
189 // Compute left-over elements
190 for(; x < window_end_x; ++x)
191 {
192 output_ptr[x] = quantize_qsymm8(input_ptr[x], dst->info()->quantization_info());
193 }
194 },
195 input, output);
196 }
197
198 template <typename TIn, typename TOut>
run_quantize_qasymm8(const ITensor * src,ITensor * dst,const Window & window)199 void CpuQuantizeKernel::run_quantize_qasymm8(const ITensor *src, ITensor *dst, const Window &window)
200 {
201 const auto window_start_x = static_cast<int>(window.x().start());
202 const auto window_end_x = static_cast<int>(window.x().end());
203
204 const UniformQuantizationInfo uqinfo_in = src->info()->quantization_info().uniform();
205 UniformQuantizationInfo uqinfo = dst->info()->quantization_info().uniform();
206 if(is_data_type_quantized_asymmetric(src->info()->data_type()))
207 {
208 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
209 }
210 #ifdef __aarch64__
211 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
212 #else //__aarch64__
213 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
214 #endif //__aarch64__
215
216 // Collapse window and reset first dimension to handle tail calculations manually
217 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
218 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
219
220 Iterator input(src, win_collapsed);
221 Iterator output(dst, win_collapsed);
222 execute_window_loop(win_collapsed, [&](const Coordinates &)
223 {
224 auto input_ptr = reinterpret_cast<const TIn *>(input.ptr());
225 auto output_ptr = reinterpret_cast<TOut *>(output.ptr());
226
227 int x = window_start_x;
228 for(; x <= (window_end_x - window_step); x += window_step)
229 {
230 wrapper::vstore(&output_ptr[x], vquantize_qasymm8<TOut>(load_value(&input_ptr[x]), uqinfo));
231 }
232 // Compute left-over elements
233 for(; x < window_end_x; ++x)
234 {
235 output_ptr[x] = Qasymm8QuantizationHelper<TOut>::quantize(input_ptr[x], uqinfo, rounding_policy);
236 }
237 },
238 input, output);
239 }
240
241 template <typename T>
run_quantize_qasymm16(const ITensor * src,ITensor * dst,const Window & window)242 void CpuQuantizeKernel::run_quantize_qasymm16(const ITensor *src, ITensor *dst, const Window &window)
243 {
244 const auto window_start_x = static_cast<int>(window.x().start());
245 const auto window_end_x = static_cast<int>(window.x().end());
246
247 const UniformQuantizationInfo uqinfo_in = src->info()->quantization_info().uniform();
248 UniformQuantizationInfo uqinfo = dst->info()->quantization_info().uniform();
249 if(is_data_type_quantized_asymmetric(src->info()->data_type()))
250 {
251 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
252 }
253 #ifdef __aarch64__
254 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
255 #else //__aarch64__
256 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
257 #endif //__aarch64__
258
259 // Collapse window and reset first dimension to handle tail calculations manually
260 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
261 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
262
263 Iterator input(src, win_collapsed);
264 Iterator output(dst, win_collapsed);
265 execute_window_loop(win_collapsed, [&](const Coordinates &)
266 {
267 auto input_ptr = reinterpret_cast<const T *>(input.ptr());
268 auto output_ptr = reinterpret_cast<uint16_t *>(output.ptr());
269
270 int x = window_start_x;
271 for(; x <= (window_end_x - window_step); x += window_step)
272 {
273 uint16x8x2_t tmp = vquantize_qasymm16(load_value(&input_ptr[x]), uqinfo);
274 vst1q_u16(&output_ptr[x], tmp.val[0]);
275 vst1q_u16(&output_ptr[x + 8], tmp.val[1]);
276 }
277 // Compute left-over elements
278 for(; x < window_end_x; ++x)
279 {
280 output_ptr[x] = quantize_qasymm16(input_ptr[x], uqinfo, rounding_policy);
281 }
282 },
283 input, output);
284 }
285
run_op(ITensorPack & tensors,const Window & window,const ThreadInfo & info)286 void CpuQuantizeKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info)
287 {
288 ARM_COMPUTE_UNUSED(info);
289 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
290 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICpuKernel::window(), window);
291 ARM_COMPUTE_ERROR_ON(_func == nullptr);
292
293 const auto src = tensors.get_const_tensor(TensorType::ACL_SRC);
294 auto dst = tensors.get_tensor(TensorType::ACL_DST);
295 (this->*_func)(src, dst, window);
296 }
297
name() const298 const char *CpuQuantizeKernel::name() const
299 {
300 return "CpuQuantizeKernel";
301 }
302 } // namespace kernels
303 } // namespace cpu
304 } // namespace arm_compute
305