1 /*
2 * Copyright (c) 2017-2022 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/gpu/cl/kernels/ClGemmLowpReductionKernel.h"
25
26 #include "arm_compute/core/CL/CLHelpers.h"
27 #include "arm_compute/core/CL/ICLTensor.h"
28 #include "arm_compute/core/KernelDescriptors.h"
29
30 #include "src/core/helpers/AutoConfiguration.h"
31 #include "src/core/helpers/WindowHelpers.h"
32
33 #include "support/Cast.h"
34 #include "support/StringSupport.h"
35
36 namespace arm_compute
37 {
38 namespace opencl
39 {
40 namespace kernels
41 {
42 namespace
43 {
validate_arguments_matrix_a_reduction(const ITensorInfo * src,const ITensorInfo * dst)44 Status validate_arguments_matrix_a_reduction(const ITensorInfo *src, const ITensorInfo *dst)
45 {
46 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src, dst);
47 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QSYMM8);
48
49 if(dst->total_size() > 0)
50 {
51 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(dst, 1, DataType::S32);
52 ARM_COMPUTE_RETURN_ERROR_ON_MSG(dst->dimension(0) != src->dimension(1), "Output vector must have length equal to the number of rows of the input matrix");
53 }
54 return Status{};
55 }
56
validate_arguments_matrix_b_reduction(const ITensorInfo * src,const ITensorInfo * dst)57 Status validate_arguments_matrix_b_reduction(const ITensorInfo *src, const ITensorInfo *dst)
58 {
59 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(src, dst);
60 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(src, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QSYMM8, DataType::QSYMM8_PER_CHANNEL);
61
62 if(dst->total_size() > 0)
63 {
64 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(dst, 1, DataType::S32);
65 ARM_COMPUTE_RETURN_ERROR_ON_MSG(dst->dimension(0) != src->dimension(0), "Output vector must have length equal to the number of columns of the input matrix");
66 }
67 return Status{};
68 }
69 } // namespace
70
IClGemmLowpReductionKernel()71 IClGemmLowpReductionKernel::IClGemmLowpReductionKernel()
72 {
73 _type = CLKernelType::ELEMENTWISE;
74 }
75
configure(const CLCompileContext & compile_context,const ITensorInfo * mtx_a,ITensorInfo * vector_sum_row,const GEMMLowpReductionKernelInfo & info)76 void ClGemmLowpMatrixAReductionKernel::configure(const CLCompileContext &compile_context, const ITensorInfo *mtx_a, ITensorInfo *vector_sum_row, const GEMMLowpReductionKernelInfo &info)
77 {
78 // Perform validate step
79 ARM_COMPUTE_ERROR_ON_NULLPTR(mtx_a, vector_sum_row);
80 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments_matrix_a_reduction(mtx_a, vector_sum_row));
81
82 // Output auto initialization if not yet initialized
83 auto_init_if_empty(*vector_sum_row, TensorShape(mtx_a->dimension(1)), 1, DataType::S32);
84
85 auto padding_info = get_padding_info({ mtx_a, vector_sum_row });
86
87 // Set the arguments to pass at compile time
88 CLBuildOptions build_opts;
89 build_opts.add_option("-DCOLS_A=" + support::cpp11::to_string(mtx_a->dimension(0)));
90 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(mtx_a->data_type()));
91 build_opts.add_option("-DACC_DATA_TYPE=" + get_cl_dot8_acc_type_from_data_type(mtx_a->data_type()));
92 build_opts.add_option_if(info.mul_by_scalar, "-DSCALAR=" + support::cpp11::to_string(info.scalar));
93
94 const bool is_dot8_supported = dot8_supported(CLKernelLibrary::get().get_device());
95
96 std::string kernel_name = "gemmlowp_matrix_a_reduction" + std::string(is_dot8_supported ? "_dot8" : "");
97
98 // A macro guard to compile ONLY the kernel of interest
99 build_opts.add_option("-D" + upper_string(kernel_name));
100
101 // Create kernel
102 _kernel = create_kernel(compile_context, kernel_name, build_opts.options());
103
104 // Configure kernel window
105 // This kernel does not need padding
106 Window win = calculate_max_window(*vector_sum_row, Steps());
107 ICLKernel::configure_internal(win);
108
109 _config_id = kernel_name;
110 _config_id += "_";
111 _config_id += support::cpp11::to_string(mtx_a->dimension(0));
112 _config_id += "_";
113 _config_id += support::cpp11::to_string(mtx_a->dimension(1));
114 _config_id += "_";
115 _config_id += support::cpp11::to_string(mtx_a->dimension(2));
116
117 ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));
118 }
119
validate(const ITensorInfo * mtx_a,const ITensorInfo * vector_sum_row,const GEMMLowpReductionKernelInfo & info)120 Status ClGemmLowpMatrixAReductionKernel::validate(const ITensorInfo *mtx_a, const ITensorInfo *vector_sum_row, const GEMMLowpReductionKernelInfo &info)
121 {
122 ARM_COMPUTE_UNUSED(info);
123 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_matrix_a_reduction(mtx_a, vector_sum_row));
124
125 return Status{};
126 }
127
run_op(ITensorPack & tensors,const Window & window,cl::CommandQueue & queue)128 void ClGemmLowpMatrixAReductionKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)
129 {
130 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
131 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);
132
133 const auto src = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC));
134 auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));
135
136 Window collapsed = window.collapse_if_possible(ICLKernel::window(), Window::DimY);
137 Window slice_in = collapsed.first_slice_window_2D();
138 Window slice_out = collapsed.first_slice_window_2D();
139
140 // Setup input slice. Its dimensions are increased in the cl kernel.
141 slice_in.set(Window::DimX, Window::Dimension(0, 0, 0));
142 slice_in.set(Window::DimY, Window::Dimension(0, 0, 0));
143 slice_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
144
145 do
146 {
147 unsigned int idx = 0;
148 add_3D_tensor_argument(idx, src, slice_in);
149 add_2D_tensor_argument(idx, dst, slice_out);
150 enqueue(queue, *this, slice_out, lws_hint());
151 }
152 while(collapsed.slide_window_slice_2D(slice_out));
153 }
154
configure(const CLCompileContext & compile_context,const ITensorInfo * mtx_b,ITensorInfo * vector_sum_col,const GEMMLowpReductionKernelInfo & info)155 void ClGemmLowpMatrixBReductionKernel::configure(const CLCompileContext &compile_context, const ITensorInfo *mtx_b, ITensorInfo *vector_sum_col, const GEMMLowpReductionKernelInfo &info)
156 {
157 ARM_COMPUTE_ERROR_ON_NULLPTR(mtx_b, vector_sum_col);
158 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments_matrix_b_reduction(mtx_b, vector_sum_col));
159
160 // Output auto initialization if not yet initialized
161 auto_init_if_empty(*vector_sum_col, TensorShape(mtx_b->dimension(0)), 1, DataType::S32);
162
163 auto padding_info = get_padding_info({ mtx_b, vector_sum_col });
164
165 const unsigned int num_elems_processed_per_iteration = adjust_vec_size(16, mtx_b->dimension(0));
166
167 // Set the arguments to pass at compile time
168 CLBuildOptions build_opts;
169 build_opts.add_option("-DVEC_SIZE=" + support::cpp11::to_string(num_elems_processed_per_iteration));
170 build_opts.add_option("-DVEC_SIZE_LEFTOVER=" + support::cpp11::to_string(mtx_b->dimension(0) % num_elems_processed_per_iteration));
171 build_opts.add_option("-DCOLS_B=" + support::cpp11::to_string(mtx_b->dimension(0)));
172 build_opts.add_option("-DROWS_B=" + support::cpp11::to_string(mtx_b->dimension(1)));
173 build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(mtx_b->data_type()));
174 build_opts.add_option("-DACC_DATA_TYPE=" + get_cl_dot8_acc_type_from_data_type(mtx_b->data_type()));
175 build_opts.add_option_if(info.mul_by_scalar, "-DSCALAR=" + support::cpp11::to_string(info.scalar));
176
177 const std::string kernel_name = "gemmlowp_matrix_b_reduction";
178
179 // A macro guard to compile ONLY the kernel of interest
180 build_opts.add_option("-D" + upper_string(kernel_name));
181
182 // Create kernel
183 _kernel = create_kernel(compile_context, kernel_name, build_opts.options());
184
185 // Configure kernel window
186 Window win = calculate_max_window(*vector_sum_col, Steps(num_elems_processed_per_iteration));
187 IClKernel::configure_internal(win);
188
189 ARM_COMPUTE_ERROR_ON(has_padding_changed(padding_info));
190 }
191
validate(const ITensorInfo * mtx_b,const ITensorInfo * vector_sum_col,const GEMMLowpReductionKernelInfo & info)192 Status ClGemmLowpMatrixBReductionKernel::validate(const ITensorInfo *mtx_b, const ITensorInfo *vector_sum_col, const GEMMLowpReductionKernelInfo &info)
193 {
194 ARM_COMPUTE_UNUSED(info);
195 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments_matrix_b_reduction(mtx_b, vector_sum_col));
196
197 return Status{};
198 }
199
run_op(ITensorPack & tensors,const Window & window,cl::CommandQueue & queue)200 void ClGemmLowpMatrixBReductionKernel::run_op(ITensorPack &tensors, const Window &window, cl::CommandQueue &queue)
201 {
202 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
203 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICLKernel::window(), window);
204
205 const auto src = utils::cast::polymorphic_downcast<const ICLTensor *>(tensors.get_const_tensor(TensorType::ACL_SRC));
206 auto dst = utils::cast::polymorphic_downcast<ICLTensor *>(tensors.get_tensor(TensorType::ACL_DST));
207
208 Window collapsed = window.collapse_if_possible(IKernel::window(), Window::DimY);
209
210 Window slice_out = collapsed.first_slice_window_2D();
211 Window slice_in = slice_out;
212
213 slice_in.set(Window::DimY, Window::Dimension(0, 0, 0));
214 slice_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
215
216 do
217 {
218 unsigned int idx = 0;
219 add_3D_tensor_argument(idx, src, slice_in);
220 add_2D_tensor_argument(idx, dst, slice_out);
221 enqueue(queue, *this, slice_out, lws_hint());
222 }
223 while(collapsed.slide_window_slice_2D(slice_out));
224 }
225 } // namespace kernels
226 } // namespace opencl
227 } // namespace arm_compute
228