1 /*
2 * Copyright (c) 2016-2021 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "arm_compute/runtime/NEON/NEFunctions.h"
25
26 #include "arm_compute/core/Types.h"
27 #include "arm_compute/runtime/Allocator.h"
28 #include "arm_compute/runtime/BlobLifetimeManager.h"
29 #include "arm_compute/runtime/MemoryManagerOnDemand.h"
30 #include "arm_compute/runtime/PoolManager.h"
31 #include "utils/Utils.h"
32
33 using namespace arm_compute;
34 using namespace utils;
35
36 class NEONCNNExample : public Example
37 {
38 public:
do_setup(int argc,char ** argv)39 bool do_setup(int argc, char **argv) override
40 {
41 ARM_COMPUTE_UNUSED(argc);
42 ARM_COMPUTE_UNUSED(argv);
43
44 // Create memory manager components
45 // We need 2 memory managers: 1 for handling the tensors within the functions (mm_layers) and 1 for handling the input and output tensors of the functions (mm_transitions))
46 auto lifetime_mgr0 = std::make_shared<BlobLifetimeManager>(); // Create lifetime manager
47 auto lifetime_mgr1 = std::make_shared<BlobLifetimeManager>(); // Create lifetime manager
48 auto pool_mgr0 = std::make_shared<PoolManager>(); // Create pool manager
49 auto pool_mgr1 = std::make_shared<PoolManager>(); // Create pool manager
50 auto mm_layers = std::make_shared<MemoryManagerOnDemand>(lifetime_mgr0, pool_mgr0); // Create the memory manager
51 auto mm_transitions = std::make_shared<MemoryManagerOnDemand>(lifetime_mgr1, pool_mgr1); // Create the memory manager
52
53 // The weights and biases tensors should be initialized with the values inferred with the training
54
55 // Set memory manager where allowed to manage internal memory requirements
56 conv0 = std::make_unique<NEConvolutionLayer>(mm_layers);
57 conv1 = std::make_unique<NEConvolutionLayer>(mm_layers);
58 fc0 = std::make_unique<NEFullyConnectedLayer>(mm_layers);
59 softmax = std::make_unique<NESoftmaxLayer>(mm_layers);
60
61 /* [Initialize tensors] */
62
63 // Initialize src tensor
64 constexpr unsigned int width_src_image = 32;
65 constexpr unsigned int height_src_image = 32;
66 constexpr unsigned int ifm_src_img = 1;
67
68 const TensorShape src_shape(width_src_image, height_src_image, ifm_src_img);
69 src.allocator()->init(TensorInfo(src_shape, 1, DataType::F32));
70
71 // Initialize tensors of conv0
72 constexpr unsigned int kernel_x_conv0 = 5;
73 constexpr unsigned int kernel_y_conv0 = 5;
74 constexpr unsigned int ofm_conv0 = 8;
75
76 const TensorShape weights_shape_conv0(kernel_x_conv0, kernel_y_conv0, src_shape.z(), ofm_conv0);
77 const TensorShape biases_shape_conv0(weights_shape_conv0[3]);
78 const TensorShape out_shape_conv0(src_shape.x(), src_shape.y(), weights_shape_conv0[3]);
79
80 weights0.allocator()->init(TensorInfo(weights_shape_conv0, 1, DataType::F32));
81 biases0.allocator()->init(TensorInfo(biases_shape_conv0, 1, DataType::F32));
82 out_conv0.allocator()->init(TensorInfo(out_shape_conv0, 1, DataType::F32));
83
84 // Initialize tensor of act0
85 out_act0.allocator()->init(TensorInfo(out_shape_conv0, 1, DataType::F32));
86
87 // Initialize tensor of pool0
88 TensorShape out_shape_pool0 = out_shape_conv0;
89 out_shape_pool0.set(0, out_shape_pool0.x() / 2);
90 out_shape_pool0.set(1, out_shape_pool0.y() / 2);
91 out_pool0.allocator()->init(TensorInfo(out_shape_pool0, 1, DataType::F32));
92
93 // Initialize tensors of conv1
94 constexpr unsigned int kernel_x_conv1 = 3;
95 constexpr unsigned int kernel_y_conv1 = 3;
96 constexpr unsigned int ofm_conv1 = 16;
97
98 const TensorShape weights_shape_conv1(kernel_x_conv1, kernel_y_conv1, out_shape_pool0.z(), ofm_conv1);
99
100 const TensorShape biases_shape_conv1(weights_shape_conv1[3]);
101 const TensorShape out_shape_conv1(out_shape_pool0.x(), out_shape_pool0.y(), weights_shape_conv1[3]);
102
103 weights1.allocator()->init(TensorInfo(weights_shape_conv1, 1, DataType::F32));
104 biases1.allocator()->init(TensorInfo(biases_shape_conv1, 1, DataType::F32));
105 out_conv1.allocator()->init(TensorInfo(out_shape_conv1, 1, DataType::F32));
106
107 // Initialize tensor of act1
108 out_act1.allocator()->init(TensorInfo(out_shape_conv1, 1, DataType::F32));
109
110 // Initialize tensor of pool1
111 TensorShape out_shape_pool1 = out_shape_conv1;
112 out_shape_pool1.set(0, out_shape_pool1.x() / 2);
113 out_shape_pool1.set(1, out_shape_pool1.y() / 2);
114 out_pool1.allocator()->init(TensorInfo(out_shape_pool1, 1, DataType::F32));
115
116 // Initialize tensor of fc0
117 constexpr unsigned int num_labels = 128;
118
119 const TensorShape weights_shape_fc0(out_shape_pool1.x() * out_shape_pool1.y() * out_shape_pool1.z(), num_labels);
120 const TensorShape biases_shape_fc0(num_labels);
121 const TensorShape out_shape_fc0(num_labels);
122
123 weights2.allocator()->init(TensorInfo(weights_shape_fc0, 1, DataType::F32));
124 biases2.allocator()->init(TensorInfo(biases_shape_fc0, 1, DataType::F32));
125 out_fc0.allocator()->init(TensorInfo(out_shape_fc0, 1, DataType::F32));
126
127 // Initialize tensor of act2
128 out_act2.allocator()->init(TensorInfo(out_shape_fc0, 1, DataType::F32));
129
130 // Initialize tensor of softmax
131 const TensorShape out_shape_softmax(out_shape_fc0.x());
132 out_softmax.allocator()->init(TensorInfo(out_shape_softmax, 1, DataType::F32));
133
134 constexpr auto data_layout = DataLayout::NCHW;
135
136 /* -----------------------End: [Initialize tensors] */
137
138 /* [Configure functions] */
139
140 // in:32x32x1: 5x5 convolution, 8 output features maps (OFM)
141 conv0->configure(&src, &weights0, &biases0, &out_conv0, PadStrideInfo(1 /* stride_x */, 1 /* stride_y */, 2 /* pad_x */, 2 /* pad_y */));
142
143 // in:32x32x8, out:32x32x8, Activation function: relu
144 act0.configure(&out_conv0, &out_act0, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU));
145
146 // in:32x32x8, out:16x16x8 (2x2 pooling), Pool type function: Max
147 pool0.configure(&out_act0, &out_pool0, PoolingLayerInfo(PoolingType::MAX, 2, data_layout, PadStrideInfo(2 /* stride_x */, 2 /* stride_y */)));
148
149 // in:16x16x8: 3x3 convolution, 16 output features maps (OFM)
150 conv1->configure(&out_pool0, &weights1, &biases1, &out_conv1, PadStrideInfo(1 /* stride_x */, 1 /* stride_y */, 1 /* pad_x */, 1 /* pad_y */));
151
152 // in:16x16x16, out:16x16x16, Activation function: relu
153 act1.configure(&out_conv1, &out_act1, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU));
154
155 // in:16x16x16, out:8x8x16 (2x2 pooling), Pool type function: Average
156 pool1.configure(&out_act1, &out_pool1, PoolingLayerInfo(PoolingType::AVG, 2, data_layout, PadStrideInfo(2 /* stride_x */, 2 /* stride_y */)));
157
158 // in:8x8x16, out:128
159 fc0->configure(&out_pool1, &weights2, &biases2, &out_fc0);
160
161 // in:128, out:128, Activation function: relu
162 act2.configure(&out_fc0, &out_act2, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU));
163
164 // in:128, out:128
165 softmax->configure(&out_act2, &out_softmax);
166
167 /* -----------------------End: [Configure functions] */
168
169 /*[ Add tensors to memory manager ]*/
170
171 // We need 2 memory groups for handling the input and output
172 // We call explicitly allocate after manage() in order to avoid overlapping lifetimes
173 memory_group0 = std::make_unique<MemoryGroup>(mm_transitions);
174 memory_group1 = std::make_unique<MemoryGroup>(mm_transitions);
175
176 memory_group0->manage(&out_conv0);
177 out_conv0.allocator()->allocate();
178 memory_group1->manage(&out_act0);
179 out_act0.allocator()->allocate();
180 memory_group0->manage(&out_pool0);
181 out_pool0.allocator()->allocate();
182 memory_group1->manage(&out_conv1);
183 out_conv1.allocator()->allocate();
184 memory_group0->manage(&out_act1);
185 out_act1.allocator()->allocate();
186 memory_group1->manage(&out_pool1);
187 out_pool1.allocator()->allocate();
188 memory_group0->manage(&out_fc0);
189 out_fc0.allocator()->allocate();
190 memory_group1->manage(&out_act2);
191 out_act2.allocator()->allocate();
192 memory_group0->manage(&out_softmax);
193 out_softmax.allocator()->allocate();
194
195 /* -----------------------End: [ Add tensors to memory manager ] */
196
197 /* [Allocate tensors] */
198
199 // Now that the padding requirements are known we can allocate all tensors
200 src.allocator()->allocate();
201 weights0.allocator()->allocate();
202 weights1.allocator()->allocate();
203 weights2.allocator()->allocate();
204 biases0.allocator()->allocate();
205 biases1.allocator()->allocate();
206 biases2.allocator()->allocate();
207
208 /* -----------------------End: [Allocate tensors] */
209
210 // Populate the layers manager. (Validity checks, memory allocations etc)
211 mm_layers->populate(allocator, 1 /* num_pools */);
212
213 // Populate the transitions manager. (Validity checks, memory allocations etc)
214 mm_transitions->populate(allocator, 2 /* num_pools */);
215
216 return true;
217 }
do_run()218 void do_run() override
219 {
220 // Acquire memory for the memory groups
221 memory_group0->acquire();
222 memory_group1->acquire();
223
224 conv0->run();
225 act0.run();
226 pool0.run();
227 conv1->run();
228 act1.run();
229 pool1.run();
230 fc0->run();
231 act2.run();
232 softmax->run();
233
234 // Release memory
235 memory_group0->release();
236 memory_group1->release();
237 }
238
239 private:
240 // The src tensor should contain the input image
241 Tensor src{};
242
243 // Intermediate tensors used
244 Tensor weights0{};
245 Tensor weights1{};
246 Tensor weights2{};
247 Tensor biases0{};
248 Tensor biases1{};
249 Tensor biases2{};
250 Tensor out_conv0{};
251 Tensor out_conv1{};
252 Tensor out_act0{};
253 Tensor out_act1{};
254 Tensor out_act2{};
255 Tensor out_pool0{};
256 Tensor out_pool1{};
257 Tensor out_fc0{};
258 Tensor out_softmax{};
259
260 // Allocator
261 Allocator allocator{};
262
263 // Memory groups
264 std::unique_ptr<MemoryGroup> memory_group0{};
265 std::unique_ptr<MemoryGroup> memory_group1{};
266
267 // Layers
268 std::unique_ptr<NEConvolutionLayer> conv0{};
269 std::unique_ptr<NEConvolutionLayer> conv1{};
270 std::unique_ptr<NEFullyConnectedLayer> fc0{};
271 std::unique_ptr<NESoftmaxLayer> softmax{};
272 NEPoolingLayer pool0{};
273 NEPoolingLayer pool1{};
274 NEActivationLayer act0{};
275 NEActivationLayer act1{};
276 NEActivationLayer act2{};
277 };
278
279 /** Main program for cnn test
280 *
281 * The example implements the following CNN architecture:
282 *
283 * Input -> conv0:5x5 -> act0:relu -> pool:2x2 -> conv1:3x3 -> act1:relu -> pool:2x2 -> fc0 -> act2:relu -> softmax
284 *
285 * @param[in] argc Number of arguments
286 * @param[in] argv Arguments
287 */
main(int argc,char ** argv)288 int main(int argc, char **argv)
289 {
290 return utils::run_example<NEONCNNExample>(argc, argv);
291 }
292