1 /*
2 * Copyright (c) 2023-2024 Tomeu Vizoso <[email protected]>
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include <fcntl.h>
7 #include <filesystem>
8 #include <fstream>
9 #include <gtest/gtest.h>
10 #include <xtensor/xrandom.hpp>
11
12 #include <iostream>
13 #include "tensorflow/lite/c/c_api.h"
14 #include "test_executor.h"
15
16 #define TEST_CONV2D 1
17 #define TEST_DEPTHWISE 1
18 #define TEST_ADD 1
19 #define TEST_MOBILENETV1 1
20 #define TEST_MOBILEDET 1
21
22 #define TOLERANCE 2
23 #define MODEL_TOLERANCE 8
24 #define QUANT_TOLERANCE 2
25
26 std::vector<bool> is_signed{false}; /* TODO: Support INT8? */
27 std::vector<bool> padding_same{false, true};
28 std::vector<int> stride{1, 2};
29 std::vector<int> output_channels{1, 32, 120, 128, 160, 256};
30 std::vector<int> input_channels{1, 32, 120, 128, 256};
31 std::vector<int> dw_channels{1, 32, 120, 128, 256};
32 std::vector<int> dw_weight_size{3, 5};
33 std::vector<int> weight_size{1, 3, 5};
34 std::vector<int> input_size{3, 5, 8, 80, 112};
35
36 static bool
cache_is_enabled(void)37 cache_is_enabled(void)
38 {
39 return getenv("TEFLON_ENABLE_CACHE");
40 }
41
42 static bool
read_into(const char * path,std::vector<uint8_t> & buf)43 read_into(const char *path, std::vector<uint8_t> &buf)
44 {
45 FILE *f = fopen(path, "rb");
46 if (f == NULL)
47 return false;
48
49 fseek(f, 0, SEEK_END);
50 long fsize = ftell(f);
51 fseek(f, 0, SEEK_SET);
52
53 buf.resize(fsize);
54 fread(buf.data(), fsize, 1, f);
55
56 fclose(f);
57
58 return true;
59 }
60
61 static void
set_seed(unsigned seed)62 set_seed(unsigned seed)
63 {
64 srand(seed);
65 xt::random::seed(seed);
66 }
67
68 static void
test_model(std::vector<uint8_t> buf,std::string cache_dir,unsigned tolerance)69 test_model(std::vector<uint8_t> buf, std::string cache_dir, unsigned tolerance)
70 {
71 std::vector<std::vector<uint8_t>> input;
72 std::vector<std::vector<uint8_t>> cpu_output;
73 std::ostringstream input_cache;
74 input_cache << cache_dir << "/"
75 << "input.data";
76
77 std::ostringstream output_cache;
78 output_cache << cache_dir << "/"
79 << "output.data";
80
81 TfLiteModel *model = TfLiteModelCreate(buf.data(), buf.size());
82 assert(model);
83
84 if (cache_is_enabled()) {
85 input.resize(1);
86 bool ret = read_into(input_cache.str().c_str(), input[0]);
87
88 if (ret) {
89 cpu_output.resize(1);
90 ret = read_into(output_cache.str().c_str(), cpu_output[0]);
91 }
92 }
93
94 if (cpu_output.size() == 0 || cpu_output[0].size() == 0) {
95 input.resize(0);
96 cpu_output.resize(0);
97
98 cpu_output = run_model(model, EXECUTOR_CPU, input);
99
100 if (cache_is_enabled()) {
101 std::ofstream file(input_cache.str().c_str(), std::ios::out | std::ios::binary);
102 file.write(reinterpret_cast<const char *>(input[0].data()), input[0].size());
103 file.close();
104
105 file = std::ofstream(output_cache.str().c_str(), std::ios::out | std::ios::binary);
106 file.write(reinterpret_cast<const char *>(cpu_output[0].data()), cpu_output[0].size());
107 file.close();
108 }
109 }
110
111 std::vector<std::vector<uint8_t>> npu_output = run_model(model, EXECUTOR_NPU, input);
112
113 EXPECT_EQ(cpu_output.size(), npu_output.size()) << "Array sizes differ.";
114 for (size_t i = 0; i < cpu_output.size(); i++) {
115 EXPECT_EQ(cpu_output[i].size(), npu_output[i].size()) << "Array sizes differ (" << i << ").";
116
117 for (size_t j = 0; j < cpu_output[i].size(); j++) {
118 if (abs(cpu_output[i][j] - npu_output[i][j]) > tolerance) {
119 std::cout << "CPU: ";
120 for (int k = 0; k < std::min(int(cpu_output[i].size()), 24); k++)
121 std::cout << std::setfill('0') << std::setw(2) << std::hex << int(cpu_output[i][k]) << " ";
122 std::cout << "\n";
123 std::cout << "NPU: ";
124 for (int k = 0; k < std::min(int(npu_output[i].size()), 24); k++)
125 std::cout << std::setfill('0') << std::setw(2) << std::hex << int(npu_output[i][k]) << " ";
126 std::cout << "\n";
127
128 FAIL() << "Output at " << j << " from the NPU (" << std::setfill('0') << std::setw(2) << std::hex << int(npu_output[i][j]) << ") doesn't match that from the CPU (" << std::setfill('0') << std::setw(2) << std::hex << int(cpu_output[i][j]) << ").";
129 }
130 }
131 }
132
133 TfLiteModelDelete(model);
134 }
135
136 static void
test_model_file(std::string file_name)137 test_model_file(std::string file_name)
138 {
139 set_seed(4);
140
141 std::ifstream model_file(file_name, std::ios::binary);
142 std::vector<uint8_t> buffer((std::istreambuf_iterator<char>(model_file)),
143 std::istreambuf_iterator<char>());
144 test_model(buffer, "", MODEL_TOLERANCE);
145 }
146
147 void
test_conv(int input_size,int weight_size,int input_channels,int output_channels,int stride,bool padding_same,bool is_signed,bool depthwise,int seed)148 test_conv(int input_size, int weight_size, int input_channels, int output_channels,
149 int stride, bool padding_same, bool is_signed, bool depthwise, int seed)
150 {
151 std::vector<uint8_t> buf;
152 std::ostringstream cache_dir, model_cache;
153 cache_dir << "/var/cache/teflon_tests/" << input_size << "_" << weight_size << "_" << input_channels << "_" << output_channels << "_" << stride << "_" << padding_same << "_" << is_signed << "_" << depthwise << "_" << seed;
154 model_cache << cache_dir.str() << "/"
155 << "model.tflite";
156
157 if (weight_size > input_size)
158 GTEST_SKIP();
159
160 set_seed(seed);
161
162 if (cache_is_enabled()) {
163 if (access(model_cache.str().c_str(), F_OK) == 0) {
164 read_into(model_cache.str().c_str(), buf);
165 }
166 }
167
168 if (buf.size() == 0) {
169 buf = conv2d_generate_model(input_size, weight_size,
170 input_channels, output_channels,
171 stride, padding_same, is_signed,
172 depthwise);
173
174 if (cache_is_enabled()) {
175 if (access(cache_dir.str().c_str(), F_OK) != 0) {
176 ASSERT_TRUE(std::filesystem::create_directories(cache_dir.str().c_str()));
177 }
178 std::ofstream file(model_cache.str().c_str(), std::ios::out | std::ios::binary);
179 file.write(reinterpret_cast<const char *>(buf.data()), buf.size());
180 file.close();
181 }
182 }
183
184 test_model(buf, cache_dir.str(), TOLERANCE);
185 }
186
187 void
test_add(int input_size,int weight_size,int input_channels,int output_channels,int stride,bool padding_same,bool is_signed,bool depthwise,int seed,unsigned tolerance)188 test_add(int input_size, int weight_size, int input_channels, int output_channels,
189 int stride, bool padding_same, bool is_signed, bool depthwise, int seed,
190 unsigned tolerance)
191 {
192 std::vector<uint8_t> buf;
193 std::ostringstream cache_dir, model_cache;
194 cache_dir << "/var/cache/teflon_tests/"
195 << "add_" << input_size << "_" << weight_size << "_" << input_channels << "_" << output_channels << "_" << stride << "_" << padding_same << "_" << is_signed << "_" << depthwise << "_" << seed;
196 model_cache << cache_dir.str() << "/"
197 << "model.tflite";
198
199 if (weight_size > input_size)
200 GTEST_SKIP();
201
202 set_seed(seed);
203
204 if (cache_is_enabled()) {
205 if (access(model_cache.str().c_str(), F_OK) == 0) {
206 read_into(model_cache.str().c_str(), buf);
207 }
208 }
209
210 if (buf.size() == 0) {
211 buf = add_generate_model(input_size, weight_size,
212 input_channels, output_channels,
213 stride, padding_same, is_signed,
214 depthwise);
215
216 if (cache_is_enabled()) {
217 if (access(cache_dir.str().c_str(), F_OK) != 0) {
218 ASSERT_TRUE(std::filesystem::create_directories(cache_dir.str().c_str()));
219 }
220 std::ofstream file(model_cache.str().c_str(), std::ios::out | std::ios::binary);
221 file.write(reinterpret_cast<const char *>(buf.data()), buf.size());
222 file.close();
223 }
224 }
225
226 test_model(buf, cache_dir.str(), tolerance);
227 }
228
229 #if TEST_CONV2D
230
231 class Conv2D : public testing::TestWithParam<std::tuple<bool, bool, int, int, int, int, int>> {};
232
TEST_P(Conv2D,Op)233 TEST_P(Conv2D, Op)
234 {
235 test_conv(std::get<6>(GetParam()),
236 std::get<5>(GetParam()),
237 std::get<4>(GetParam()),
238 std::get<3>(GetParam()),
239 std::get<2>(GetParam()),
240 std::get<1>(GetParam()),
241 std::get<0>(GetParam()),
242 false, /* depthwise */
243 4);
244 }
245
246 static inline std::string
Conv2DTestCaseName(const testing::TestParamInfo<std::tuple<bool,bool,int,int,int,int,int>> & info)247 Conv2DTestCaseName(
248 const testing::TestParamInfo<std::tuple<bool, bool, int, int, int, int, int>> &info)
249 {
250 std::string name = "";
251
252 name += "input_size_" + std::to_string(std::get<6>(info.param));
253 name += "_weight_size_" + std::to_string(std::get<5>(info.param));
254 name += "_input_channels_" + std::to_string(std::get<4>(info.param));
255 name += "_output_channels_" + std::to_string(std::get<3>(info.param));
256 name += "_stride_" + std::to_string(std::get<2>(info.param));
257 name += "_padding_same_" + std::to_string(std::get<1>(info.param));
258 name += "_is_signed_" + std::to_string(std::get<0>(info.param));
259
260 return name;
261 }
262
263 INSTANTIATE_TEST_SUITE_P(
264 , Conv2D,
265 ::testing::Combine(::testing::ValuesIn(is_signed),
266 ::testing::ValuesIn(padding_same),
267 ::testing::ValuesIn(stride),
268 ::testing::ValuesIn(output_channels),
269 ::testing::ValuesIn(input_channels),
270 ::testing::ValuesIn(weight_size),
271 ::testing::ValuesIn(input_size)),
272 Conv2DTestCaseName);
273
274 #endif
275
276 #if TEST_DEPTHWISE
277
278 class DepthwiseConv2D : public testing::TestWithParam<std::tuple<bool, bool, int, int, int, int>> {};
279
TEST_P(DepthwiseConv2D,Op)280 TEST_P(DepthwiseConv2D, Op)
281 {
282 test_conv(std::get<5>(GetParam()),
283 std::get<4>(GetParam()),
284 std::get<3>(GetParam()),
285 std::get<3>(GetParam()),
286 std::get<2>(GetParam()),
287 std::get<1>(GetParam()),
288 std::get<0>(GetParam()),
289 true, /* depthwise */
290 4);
291 }
292
293 static inline std::string
DepthwiseConv2DTestCaseName(const testing::TestParamInfo<std::tuple<bool,bool,int,int,int,int>> & info)294 DepthwiseConv2DTestCaseName(
295 const testing::TestParamInfo<std::tuple<bool, bool, int, int, int, int>> &info)
296 {
297 std::string name = "";
298
299 name += "input_size_" + std::to_string(std::get<5>(info.param));
300 name += "_weight_size_" + std::to_string(std::get<4>(info.param));
301 name += "_channels_" + std::to_string(std::get<3>(info.param));
302 name += "_stride_" + std::to_string(std::get<2>(info.param));
303 name += "_padding_same_" + std::to_string(std::get<1>(info.param));
304 name += "_is_signed_" + std::to_string(std::get<0>(info.param));
305
306 return name;
307 }
308
309 INSTANTIATE_TEST_SUITE_P(
310 , DepthwiseConv2D,
311 ::testing::Combine(::testing::ValuesIn(is_signed),
312 ::testing::ValuesIn(padding_same),
313 ::testing::ValuesIn(stride),
314 ::testing::ValuesIn(dw_channels),
315 ::testing::ValuesIn(dw_weight_size),
316 ::testing::ValuesIn(input_size)),
317 DepthwiseConv2DTestCaseName);
318
319 #endif
320
321 #if TEST_ADD
322
323 class Add : public testing::TestWithParam<std::tuple<bool, bool, int, int, int, int, int>> {};
324
TEST_P(Add,Op)325 TEST_P(Add, Op)
326 {
327 test_add(std::get<6>(GetParam()),
328 std::get<5>(GetParam()),
329 std::get<4>(GetParam()),
330 std::get<3>(GetParam()),
331 std::get<2>(GetParam()),
332 std::get<1>(GetParam()),
333 std::get<0>(GetParam()),
334 false, /* depthwise */
335 4,
336 TOLERANCE);
337 }
338
339 static inline std::string
AddTestCaseName(const testing::TestParamInfo<std::tuple<bool,bool,int,int,int,int,int>> & info)340 AddTestCaseName(
341 const testing::TestParamInfo<std::tuple<bool, bool, int, int, int, int, int>> &info)
342 {
343 std::string name = "";
344
345 name += "input_size_" + std::to_string(std::get<6>(info.param));
346 name += "_weight_size_" + std::to_string(std::get<5>(info.param));
347 name += "_input_channels_" + std::to_string(std::get<4>(info.param));
348 name += "_output_channels_" + std::to_string(std::get<3>(info.param));
349 name += "_stride_" + std::to_string(std::get<2>(info.param));
350 name += "_padding_same_" + std::to_string(std::get<1>(info.param));
351 name += "_is_signed_" + std::to_string(std::get<0>(info.param));
352
353 return name;
354 }
355
356 INSTANTIATE_TEST_SUITE_P(
357 , Add,
358 ::testing::Combine(::testing::ValuesIn(is_signed),
359 ::testing::ValuesIn(padding_same),
360 ::testing::ValuesIn(stride),
361 ::testing::ValuesIn(output_channels),
362 ::testing::ValuesIn(input_channels),
363 ::testing::ValuesIn(weight_size),
364 ::testing::ValuesIn(input_size)),
365 AddTestCaseName);
366
367 class AddQuant : public testing::TestWithParam<int> {};
368
TEST_P(AddQuant,Op)369 TEST_P(AddQuant, Op)
370 {
371 test_add(40,
372 1,
373 1,
374 1,
375 1,
376 false, /* padding_same */
377 false, /* is_signed */
378 false, /* depthwise */
379 GetParam(),
380 QUANT_TOLERANCE);
381 }
382
383 INSTANTIATE_TEST_SUITE_P(
384 , AddQuant,
385 ::testing::Range(0, 100));
386
387 #endif
388
389 #if TEST_MOBILENETV1
390
391 class MobileNetV1 : public ::testing::Test {};
392
393 class MobileNetV1Param : public testing::TestWithParam<int> {};
394
TEST(MobileNetV1,Whole)395 TEST(MobileNetV1, Whole)
396 {
397 std::ostringstream file_path;
398 assert(getenv("TEFLON_TEST_DATA"));
399 file_path << getenv("TEFLON_TEST_DATA") << "/mobilenet_v1_1.0_224_quant.tflite";
400
401 test_model_file(file_path.str());
402 }
403
TEST_P(MobileNetV1Param,Op)404 TEST_P(MobileNetV1Param, Op)
405 {
406 std::ostringstream file_path;
407 assert(getenv("TEFLON_TEST_DATA"));
408 file_path << getenv("TEFLON_TEST_DATA") << "/mb" << GetParam() << ".tflite";
409
410 test_model_file(file_path.str());
411 }
412
413 static inline std::string
MobileNetV1TestCaseName(const testing::TestParamInfo<int> & info)414 MobileNetV1TestCaseName(
415 const testing::TestParamInfo<int> &info)
416 {
417 std::string name = "";
418
419 name += "mb";
420 name += std::to_string(info.param);
421
422 return name;
423 }
424
425 INSTANTIATE_TEST_SUITE_P(
426 , MobileNetV1Param,
427 ::testing::Range(0, 28),
428 MobileNetV1TestCaseName);
429
430 #endif
431
432 #if TEST_MOBILEDET
433
434 class MobileDet : public ::testing::Test {};
435
436 class MobileDetParam : public testing::TestWithParam<int> {};
437
TEST(MobileDet,Whole)438 TEST(MobileDet, Whole)
439 {
440 std::ostringstream file_path;
441 assert(getenv("TEFLON_TEST_DATA"));
442 file_path << getenv("TEFLON_TEST_DATA") << "/ssdlite_mobiledet_coco_qat_postprocess.tflite";
443
444 test_model_file(file_path.str());
445 }
446
TEST_P(MobileDetParam,Op)447 TEST_P(MobileDetParam, Op)
448 {
449 std::ostringstream file_path;
450 assert(getenv("TEFLON_TEST_DATA"));
451 file_path << getenv("TEFLON_TEST_DATA") << "/mobiledet" << GetParam() << ".tflite";
452
453 test_model_file(file_path.str());
454 }
455
456 static inline std::string
MobileDetTestCaseName(const testing::TestParamInfo<int> & info)457 MobileDetTestCaseName(
458 const testing::TestParamInfo<int> &info)
459 {
460 std::string name = "";
461
462 name += "mobiledet";
463 name += std::to_string(info.param);
464
465 return name;
466 }
467
468 INSTANTIATE_TEST_SUITE_P(
469 , MobileDetParam,
470 ::testing::Range(0, 121),
471 MobileDetTestCaseName);
472
473 #endif
474
475 int
main(int argc,char ** argv)476 main(int argc, char **argv)
477 {
478 if (argc > 1 && !strcmp(argv[1], "generate_model")) {
479 std::vector<uint8_t> buf;
480
481 assert(argc == 11);
482
483 std::cout << "Generating model to ./model.tflite\n";
484
485 int n = 2;
486 int input_size = atoi(argv[n++]);
487 int weight_size = atoi(argv[n++]);
488 int input_channels = atoi(argv[n++]);
489 int output_channels = atoi(argv[n++]);
490 int stride = atoi(argv[n++]);
491 int padding_same = atoi(argv[n++]);
492 int is_signed = atoi(argv[n++]);
493 int depthwise = atoi(argv[n++]);
494 int seed = atoi(argv[n++]);
495
496 set_seed(seed);
497
498 buf = conv2d_generate_model(input_size, weight_size,
499 input_channels, output_channels,
500 stride, padding_same, is_signed,
501 depthwise);
502
503 int fd = open("model.tflite", O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
504 write(fd, buf.data(), buf.size());
505 close(fd);
506
507 return 0;
508 } else if (argc > 1 && !strcmp(argv[1], "run_model")) {
509 test_model_file(std::string(argv[2]));
510 } else {
511 testing::InitGoogleTest(&argc, argv);
512 return RUN_ALL_TESTS();
513 }
514 }
515