1 //
2 // Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #define LOG_TAG "arm-armnn-sl"
7
8 #include "CanonicalUtils.hpp"
9
10 #include <armnn/Utils.hpp>
11 #include <armnn/utility/Assert.hpp>
12 #include <armnnSerializer/ISerializer.hpp>
13 #include <armnnUtils/Permute.hpp>
14
15 #include <ghc/filesystem.hpp>
16 namespace fs = ghc::filesystem;
17 #include <half/half.hpp>
18 #include <log/log.h>
19
20 #include <cassert>
21 #include <cerrno>
22 #include <cinttypes>
23 #include <cstdio>
24 #include <sstream>
25 #include <time.h>
26 #include <variant>
27
28 namespace armnn
29 {
30 using Half = half_float::half; //import half float implementation
31 } //namespace armnn
32
33 using namespace android;
34 using namespace android::nn;
35
36 namespace armnn_driver
37 {
38 const armnn::PermutationVector g_DontPermute{};
39
SwizzleAndroidNn4dTensorToArmNn(armnn::TensorInfo & tensorInfo,const void * input,void * output,const armnn::PermutationVector & mappings)40 void SwizzleAndroidNn4dTensorToArmNn(armnn::TensorInfo& tensorInfo,
41 const void* input,
42 void* output,
43 const armnn::PermutationVector& mappings)
44 {
45 assert(tensorInfo.GetNumDimensions() == 4U);
46
47 armnn::DataType dataType = tensorInfo.GetDataType();
48 switch (dataType)
49 {
50 case armnn::DataType::Float16:
51 case armnn::DataType::Float32:
52 case armnn::DataType::QAsymmU8:
53 case armnn::DataType::QSymmS8:
54 case armnn::DataType::QAsymmS8:
55 // First swizzle tensor info
56 tensorInfo = armnnUtils::Permuted(tensorInfo, mappings);
57 // Then swizzle tensor data
58 armnnUtils::Permute(tensorInfo.GetShape(), mappings, input, output, armnn::GetDataTypeSize(dataType));
59 break;
60 default:
61 VLOG(DRIVER) << "Unknown armnn::DataType for swizzling";
62 assert(0);
63 }
64 }
65
GetMemoryFromPool(DataLocation location,const std::vector<android::nn::RunTimePoolInfo> & memPools)66 void* GetMemoryFromPool(DataLocation location, const std::vector<android::nn::RunTimePoolInfo>& memPools)
67 {
68 // find the location within the pool
69 assert(location.poolIndex < memPools.size());
70
71 const android::nn::RunTimePoolInfo& memPool = memPools[location.poolIndex];
72 uint8_t* memPoolBuffer = memPool.getBuffer();
73 uint8_t* memory = memPoolBuffer + location.offset;
74 return memory;
75 }
76
GetMemoryFromPointer(const Request::Argument & requestArg)77 void* GetMemoryFromPointer(const Request::Argument& requestArg)
78 {
79 // get the pointer memory
80 auto ptrMemory = std::visit([](std::variant<const void*, void*>&& memory)
81 {
82 if (std::holds_alternative<const void*>(memory))
83 {
84 auto ptr = std::get<const void*>(memory);
85 auto ptrMemory = static_cast<const uint8_t*>(ptr);
86 return const_cast<uint8_t*>(ptrMemory);
87 }
88 else
89 {
90 auto ptr = std::get<void*>(memory);
91 return static_cast<uint8_t*>(ptr);
92 }
93 }, requestArg.location.pointer);
94 return ptrMemory;
95 }
96
GetTensorInfoForOperand(const Operand & operand)97 armnn::TensorInfo GetTensorInfoForOperand(const Operand& operand)
98 {
99 using namespace armnn;
100 bool perChannel = false;
101 bool isScalar = false;
102
103 DataType type;
104 switch (operand.type)
105 {
106 case OperandType::TENSOR_BOOL8:
107 type = armnn::DataType::Boolean;
108 break;
109 case OperandType::TENSOR_FLOAT32:
110 type = armnn::DataType::Float32;
111 break;
112 case OperandType::TENSOR_FLOAT16:
113 type = armnn::DataType::Float16;
114 break;
115 case OperandType::TENSOR_QUANT8_ASYMM:
116 type = armnn::DataType::QAsymmU8;
117 break;
118 case OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL:
119 perChannel=true;
120 ARMNN_FALLTHROUGH;
121 case OperandType::TENSOR_QUANT8_SYMM:
122 type = armnn::DataType::QSymmS8;
123 break;
124 case OperandType::TENSOR_QUANT16_SYMM:
125 type = armnn::DataType::QSymmS16;
126 break;
127 case OperandType::TENSOR_INT32:
128 type = armnn::DataType::Signed32;
129 break;
130 case OperandType::INT32:
131 type = armnn::DataType::Signed32;
132 isScalar = true;
133 break;
134 case OperandType::TENSOR_QUANT8_ASYMM_SIGNED:
135 type = armnn::DataType::QAsymmS8;
136 break;
137 default:
138 throw UnsupportedOperand<OperandType>(operand.type);
139 }
140
141 TensorInfo ret;
142 if (isScalar)
143 {
144 ret = TensorInfo(TensorShape(armnn::Dimensionality::Scalar), type);
145 }
146 else
147 {
148 if (operand.dimensions.size() == 0)
149 {
150 TensorShape tensorShape(Dimensionality::NotSpecified);
151 ret = TensorInfo(tensorShape, type);
152 }
153 else
154 {
155 bool dimensionsSpecificity[5] = { true, true, true, true, true };
156 int count = 0;
157 std::for_each(operand.dimensions.data(),
158 operand.dimensions.data() + operand.dimensions.size(),
159 [&](const unsigned int val)
160 {
161 if (val == 0)
162 {
163 dimensionsSpecificity[count] = false;
164 }
165 count++;
166 });
167
168 TensorShape tensorShape(operand.dimensions.size(), operand.dimensions.data(), dimensionsSpecificity);
169 ret = TensorInfo(tensorShape, type);
170 }
171 }
172
173 if (perChannel)
174 {
175 // ExtraParams is expected to be of type channelQuant
176 const auto& perAxisQuantParams = std::get<Operand::SymmPerChannelQuantParams>(operand.extraParams);
177
178 ret.SetQuantizationScales(perAxisQuantParams.scales);
179 ret.SetQuantizationDim(MakeOptional<unsigned int>(perAxisQuantParams.channelDim));
180 }
181 else
182 {
183 ret.SetQuantizationScale(operand.scale);
184 ret.SetQuantizationOffset(operand.zeroPoint);
185 }
186 return ret;
187 }
188
GetOperandSummary(const Operand & operand)189 std::string GetOperandSummary(const Operand& operand)
190 {
191 std::stringstream ss;
192 ss << "operand dimensions: [ ";
193 for (unsigned int i = 0; i < operand.dimensions.size(); ++i)
194 {
195 ss << operand.dimensions[i] << " ";
196 }
197 ss << "] operand type: " << operand.type;
198 return ss.str();
199 }
200
201 template <typename TensorType>
202 using DumpElementFunction = void (*)(const TensorType& tensor,
203 unsigned int elementIndex,
204 std::ofstream& fileStream);
205
206 namespace
207 {
208 template <typename TensorType, typename ElementType, typename PrintableType = ElementType>
DumpTensorElement(const TensorType & tensor,unsigned int elementIndex,std::ofstream & fileStream)209 void DumpTensorElement(const TensorType& tensor, unsigned int elementIndex, std::ofstream& fileStream)
210 {
211 const ElementType* elements = reinterpret_cast<const ElementType*>(tensor.GetMemoryArea());
212 fileStream << static_cast<PrintableType>(elements[elementIndex]) << " ";
213 }
214
215 } // namespace
216 template <typename TensorType>
DumpTensor(const std::string & dumpDir,const std::string & requestName,const std::string & tensorName,const TensorType & tensor)217 void DumpTensor(const std::string& dumpDir,
218 const std::string& requestName,
219 const std::string& tensorName,
220 const TensorType& tensor)
221 {
222 // The dump directory must exist in advance.
223 fs::path dumpPath = dumpDir;
224 const fs::path fileName = dumpPath / (requestName + "_" + tensorName + ".dump");
225
226 std::ofstream fileStream;
227 fileStream.open(fileName.c_str(), std::ofstream::out | std::ofstream::trunc);
228
229 if (!fileStream.good())
230 {
231 VLOG(DRIVER) << "Could not open file %s for writing" << fileName.c_str();
232 return;
233 }
234
235 DumpElementFunction<TensorType> dumpElementFunction = nullptr;
236
237 switch (tensor.GetDataType())
238 {
239 case armnn::DataType::Float32:
240 {
241 dumpElementFunction = &DumpTensorElement<TensorType, float>;
242 break;
243 }
244 case armnn::DataType::QAsymmU8:
245 {
246 dumpElementFunction = &DumpTensorElement<TensorType, uint8_t, uint32_t>;
247 break;
248 }
249 case armnn::DataType::Signed32:
250 {
251 dumpElementFunction = &DumpTensorElement<TensorType, int32_t>;
252 break;
253 }
254 case armnn::DataType::Float16:
255 {
256 dumpElementFunction = &DumpTensorElement<TensorType, armnn::Half>;
257 break;
258 }
259 case armnn::DataType::QAsymmS8:
260 {
261 dumpElementFunction = &DumpTensorElement<TensorType, int8_t, int32_t>;
262 break;
263 }
264 case armnn::DataType::Boolean:
265 {
266 dumpElementFunction = &DumpTensorElement<TensorType, bool>;
267 break;
268 }
269 default:
270 {
271 dumpElementFunction = nullptr;
272 }
273 }
274
275 if (dumpElementFunction != nullptr)
276 {
277 const unsigned int numDimensions = tensor.GetNumDimensions();
278 const armnn::TensorShape shape = tensor.GetShape();
279
280 if (!shape.AreAllDimensionsSpecified())
281 {
282 fileStream << "Cannot dump tensor elements: not all dimensions are specified" << std::endl;
283 return;
284 }
285 fileStream << "# Number of elements " << tensor.GetNumElements() << std::endl;
286
287 if (numDimensions == 0)
288 {
289 fileStream << "# Shape []" << std::endl;
290 return;
291 }
292 fileStream << "# Shape [" << shape[0];
293 for (unsigned int d = 1; d < numDimensions; ++d)
294 {
295 fileStream << "," << shape[d];
296 }
297 fileStream << "]" << std::endl;
298 fileStream << "Each line contains the data of each of the elements of dimension0. In NCHW and NHWC, each line"
299 " will be a batch" << std::endl << std::endl;
300
301 // Split will create a new line after all elements of the first dimension
302 // (in a 4, 3, 2, 3 tensor, there will be 4 lines of 18 elements)
303 unsigned int split = 1;
304 if (numDimensions == 1)
305 {
306 split = shape[0];
307 }
308 else
309 {
310 for (unsigned int i = 1; i < numDimensions; ++i)
311 {
312 split *= shape[i];
313 }
314 }
315
316 // Print all elements in the tensor
317 for (unsigned int elementIndex = 0; elementIndex < tensor.GetNumElements(); ++elementIndex)
318 {
319 (*dumpElementFunction)(tensor, elementIndex, fileStream);
320
321 if ( (elementIndex + 1) % split == 0 )
322 {
323 fileStream << std::endl;
324 }
325 }
326 fileStream << std::endl;
327 }
328 else
329 {
330 fileStream << "Cannot dump tensor elements: Unsupported data type "
331 << static_cast<unsigned int>(tensor.GetDataType()) << std::endl;
332 }
333
334 if (!fileStream.good())
335 {
336 VLOG(DRIVER) << "An error occurred when writing to file %s" << fileName.c_str();
337 }
338 }
339
340 template void DumpTensor<armnn::ConstTensor>(const std::string& dumpDir,
341 const std::string& requestName,
342 const std::string& tensorName,
343 const armnn::ConstTensor& tensor);
344
345 template void DumpTensor<armnn::Tensor>(const std::string& dumpDir,
346 const std::string& requestName,
347 const std::string& tensorName,
348 const armnn::Tensor& tensor);
349
DumpJsonProfilingIfRequired(bool gpuProfilingEnabled,const std::string & dumpDir,armnn::NetworkId networkId,const armnn::IProfiler * profiler)350 void DumpJsonProfilingIfRequired(bool gpuProfilingEnabled,
351 const std::string& dumpDir,
352 armnn::NetworkId networkId,
353 const armnn::IProfiler* profiler)
354 {
355 // Check if profiling is required.
356 if (!gpuProfilingEnabled)
357 {
358 return;
359 }
360
361 // The dump directory must exist in advance.
362 if (dumpDir.empty())
363 {
364 return;
365 }
366
367 ARMNN_ASSERT(profiler);
368
369 // Set the name of the output profiling file.
370 fs::path dumpPath = dumpDir;
371 const fs::path fileName = dumpPath / (std::to_string(networkId) + "_profiling.json");
372
373 // Open the ouput file for writing.
374 std::ofstream fileStream;
375 fileStream.open(fileName.c_str(), std::ofstream::out | std::ofstream::trunc);
376
377 if (!fileStream.good())
378 {
379 VLOG(DRIVER) << "Could not open file %s for writing" << fileName.c_str();
380 return;
381 }
382
383 // Write the profiling info to a JSON file.
384 profiler->Print(fileStream);
385 }
386
ExportNetworkGraphToDotFile(const armnn::IOptimizedNetwork & optimizedNetwork,const std::string & dumpDir)387 std::string ExportNetworkGraphToDotFile(const armnn::IOptimizedNetwork& optimizedNetwork,
388 const std::string& dumpDir)
389 {
390 std::string fileName;
391 // The dump directory must exist in advance.
392 if (dumpDir.empty())
393 {
394 return fileName;
395 }
396
397 std::string timestamp = GetFileTimestamp();
398 if (timestamp.empty())
399 {
400 return fileName;
401 }
402
403 // Set the name of the output .dot file.
404 fs::path dumpPath = dumpDir;
405 fs::path tempFilePath = dumpPath / (timestamp + "_networkgraph.dot");
406 fileName = tempFilePath.string();
407
408 VLOG(DRIVER) << "Exporting the optimized network graph to file: %s" << fileName.c_str();
409
410 // Write the network graph to a dot file.
411 std::ofstream fileStream;
412 fileStream.open(fileName, std::ofstream::out | std::ofstream::trunc);
413
414 if (!fileStream.good())
415 {
416 VLOG(DRIVER) << "Could not open file %s for writing" << fileName.c_str();
417 return fileName;
418 }
419
420 if (optimizedNetwork.SerializeToDot(fileStream) != armnn::Status::Success)
421 {
422 VLOG(DRIVER) << "An error occurred when writing to file %s" << fileName.c_str();
423 }
424 return fileName;
425 }
426
SerializeNetwork(const armnn::INetwork & network,const std::string & dumpDir,std::vector<uint8_t> & dataCacheData,bool dataCachingActive)427 std::string SerializeNetwork(const armnn::INetwork& network,
428 const std::string& dumpDir,
429 std::vector<uint8_t>& dataCacheData,
430 bool dataCachingActive)
431 {
432 std::string fileName;
433 bool bSerializeToFile = true;
434 if (dumpDir.empty())
435 {
436 bSerializeToFile = false;
437 }
438 else
439 {
440 std::string timestamp = GetFileTimestamp();
441 if (timestamp.empty())
442 {
443 bSerializeToFile = false;
444 }
445 }
446 if (!bSerializeToFile && !dataCachingActive)
447 {
448 return fileName;
449 }
450
451 auto serializer(armnnSerializer::ISerializer::Create());
452 // Serialize the Network
453 serializer->Serialize(network);
454 if (dataCachingActive)
455 {
456 std::stringstream stream;
457 auto serialized = serializer->SaveSerializedToStream(stream);
458 if (serialized)
459 {
460 std::string const serializedString{stream.str()};
461 std::copy(serializedString.begin(),
462 serializedString.end(),
463 std::back_inserter(dataCacheData));
464 }
465 }
466
467 if (bSerializeToFile)
468 {
469 // Set the name of the output .armnn file.
470 fs::path dumpPath = dumpDir;
471 std::string timestamp = GetFileTimestamp();
472 fs::path tempFilePath = dumpPath / (timestamp + "_network.armnn");
473 fileName = tempFilePath.string();
474
475 // Save serialized network to a file
476 std::ofstream serializedFile(fileName, std::ios::out | std::ios::binary);
477 auto serialized = serializer->SaveSerializedToStream(serializedFile);
478 if (!serialized)
479 {
480 VLOG(DRIVER) << "An error occurred when serializing to file %s" << fileName.c_str();
481 }
482 }
483 return fileName;
484 }
485
IsDynamicTensor(const armnn::TensorInfo & tensorInfo)486 bool IsDynamicTensor(const armnn::TensorInfo& tensorInfo)
487 {
488 if (tensorInfo.GetShape().GetDimensionality() == armnn::Dimensionality::NotSpecified)
489 {
490 return true;
491 }
492 // Account for the usage of the TensorShape empty constructor
493 if (tensorInfo.GetNumDimensions() == 0)
494 {
495 return true;
496 }
497 return !tensorInfo.GetShape().AreAllDimensionsSpecified();
498 }
499
AreDynamicTensorsSupported()500 bool AreDynamicTensorsSupported() //TODO
501 {
502 return true;
503 }
504
isQuantizedOperand(const OperandType & operandType)505 bool isQuantizedOperand(const OperandType& operandType)
506 {
507 if (operandType == OperandType::TENSOR_QUANT8_ASYMM ||
508 operandType == OperandType::TENSOR_QUANT8_SYMM_PER_CHANNEL ||
509 operandType == OperandType::TENSOR_QUANT8_SYMM ||
510 operandType == OperandType::TENSOR_QUANT16_SYMM ||
511 operandType == OperandType::TENSOR_QUANT8_ASYMM_SIGNED)
512 {
513 return true;
514 }
515 else
516 {
517 return false;
518 }
519 }
520
GetModelSummary(const Model & model)521 std::string GetModelSummary(const Model& model)
522 {
523 std::stringstream result;
524
525 result << model.main.inputIndexes.size() << " input(s), "
526 << model.main.operations.size() << " operation(s), "
527 << model.main.outputIndexes.size() << " output(s), "
528 << model.main.operands.size() << " operand(s) "
529 << std::endl;
530
531 result << "Inputs: ";
532 for (uint32_t i = 0; i < model.main.inputIndexes.size(); i++)
533 {
534 result << GetOperandSummary(model.main.operands[model.main.inputIndexes[i]]) << ", ";
535 }
536 result << std::endl;
537
538 result << "Operations: ";
539 for (uint32_t i = 0; i < model.main.operations.size(); i++)
540 {
541 result << model.main.operations[i].type << ", ";
542 }
543 result << std::endl;
544
545 result << "Outputs: ";
546 for (uint32_t i = 0; i < model.main.outputIndexes.size(); i++)
547 {
548 result << GetOperandSummary(model.main.operands[model.main.outputIndexes[i]]) << ", ";
549 }
550 result << std::endl;
551
552 return result.str();
553 }
554
GetFileTimestamp()555 std::string GetFileTimestamp()
556 {
557 // used to get a timestamp to name diagnostic files (the ArmNN serialized graph
558 // and getSupportedOperations.txt files)
559 timespec ts;
560 int iRet = clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
561 std::stringstream ss;
562 if (iRet == 0)
563 {
564 ss << std::to_string(ts.tv_sec) << "_" << std::to_string(ts.tv_nsec);
565 }
566 else
567 {
568 VLOG(DRIVER) << "clock_gettime failed with errno " <<
569 std::to_string(errno).c_str() << " : " <<
570 std::strerror(errno);
571 }
572 return ss.str();
573 }
574
RenameExportedFiles(const std::string & existingSerializedFileName,const std::string & existingDotFileName,const std::string & dumpDir,const armnn::NetworkId networkId)575 void RenameExportedFiles(const std::string& existingSerializedFileName,
576 const std::string& existingDotFileName,
577 const std::string& dumpDir,
578 const armnn::NetworkId networkId)
579 {
580 if (dumpDir.empty())
581 {
582 return;
583 }
584 RenameFile(existingSerializedFileName, std::string("_network.armnn"), dumpDir, networkId);
585 RenameFile(existingDotFileName, std::string("_networkgraph.dot"), dumpDir, networkId);
586 }
587
RenameFile(const std::string & existingName,const std::string & extension,const std::string & dumpDir,const armnn::NetworkId networkId)588 void RenameFile(const std::string& existingName,
589 const std::string& extension,
590 const std::string& dumpDir,
591 const armnn::NetworkId networkId)
592 {
593 if (existingName.empty() || dumpDir.empty())
594 {
595 return;
596 }
597
598 fs::path dumpPath = dumpDir;
599 const fs::path newFileName = dumpPath / (std::to_string(networkId) + extension);
600 int iRet = rename(existingName.c_str(), newFileName.c_str());
601 if (iRet != 0)
602 {
603 std::stringstream ss;
604 ss << "rename of [" << existingName << "] to [" << newFileName << "] failed with errno "
605 << std::to_string(errno) << " : " << std::strerror(errno);
606 VLOG(DRIVER) << ss.str().c_str();
607 }
608 }
609
CommitPools(std::vector<::android::nn::RunTimePoolInfo> & memPools)610 void CommitPools(std::vector<::android::nn::RunTimePoolInfo>& memPools)
611 {
612 // Commit output buffers.
613 // Note that we update *all* pools, even if they aren't actually used as outputs -
614 // this is simpler and is what the CpuExecutor does.
615 for (auto& pool : memPools)
616 {
617 // Type android::nn::RunTimePoolInfo has changed between Android P & Q and Android R, where
618 // update() has been removed and flush() added.
619 pool.flush();
620 }
621 }
622 } // namespace armnn_driver
623