1 //
2 // Copyright © 2022-2023 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #include "TfliteExecutor.hpp"
7 #include "tensorflow/lite/kernels/kernel_util.h"
8
TfLiteExecutor(const ExecuteNetworkParams & params,armnn::IRuntime::CreationOptions runtimeOptions)9 TfLiteExecutor::TfLiteExecutor(const ExecuteNetworkParams& params, armnn::IRuntime::CreationOptions runtimeOptions)
10 : m_Params(params)
11 {
12 m_Model = tflite::FlatBufferModel::BuildFromFile(m_Params.m_ModelPath.c_str());
13 if (!m_Model)
14 {
15 LogAndThrow("Failed to load TfLite model from: " + m_Params.m_ModelPath);
16 }
17 m_TfLiteInterpreter = std::make_unique<Interpreter>();
18 tflite::ops::builtin::BuiltinOpResolver resolver;
19
20 tflite::InterpreterBuilder builder(*m_Model, resolver);
21 if (builder(&m_TfLiteInterpreter) != kTfLiteOk)
22 {
23 LogAndThrow("Error loading the model into the TfLiteInterpreter.");
24 }
25 if (m_TfLiteInterpreter->AllocateTensors() != kTfLiteOk)
26 {
27 LogAndThrow("Failed to allocate tensors in the TfLiteInterpreter.");
28 }
29 if (m_Params.m_TfLiteExecutor == ExecuteNetworkParams::TfLiteExecutor::ArmNNTfLiteDelegate)
30 {
31 // Create the Armnn Delegate
32 // Populate a DelegateOptions from the ExecuteNetworkParams.
33 armnnDelegate::DelegateOptions delegateOptions = m_Params.ToDelegateOptions();
34 delegateOptions.SetRuntimeOptions(runtimeOptions);
35 std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)>
36 theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions),
37 armnnDelegate::TfLiteArmnnDelegateDelete);
38 // Register armnn_delegate to TfLiteInterpreter
39 if (m_TfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate)) != kTfLiteOk)
40 {
41 LogAndThrow("Could not register ArmNN TfLite Delegate to TfLiteInterpreter.");
42 }
43 }
44 else
45 {
46 std::cout << "Running on TfLite without ArmNN delegate\n";
47 }
48
49 const size_t numInputs = m_TfLiteInterpreter->inputs().size();
50
51 for(unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex)
52 {
53 armnn::Optional<std::string> dataFile = m_Params.m_GenerateTensorData
54 ? armnn::EmptyOptional()
55 : armnn::MakeOptional<std::string>(m_Params.m_InputTensorDataFilePaths[inputIndex]);
56
57 int input = m_TfLiteInterpreter->inputs()[inputIndex];
58 const auto& inputName = m_TfLiteInterpreter->tensor(input)->name;
59
60 // Before we start, check if the tensor is constant.
61 if (!tflite::IsConstantTensor(m_TfLiteInterpreter->tensor(input)))
62 {
63 TfLiteIntArray* inputDims = m_TfLiteInterpreter->tensor(input)->dims;
64
65 unsigned int inputSize = 1;
66 for (unsigned int dim = 0; dim < static_cast<unsigned int>(inputDims->size); ++dim)
67 {
68 inputSize *= inputDims->data[dim];
69 }
70
71 const auto& dataType = m_TfLiteInterpreter->tensor(input)->type;
72
73 switch (dataType)
74 {
75 case kTfLiteFloat32:
76 {
77 auto inputData = m_TfLiteInterpreter->typed_tensor<float>(input);
78 PopulateTensorWithData<float>(inputData, inputSize, dataFile, inputName);
79 break;
80 }
81 case kTfLiteInt32:
82 {
83 auto inputData = m_TfLiteInterpreter->typed_tensor<int32_t>(input);
84 PopulateTensorWithData<int32_t>(inputData, inputSize, dataFile, inputName);
85 break;
86 }
87 case kTfLiteUInt8:
88 {
89 auto inputData = m_TfLiteInterpreter->typed_tensor<uint8_t>(input);
90 PopulateTensorWithData<uint8_t>(inputData, inputSize, dataFile, inputName);
91 break;
92 }
93 case kTfLiteInt16:
94 {
95 auto inputData = m_TfLiteInterpreter->typed_tensor<int16_t>(input);
96 PopulateTensorWithData<int16_t>(inputData, inputSize, dataFile, inputName);
97 break;
98 }
99 case kTfLiteInt8:
100 {
101 auto inputData = m_TfLiteInterpreter->typed_tensor<int8_t>(input);
102 PopulateTensorWithData<int8_t>(inputData, inputSize, dataFile, inputName);
103 break;
104 }
105 default:
106 {
107 LogAndThrow("Unsupported input tensor data type");
108 }
109 }
110 }
111 else
112 {
113 ARMNN_LOG(info) << "Input tensor \"" << inputName << "\" is constant and will not be populated with data.";
114 }
115 }
116 }
117
Execute()118 std::vector<const void *> TfLiteExecutor::Execute()
119 {
120 int status = 0;
121 std::vector<const void*> results;
122 for (size_t x = 0; x < m_Params.m_Iterations; x++)
123 {
124 // Start timer to record inference time in milliseconds.
125 const auto start_time = armnn::GetTimeNow();
126 // Run the inference
127 status = m_TfLiteInterpreter->Invoke();
128 const auto duration = armnn::GetTimeDuration(start_time);
129
130 if (!m_Params.m_DontPrintOutputs)
131 {
132 // Print out the output
133 for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex)
134 {
135 auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex];
136 TfLiteIntArray* outputDims = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims;
137 // If we've been asked to write to a file then set a file output stream. Otherwise use stdout.
138 FILE* outputTensorFile = stdout;
139 if (!m_Params.m_OutputTensorFiles.empty())
140 {
141 outputTensorFile = fopen(m_Params.m_OutputTensorFiles[outputIndex].c_str(), "w");
142 if (outputTensorFile == NULL)
143 {
144 LogAndThrow("Specified output tensor file, \"" + m_Params.m_OutputTensorFiles[outputIndex] +
145 "\", cannot be created. Defaulting to stdout. Error was: " + std::strerror(errno));
146 }
147 else
148 {
149 ARMNN_LOG(info) << "Writing output " << outputIndex << "' of iteration: " << x + 1
150 << " to file: '" << m_Params.m_OutputTensorFiles[outputIndex] << "'";
151 }
152 }
153 long outputSize = 1;
154 for (unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim)
155 {
156 outputSize *= outputDims->data[dim];
157 }
158
159 std::cout << m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->name << ": ";
160 results.push_back(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation);
161
162 switch (m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->type)
163 {
164
165 case kTfLiteFloat32:
166 {
167 auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<float>(
168 tfLiteDelegateOutputId);
169
170 for (int i = 0; i < outputSize; ++i)
171 {
172 fprintf(outputTensorFile, "%f ", tfLiteDelegateOutputData[i]);
173 }
174 break;
175 }
176 case kTfLiteInt32:
177 {
178 auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<int32_t>(
179 tfLiteDelegateOutputId);
180 for (int i = 0; i < outputSize; ++i)
181 {
182 fprintf(outputTensorFile, "%d ", tfLiteDelegateOutputData[i]);
183 }
184 break;
185 }
186 case kTfLiteUInt8:
187 {
188 auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<uint8_t>(
189 tfLiteDelegateOutputId);
190 for (int i = 0; i < outputSize; ++i)
191 {
192 fprintf(outputTensorFile, "%u ", tfLiteDelegateOutputData[i]);
193 }
194 break;
195 }
196 case kTfLiteInt8:
197 {
198 auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<int8_t>(
199 tfLiteDelegateOutputId);
200 for (int i = 0; i < outputSize; ++i)
201 {
202 fprintf(outputTensorFile, "%d ", tfLiteDelegateOutputData[i]);
203 }
204 break;
205 }
206 case kTfLiteBool:
207 {
208 auto tfLiteDelegateOutputData = m_TfLiteInterpreter->typed_tensor<bool>(
209 tfLiteDelegateOutputId);
210 for (int i = 0; i < outputSize; ++i) {
211 fprintf(outputTensorFile, "%u ", tfLiteDelegateOutputData[i]);
212 }
213 break;
214 }
215 default:
216 {
217 LogAndThrow("Unsupported output type");
218 }
219 }
220
221 std::cout << std::endl;
222 }
223 }
224 CheckInferenceTimeThreshold(duration, m_Params.m_ThresholdTime);
225 }
226
227 std::cout << status;
228 return results;
229 }
230
CompareAndPrintResult(std::vector<const void * > otherOutput)231 void TfLiteExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput)
232 {
233 for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex)
234 {
235 auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex];
236 size_t size = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes;
237 double result = ComputeByteLevelRMSE(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
238 otherOutput[outputIndex], size);
239 std::cout << "Byte level root mean square error: " << result << "\n";
240 }
241 };
242