1 //
2 // Copyright © 2022 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #define LOG_TAG "arm-armnn-sl"
7
8 #include "ModelToINetworkTransformer.hpp"
9 #include "CanonicalUtils.hpp"
10 #include "Converter.hpp"
11
12 #include <log/log.h>
13 #include <type_traits>
14
15 namespace armnn_driver
16 {
17
ModelToINetworkTransformer(const std::vector<armnn::BackendId> & backends,const Model & model,const std::set<unsigned int> & forcedUnsupportedOperations)18 ModelToINetworkTransformer::ModelToINetworkTransformer(
19 const std::vector<armnn::BackendId>& backends,
20 const Model& model,
21 const std::set<unsigned int>& forcedUnsupportedOperations)
22 : m_Data(backends)
23 , m_Model(model)
24 , m_ForcedUnsupportedOperations(forcedUnsupportedOperations)
25 , m_ConversionResult(ConversionResult::Success)
26 {
27 try
28 {
29 Convert();
30 }
31 catch (std::exception& e)
32 {
33 m_ConversionResult = ConversionResult::UnsupportedFeature;
34 VLOG(DRIVER) << "ModelToINetworkTransformer: Unexpected exception: " << e.what();
35 assert(false);
36 }
37 }
38
Convert()39 void ModelToINetworkTransformer::Convert()
40 {
41 VLOG(DRIVER) << "ModelToINetworkTransformer: Convert()";
42 //VLOG(DRIVER) << "ModelToINetworkTransformer: Convert(): " << GetModelSummary(m_Model).c_str();
43
44 // map the memory pool into shared pointers
45 m_Data.m_MemPools.clear();
46 if (!setRunTimePoolInfosFromCanonicalMemories(&m_Data.m_MemPools, m_Model.pools))
47 {
48 VLOG(DRIVER) << "Setting of run time pool infos from Hidl Memories has failed." << __func__;
49 m_ConversionResult = ConversionResult::ErrorMappingPools;
50 return;
51 }
52
53 using NetworkOptions = std::vector<armnn::BackendOptions>;
54 NetworkOptions networkOptions;
55 armnn::BackendOptions shapeInferenceMethodOption("ShapeInferenceMethod",
56 {
57 { "InferAndValidate", true }
58 });
59
60 networkOptions.push_back(shapeInferenceMethodOption);
61
62 // Create armnn::INetwork
63 m_Data.m_Network = armnn::INetwork::Create(networkOptions);
64
65 // add operations to it
66 // track which layer outputs each operand
67 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): m_OutputSlotForOperand";
68 m_Data.m_OutputSlotForOperand = std::vector<armnn::IOutputSlot*>(m_Model.main.operands.size(), nullptr);
69 try
70 {
71 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): for m_Model.inputIndexes.size()";
72 for (uint32_t i = 0; i < m_Model.main.inputIndexes.size(); i++)
73 {
74 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): m_Model.inputIndexes[i]";
75 // inputs in android nn are represented by operands
76 uint32_t inputIndex = m_Model.main.inputIndexes[i];
77 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): m_Model.operands[inputIndex]";
78 const Operand& operand = m_Model.main.operands[inputIndex];
79 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): GetTensorInfoForOperand(operand)";
80
81 const armnn::TensorInfo& tensor = GetTensorInfoForOperand(operand);
82 const std::string layerName = "Input_" + std::to_string(i);
83 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): m_Data.m_Network->AddInputLayer(...)";
84 armnn::IConnectableLayer* layer = m_Data.m_Network->AddInputLayer(i, layerName.c_str());
85
86 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): layer->GetOutputSlot(0)";
87 armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(0);
88 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): outputSlot.SetTensorInfo(...)";
89 outputSlot.SetTensorInfo(GetTensorInfoForOperand(operand));
90
91 VLOG(DRIVER) << "ModelToINetworkTransformer::Convert(): store for later layers";
92 // store for later layers
93 m_Data.m_OutputSlotForOperand[inputIndex] = &outputSlot;
94 }
95 }
96 catch (UnsupportedOperand<OperandType>& e)
97 {
98 VLOG(DRIVER) << __func__ << "Operand type: " << e.m_type << " is not supported in ArmnnDriver";
99 m_ConversionResult = ConversionResult::UnsupportedFeature;
100 }
101 catch (const armnn::InvalidArgumentException& e)
102 {
103 Fail("%s: Failed to convert input operand to TensorShape: %s", __func__, e.what());
104 m_ConversionResult = ConversionResult::UnsupportedFeature;
105 }
106 bool UnsupportedDynamicOperation = false;
107 for (uint32_t operationIdx = 0; operationIdx < m_Model.main.operations.size(); operationIdx++)
108 {
109 const auto& operation = m_Model.main.operations[operationIdx];
110
111 bool ok = true;
112 if (m_ForcedUnsupportedOperations.find(operationIdx) != m_ForcedUnsupportedOperations.end())
113 {
114 Fail("%s: Operation at index %i has been forced to be unsupported.", __func__, operationIdx);
115 ok = false;
116 }
117
118 if (ok)
119 {
120 try
121 {
122 ok = Converter::ConvertOperation(operation, m_Model, m_Data);
123 }
124 catch (UnsupportedOperand<OperandType>& e)
125 {
126 VLOG(DRIVER) << __func__ << "Operation type: " << e.m_type << "is not supported in ArmnnDriver";
127 ok = false;
128 }
129 catch (const armnn::InvalidArgumentException& e)
130 {
131 Fail("%s: Failed to convert operation in %s", __func__, e.what());
132 ok = false;
133 }
134 }
135
136 // Store whether this operation was successfully converted.
137 m_OperationSupported.emplace(operationIdx, ok);
138
139 // Any single operation failing will fail the entire conversion.
140 // We still need to continue and check the other ones.
141 if (!ok)
142 {
143 if (m_Data.m_DynamicInputsEncountered)
144 {
145 Fail("%s: The unsupported operation at index %i has dynamic inputs.", __func__, operationIdx);
146 UnsupportedDynamicOperation = true;
147 }
148
149 m_ConversionResult = ConversionResult::UnsupportedFeature;
150 }
151 m_Data.m_DynamicInputsEncountered = false;
152 }
153
154 // Due to the NNAPI partitioner not supporting partition boundaries of unknown size,
155 // any operations who's outputs connect to an unsupported operation with with dynamic inputs
156 // will cause a failure.
157
158 // The simplest solution to this problem is to not support any operations in a model containing
159 // an unsupported operation with with dynamic inputs.
160 if (UnsupportedDynamicOperation)
161 {
162 Fail("%s: Unsupported operation with dynamic inputs found. Retroactively setting all operations to unsupported",
163 __func__);
164 for (auto& operation : m_OperationSupported)
165 {
166 operation.second = false;
167 }
168 }
169
170 try
171 {
172 if (m_ConversionResult == ConversionResult::Success)
173 {
174 for (uint32_t i = 0; i < m_Model.main.outputIndexes.size(); i++)
175 {
176 // outputs in android nn are represented by operands
177 uint32_t outputIndex = m_Model.main.outputIndexes[i];
178 const auto& operand = m_Model.main.operands[outputIndex];
179 const armnn::TensorInfo& tensor = GetTensorInfoForOperand(operand);
180 const std::string layerName = "Output_" + std::to_string(i);
181 armnn::IConnectableLayer* layer = m_Data.m_Network->AddOutputLayer(i, layerName.c_str());
182
183 assert(m_Data.m_OutputSlotForOperand[outputIndex]);
184 m_Data.m_OutputSlotForOperand[outputIndex]->Connect(layer->GetInputSlot(0));
185 }
186 }
187 }
188 catch (const armnn::InvalidArgumentException& e)
189 {
190 Fail("%s: Failed to convert output operand to TensorShape: %s", __func__, e.what());
191 m_ConversionResult = ConversionResult::UnsupportedFeature;
192 }
193 }
194
IsOperationSupported(uint32_t operationIndex) const195 bool ModelToINetworkTransformer::IsOperationSupported(uint32_t operationIndex) const
196 {
197 std::map<uint32_t, bool>::const_iterator it = m_OperationSupported.find(operationIndex);
198 assert(it != m_OperationSupported.end());
199 return it->second;
200 }
201
202 } // armnn_driver
203