xref: /aosp_15_r20/external/armnn/delegate/classic/src/UnidirectionalSequenceLstm.hpp (revision 89c4ff92f2867872bb9e2354d150bf0c8c502810) 
1 //
2 // Copyright © 2022-2023 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 
6 #pragma once
7 
8 #include <ClassicDelegateUtils.hpp>
9 
10 #include <armnn/LstmParams.hpp>
11 #include <armnn/Tensor.hpp>
12 #include <armnn/utility/IgnoreUnused.hpp>
13 
14 #include <tensorflow/lite/builtin_ops.h>
15 #include <tensorflow/lite/c/builtin_op_data.h>
16 #include <tensorflow/lite/c/common.h>
17 #include <tensorflow/lite/minimal_logging.h>
18 
19 namespace armnnDelegate
20 {
21 
VisitUnidirectionalSequenceLstmOperator(DelegateData & delegateData,TfLiteContext * tfLiteContext,TfLiteNode * tfLiteNode,int nodeIndex,int32_t operatorCode)22 TfLiteStatus VisitUnidirectionalSequenceLstmOperator(DelegateData& delegateData,
23                                                      TfLiteContext* tfLiteContext,
24                                                      TfLiteNode* tfLiteNode,
25                                                      int nodeIndex,
26                                                      int32_t operatorCode)
27 {
28     auto numInputs = tfLiteNode->inputs->size;
29     if (numInputs < 2)
30     {
31         TF_LITE_MAYBE_KERNEL_LOG(
32                 tfLiteContext, "TfLiteArmnnDelegate: Minimum number of inputs (%d != %d) in node #%d",
33                 2, numInputs, nodeIndex);
34         return kTfLiteError;
35     }
36 
37     const auto nodeParams = reinterpret_cast<TfLiteUnidirectionalSequenceLSTMParams *>(tfLiteNode->builtin_data);
38     const TfLiteTensor* tfLiteTensors = tfLiteContext->tensors;
39 
40     const TfLiteTensor& tfLiteInputTensor = tfLiteTensors[tfLiteNode->inputs->data[0]];
41     if (!IsValid(tfLiteContext, tfLiteInputTensor, operatorCode, nodeIndex))
42     {
43         return kTfLiteError;
44     }
45 
46     const TfLiteTensor& tfLiteOutputTensor = tfLiteTensors[tfLiteNode->outputs->data[0]];
47     if (!IsValid(tfLiteContext, tfLiteOutputTensor, operatorCode, nodeIndex))
48     {
49         return kTfLiteError;
50     }
51 
52     // Set the params structure for the AddUnidirectionalSequenceLstmLayer call
53     // Please refer to each operand at
54     // https://www.tensorflow.org/mlir/tfl_ops#tflunidirectional_sequence_lstm_tflunidirectionalsequencelstmop
55     armnn::LstmInputParams params;
56 
57     if (IsOptionalOperandPresent(tfLiteNode, 1))
58     {
59         params.m_InputToInputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 1);
60     }
61 
62     params.m_InputToForgetWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 2);
63     params.m_InputToCellWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 3);
64     params.m_InputToOutputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 4);
65 
66     // Recurrent weight tensors of size {n_cell, n_output}
67     if (IsOptionalOperandPresent(tfLiteNode, 5))
68     {
69         params.m_RecurrentToInputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 5);
70     }
71 
72     params.m_RecurrentToForgetWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 6);
73     params.m_RecurrentToCellWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 7);
74     params.m_RecurrentToOutputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 8);
75 
76     // Peephole weights tensors of size {n_cell}, representing a diagonal matrix.
77     if (IsOptionalOperandPresent(tfLiteNode, 9))
78     {
79         params.m_CellToInputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 9);
80     }
81 
82     if (IsOptionalOperandPresent(tfLiteNode, 10))
83     {
84         params.m_CellToForgetWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 10);
85     }
86 
87     if (IsOptionalOperandPresent(tfLiteNode, 11))
88     {
89         params.m_CellToOutputWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 11);
90     }
91 
92     // Gates bias tensors of size {n_cell}
93     if (IsOptionalOperandPresent(tfLiteNode, 12))
94     {
95         params.m_InputGateBias = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 12);
96     }
97 
98     params.m_ForgetGateBias = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 13);
99     params.m_CellBias = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 14);
100     params.m_OutputGateBias = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 15);
101 
102     // Projection weight tensor of size {n_output, n_cell}
103     if (IsOptionalOperandPresent(tfLiteNode, 16))
104     {
105         params.m_ProjectionWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 16);
106     }
107     // Projection bias tensor of size {n_output}
108     if (IsOptionalOperandPresent(tfLiteNode, 17))
109     {
110         params.m_ProjectionBias = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 17);
111     }
112 
113     // These state tensors are defined as variable tensors, and will be modified by this op.
114     armnn::TensorInfo outputStateInInfo = GetTensorInfoForTfLiteTensor(tfLiteTensors[tfLiteNode->inputs->data[18]]);
115     armnn::TensorInfo cellStateInInfo = GetTensorInfoForTfLiteTensor(tfLiteTensors[tfLiteNode->inputs->data[19]]);
116 
117     // Layer norm coefficient tensors of size {n_cell}, representing a diagonal matrix.
118     if (IsOptionalOperandPresent(tfLiteNode, 20))
119     {
120         params.m_InputLayerNormWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 20);
121     }
122 
123     if (IsOptionalOperandPresent(tfLiteNode, 21))
124     {
125         params.m_ForgetLayerNormWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 21);
126     }
127 
128     if (IsOptionalOperandPresent(tfLiteNode, 22))
129     {
130         params.m_CellLayerNormWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 22);
131     }
132 
133     if (IsOptionalOperandPresent(tfLiteNode, 23))
134     {
135         params.m_OutputLayerNormWeights = GetConstTensorForTfLiteTensor(tfLiteTensors, tfLiteNode, 23);
136     }
137 
138     // set the layer descriptor
139     armnn::UnidirectionalSequenceLstmDescriptor desc;
140     desc.m_ActivationFunc    = NonNegative(nodeParams->activation, nodeIndex);
141     desc.m_ClippingThresCell = nodeParams->cell_clip;
142     desc.m_ClippingThresProj = nodeParams->proj_clip;
143     desc.m_CifgEnabled       = (params.m_InputToInputWeights == nullptr
144                                 || params.m_RecurrentToInputWeights == nullptr
145                                 || params.m_InputGateBias == nullptr);
146     desc.m_PeepholeEnabled   = (params.m_CellToForgetWeights != nullptr || params.m_CellToOutputWeights != nullptr);
147     desc.m_ProjectionEnabled = (params.m_ProjectionWeights != nullptr);
148     desc.m_LayerNormEnabled  = (params.m_InputLayerNormWeights != nullptr
149                                 || params.m_ForgetLayerNormWeights != nullptr
150                                 || params.m_CellLayerNormWeights != nullptr
151                                 || params.m_OutputLayerNormWeights != nullptr);
152     desc.m_TimeMajor = nodeParams->time_major;
153 
154     if (tfLiteNode->intermediates->size > 3 && desc.m_LayerNormEnabled)
155     {
156         auto inputIntermediateTensorInfo = GetTensorInfoForTfLiteTensor(
157                 tfLiteTensors[tfLiteNode->intermediates->data[0]]);
158         auto forgetIntermediateTensorInfo = GetTensorInfoForTfLiteTensor(
159                 tfLiteTensors[tfLiteNode->intermediates->data[1]]);
160         auto cellIntermediateTensorInfo = GetTensorInfoForTfLiteTensor(
161                 tfLiteTensors[tfLiteNode->intermediates->data[2]]);
162         auto outputIntermediateTensorInfo = GetTensorInfoForTfLiteTensor(
163                 tfLiteTensors[tfLiteNode->intermediates->data[3]]);
164 
165         desc.m_InputIntermediateScale = inputIntermediateTensorInfo.GetQuantizationScale();
166         desc.m_ForgetIntermediateScale = forgetIntermediateTensorInfo.GetQuantizationScale();
167         desc.m_CellIntermediateScale = cellIntermediateTensorInfo.GetQuantizationScale();
168         desc.m_OutputIntermediateScale = outputIntermediateTensorInfo.GetQuantizationScale();
169     }
170     else
171     {
172         float defaultIntermediate = std::pow(2, -12);
173         desc.m_InputIntermediateScale = defaultIntermediate;
174         desc.m_ForgetIntermediateScale = defaultIntermediate;
175         desc.m_CellIntermediateScale = defaultIntermediate;
176         desc.m_OutputIntermediateScale = defaultIntermediate;
177     }
178     if (tfLiteNode->intermediates->size > 4)
179     {
180         auto hiddentensorInfo = GetTensorInfoForTfLiteTensor(tfLiteTensors[tfLiteNode->intermediates->data[4]]);
181         desc.m_HiddenStateScale = hiddentensorInfo.GetQuantizationScale();
182         desc.m_HiddenStateZeroPoint = hiddentensorInfo.GetQuantizationOffset();
183     }
184     const armnn::TensorInfo& inputTensorInfo = GetTensorInfoForTfLiteTensor(tfLiteInputTensor);
185     const armnn::TensorInfo& outputTensorInfo = GetTensorInfoForTfLiteTensor(tfLiteOutputTensor, true);
186 
187     unsigned int batchSize  = inputTensorInfo.GetShape()[0];
188     unsigned int outputSize = outputTensorInfo.GetShape()[2];
189     unsigned int numUnits   = cellStateInInfo.GetShape()[1];
190 
191     armnn::DataType dataType = inputTensorInfo.GetDataType();
192     float qScale = inputTensorInfo.GetQuantizationScale();
193     float qOffset = inputTensorInfo.GetQuantizationOffset();
194 
195     armnn::TensorInfo scratchBufferTensorInfo({batchSize, numUnits * 3}, dataType, qScale, qOffset);
196     if (!desc.m_CifgEnabled)
197     {
198         scratchBufferTensorInfo = armnn::TensorInfo({batchSize, numUnits * 4}, dataType, qScale, qOffset);
199     }
200     armnn::TensorInfo cellStateOutTensorInfo({batchSize, numUnits},
201                                              cellStateInInfo.GetDataType(),
202                                              cellStateInInfo.GetQuantizationScale(),
203                                              cellStateInInfo.GetQuantizationOffset());
204 
205     armnn::TensorInfo outputStateOutTensorInfo({batchSize, outputSize}, dataType, qScale, qOffset);
206 
207     armnn::LstmInputParamsInfo paramsInfo;
208     paramsInfo.m_InputToForgetWeights     = &(params.m_InputToForgetWeights->GetInfo());
209     paramsInfo.m_InputToCellWeights       = &(params.m_InputToCellWeights->GetInfo());
210     paramsInfo.m_InputToOutputWeights     = &(params.m_InputToOutputWeights->GetInfo());
211     paramsInfo.m_RecurrentToForgetWeights = &(params.m_RecurrentToForgetWeights->GetInfo());
212     paramsInfo.m_RecurrentToCellWeights   = &(params.m_RecurrentToCellWeights->GetInfo());
213     paramsInfo.m_RecurrentToOutputWeights = &(params.m_RecurrentToOutputWeights->GetInfo());
214     paramsInfo.m_ForgetGateBias           = &(params.m_ForgetGateBias->GetInfo());
215     paramsInfo.m_CellBias                 = &(params.m_CellBias->GetInfo());
216     paramsInfo.m_OutputGateBias           = &(params.m_OutputGateBias->GetInfo());
217 
218     if (!desc.m_CifgEnabled)
219     {
220         paramsInfo.m_InputToInputWeights = &(params.m_InputToInputWeights->GetInfo());
221         paramsInfo.m_RecurrentToInputWeights = &(params.m_RecurrentToInputWeights->GetInfo());
222         if (params.m_CellToInputWeights != nullptr)
223         {
224             paramsInfo.m_CellToInputWeights = &(params.m_CellToInputWeights->GetInfo());
225         }
226         paramsInfo.m_InputGateBias = &(params.m_InputGateBias->GetInfo());
227     }
228 
229     if (desc.m_ProjectionEnabled)
230     {
231         paramsInfo.m_ProjectionWeights = &(params.m_ProjectionWeights->GetInfo());
232         if (params.m_ProjectionBias != nullptr)
233         {
234             paramsInfo.m_ProjectionBias = &(params.m_ProjectionBias->GetInfo());
235         }
236     }
237 
238     if (desc.m_PeepholeEnabled)
239     {
240         paramsInfo.m_CellToForgetWeights = &(params.m_CellToForgetWeights->GetInfo());
241         paramsInfo.m_CellToOutputWeights = &(params.m_CellToOutputWeights->GetInfo());
242     }
243 
244     if (desc.m_LayerNormEnabled)
245     {
246         if(!desc.m_CifgEnabled)
247         {
248             paramsInfo.m_InputLayerNormWeights = &(params.m_InputLayerNormWeights->GetInfo());
249         }
250         paramsInfo.m_ForgetLayerNormWeights = &(params.m_ForgetLayerNormWeights->GetInfo());
251         paramsInfo.m_CellLayerNormWeights = &(params.m_CellLayerNormWeights->GetInfo());
252         paramsInfo.m_OutputLayerNormWeights = &(params.m_OutputLayerNormWeights->GetInfo());
253     }
254 
255     bool isSupported = false;
256     armnn::BackendId setBackend;
257     auto validateFunc = [&](const armnn::TensorInfo& outputInfo, bool& isSupported)
258     {
259         FORWARD_LAYER_SUPPORT_FUNC("UNIDIRECTIONAL_SEQUENCE_LSTM",
260                                    tfLiteContext,
261                                    IsUnidirectionalSequenceLstmSupported,
262                                    delegateData.m_Backends,
263                                    isSupported,
264                                    setBackend,
265                                    inputTensorInfo,
266                                    outputStateInInfo,
267                                    cellStateInInfo,
268                                    outputStateOutTensorInfo,
269                                    cellStateOutTensorInfo,
270                                    outputInfo,
271                                    desc,
272                                    paramsInfo);
273     };
274 
275     if (!delegateData.m_Network)
276     {
277         validateFunc(outputTensorInfo, isSupported);
278         return isSupported ? kTfLiteOk : kTfLiteError;
279     }
280 
281     armnn::IConnectableLayer* layer = delegateData.m_Network->AddUnidirectionalSequenceLstmLayer(desc, params);
282     layer->SetBackendId(setBackend);
283     ARMNN_ASSERT(layer != nullptr);
284 
285     layer->GetOutputSlot(0).SetTensorInfo(outputStateOutTensorInfo);
286     layer->GetOutputSlot(1).SetTensorInfo(cellStateOutTensorInfo);
287     layer->GetOutputSlot(2).SetTensorInfo(outputTensorInfo);
288 
289     // Connect the inputs
290     // input_layer
291     delegateData.m_OutputSlotForNode[tfLiteNode->inputs->data[0]]->Connect(layer->GetInputSlot(0));
292     // cellStateIn
293     delegateData.m_OutputSlotForNode[tfLiteNode->inputs->data[18]]->Connect(layer->GetInputSlot(1));
294     //outputStateIn
295     delegateData.m_OutputSlotForNode[tfLiteNode->inputs->data[19]]->Connect(layer->GetInputSlot(2));
296 
297     armnn::IOutputSlot& outputSlot = layer->GetOutputSlot(2);
298     delegateData.m_OutputSlotForNode[static_cast<unsigned long>(tfLiteNode->outputs->data[0])] = &outputSlot;
299     return kTfLiteOk;
300 }
301 
302 } // namespace armnnDelegate