1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #include "tensorflow/lite/delegates/nnapi/acceleration_test_util.h" 16 17 namespace tflite { 18 19 const char* const NnapiAccelerationTestParams::kAccelerationTestConfig = 20 R"( 21 ## Every Test can be allowlisted or denylisted using a regexp on its test_id 22 23 ## Test_id 24 # 25 # The test_id is test_suite_name / test_name, this differs from the 26 # name used by the build because of the / separator instead of . 27 # Parameterized tests names are composed by the base test name / test / ordinal 28 # the ordinal is the position in the list of parameters generated by the 29 # cardinal product of all the different parameter sets 30 31 # Denylist/Allowlist 32 # To denylist an element simply add - before the test_id regex 33 34 ## Rules evaluation 35 # 36 # Rules are checked in order, the first matching completes the browsing 37 # This can be useful to put more specific rules first and generic default 38 # ones below 39 40 ## Test Arguments 41 # 42 # The test can be parameterized with the minimum Android SDK version 43 # to apply the acceleration validation for. 44 # If omitted will use 27 45 46 #test-id,min-android-sdk-version 47 48 # activations_test 49 FloatActivationsOpTest/Softmax[13]D,29 50 QuantizedActivationsOpTest/Softmax[13]D.+nt8,29 51 FloatActivationsOpTest/Softmax\dD 52 QuantizedActivationsOpTest/Softmax\dD.+nt8 53 FloatActivationsOpTest/LogSoftmax,29 54 FloatActivationsOpTest/PRelu,29 55 LogisticOpTest/LogisticOpTest/Sigmoid(.+nt8)?/\d+ 56 LogisticOpTest/LogisticOpTest/Sigmoid/\d+ 57 TanhOpTest/TanhOpTest/Tanh(.+nt8)?/\d+,29 58 FloatActivationsOpTest/Elu,30 59 FloatActivationsOpTest/Relu 60 FloatActivationsOpTest/Relu1 61 FloatActivationsOpTest/Relu6 62 FloatActivationsOpTest/HardSwish 63 FloatActivationsOpTest/LeakyRelu,29 64 QuantizedActivationsOpTest/HardSwish 65 QuantizedActivationsOpTest/HardSwishBias 66 -QuantizedActivationsOpTest/Relu.?Int16 67 QuantizedActivationsOpTest/Relu.* 68 -QuantizedActivationsOpTest/LeakyReluInt16,30 69 QuantizedActivationsOpTest/LeakyRelu.*,30 70 QuantizedActivationsOpTest/Relu.+nt8 71 QuantizedActivationsOpTest/PRelu,29 72 QuantizedActivationsOpTest/PReluSameShapes,29 73 QuantizedActivationsOpTest/PReluInt8.+,30 74 PReluOpTest/.*,29 75 76 77 # add_test 78 FloatAddOpModel/.+ 79 QuantizedAddOpModel/QuantizedTestsNoActivation.+nt8 80 QuantizedAddOpModel/QuantizedVariousInputShapes.+ 81 QuantizedAddOpModel/QuantizedWithScalarBroadcast.+nt8 82 QuantizedAddOpModel/QuantizedWithMixedBroadcas.+nt8 83 84 # arg_min_max_test 85 # Only tests with ConstantAxis && OutputType == TensorType_INT32 86 # (element 4 and 6 in the test parameter list) 87 # Supported only from NNAPI 1.2 88 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgFloat/[46],29 89 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+Arg.+nt8/[46],29 90 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgInt/[46],29 91 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgMulDimensions/[46],29 92 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgNegativeAxis/[46],29 93 ArgMinMaxOpTest/ArgMinMaxOpTest/Get.+ArgOutput64/[46],29 94 95 # basic_rnn_test 96 RnnOpTest/BlackBoxTest 97 98 # batch_matmul_test 99 # broadcasting is not supported 100 -BatchMatMulOpTest/BatchMatMulOpTest/.+Broadcast.+ 101 BatchMatMulOpTest/BatchMatMulOpTest/.+,1000006 102 QuantizedBatchMatMulOpTest/QuantizedBatchMatMulOpTest/SimpleTestQuantizedInt8/.+,1000006 103 104 # batch_to_space_nd_test 105 BatchToSpaceNDOpTest/SimpleConstTest.* 106 BatchToSpaceNDOpTest/BatchOneConstTest.* 107 108 # bidirectional_sequence_lstm_test 109 # Only test with non quantized weights 110 LSTMOpTest/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29 111 # Only test with non quantized weights 112 LSTMOpTest/LSTMOpTest/BlackBoxTestMergedOutput/0,29 113 LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClippingReverse,29 114 LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClipping,29 115 LSTMOpTest/BlackBoxTestWithCifgWithPeepholeNoProjectionNoClippingReversed,29 116 LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClipping,29 117 LSTMOpTest/BlackBoxTestWithPeepholeWithProjectionNoClippingBatchMajor,29 118 # Only test with non quantized weights 119 LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29 120 QuantizationOrNot/LSTMOpTest/BlackBoxTestNoCifgNoPeepholeNoProjectionNoClipping/0,29 121 QuantizationOrNot/LSTMOpTest/BlackBoxTestMergedOutput/0,29 122 QuantizationOrNot/LSTMOpTest/BlackBoxTestWithAuxInputZeroAuxWeight/0,29 123 LSTMOpTest/LSTMOpTest/BlackBoxTestWithAuxInput/0,29 124 125 # cast_test 126 CastOpModel/CastFloatToIn32 127 CastOpModel/CastInt32ToFloat,29 128 CastOpModel/CastFloatToUInt8,29 129 CastOpModel/CastUInt8ToFloat,29 130 CastOpModel/CastInt32ToUInt8,29 131 CastOpModel/CastUInt8ToInt32,29 132 133 # comparisons_test 134 ComparisonsTest/.+,29 135 136 # concatenation_test 137 ConcatenationOpTest/ThreeDimensionalOneInput 138 ConcatenationOpTest/OneTrivialInput 139 ConcatenationOpTest/TwoDimensionalOneInput 140 ConcatenationOpTest/TwoInputsTwoAxesNegativeAxes 141 ConcatenationOpTest/TwoInputsTwoAxesNegativeAxesNonQuantized 142 ConcatenationOpTest/FourInputs 143 ConcatenationOpTest/FourInputsQuantizedUint8 144 ConcatenationOpTest/FourInputsQuantizedInt8 145 ConcatenationOpTest/ThreeDimensionalNonQuantizedOneInput 146 ConcatenationOpTest/OneTrivialNonQuantizedInput 147 ConcatenationOpTest/TwoDimensionalNonQuantizedOneInput 148 ConcatenationOpTest/FourInputsQuantizedMixedRange,29 149 ConcatenationOpTest/FourInputsQuantizedMixedRangeClampingLogic,29 150 151 # conv_test 152 -ConvolutionOpTest/ConvolutionOpTest.SimplePerTensorTest/.+ 153 ConvolutionOpTest/ConvolutionOpTest.SimpleTestFloatWithDilation/.+,29 154 ConvolutionOpTest/ConvolutionOpTest.SimpleTestLargeIrregularQuantized/.+,29 155 ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedOutputMultiplierGreaterThan1/.+,29 156 ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedWithDilation/.+,29 157 ConvolutionOpTest/ConvolutionOpTest.SimplePerChannelTest/.+,29 158 ConvolutionOpTest/ConvolutionOpTest.SimpleTestQuantizedGrouped/.+,29 159 ConvolutionOpTest/ConvolutionOpTest/.+Hybrid.+,29/ 160 ConvolutionOpTest/ConvolutionOpTest/.+/\d+ 161 162 163 # depthwise_conv_test 164 .+ConvolutionOpTest/.+/\d+,29 165 166 # dequantize_test 167 DequantizeOpTest/Uint8 168 DequantizeOpTest/Int8,30 169 170 # depth_to_space_test 171 DepthToSpaceOpModel/Float32 172 DepthToSpaceOpModel/UInt8 173 DepthToSpaceOpModel/int8 174 175 # div_test 176 -FloatDivOpTest/WithBroadcast5D 177 FloatDivOpTest/.+ 178 179 # elementwise_test 180 ElementWise/Abs,29 181 ElementWise/Sin,29 182 ElementWise/Log,29 183 ElementWise/Sqrt,29 184 ElementWise/Rsqrt,29 185 ElementWise/LogicalNot,29 186 187 # embedding_lookup_test 188 EmbeddingLookupOpTest/SimpleTest 189 190 # exp_test 191 ExpOpTest/FloatTest,29 192 193 # expand_dims_test 194 # Only constant tensors models 195 ExpandDimsOpTest/.+/1,29 196 197 # fill_test 198 FillOpTest/FillOpTest/FillInt32/0,30 199 FillOpTest/FillOpTest/FillFloat/0,30 200 FillOpTest/FillOpTest/FillFloatInt32Dims/0,30 201 202 # floor_test 203 FloorOpTest/.+ 204 205 # fully_connected_test 206 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest/\d+ 207 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest2/\d+ 208 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestQuantized.+nt8/\d+,29 209 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTestSingleBatchQuantizedInt8/\d+,29 210 QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29 211 QuantizedFullyConnectedOpTest/SimpleTestQuantizedOutputMultiplierGreaterThan1Int8/\d+,29 212 HybridFullyConnectedOpTest/SimpleTestQuantizedUint8,29 213 HybridFullyConnectedOpTest/SimpleTestQuantizedInt8,29 214 HybridAsymmetricInputFullyConnectedOpTest.SimpleTestQuantizedUint8,29 215 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/SimpleTest4DInput/\d+ 216 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedUint8/\d+ 217 QuantizedFullyConnectedOpTest/QuantizedFullyConnectedOpTest/SimpleTest4dInputQuantizedOutputMultiplierGreaterThan1Uint8/\d+,29 218 FloatFullyConnectedOpTest/FloatFullyConnectedOpTest/BlackBoxTest/\d+ 219 FloatFullyConnectedOpTest/SimpleTestNoBias 220 221 # gather_test 222 GatherOpTest/Shuffle,29 223 GatherOpTest/Test1DInput1DIndex,29 224 GatherOpTest/Test2DIndexWith2DResult,29 225 FloatGatherOpTest/Duplicate,29 226 FloatGatherOpTest/Slice,29 227 FloatGatherOpTest/Axis1,29 228 FloatGatherOpTest/Axis1Slice,29 229 FloatGatherOpTest/LastAxis,29 230 TypesGatherOpTest/Float32Int32,29 231 TypesGatherOpTest/Int32Int32,29 232 TypesGatherOpTest/Uint8Int32,29 233 TypesGatherOpTest/Int8Int32,29 234 -TypesGatherOpTest/.*Int16.* 235 236 # hashtable_lookup_test 237 # All test excepted the string one should be accelerated 238 -HashtableLookupOpTest/TestString 239 HashtableLookupOpTest/.+ 240 241 # l2norm_test 242 L2NormOpTest/.+,29 243 244 # local_response_norm_test 245 LocalResponseNormOpTest/.+ 246 247 # logical_test 248 LogicalTest/.+,29 249 250 # lsh_projection_test 251 -LSHProjectionOpTest2/Sparse3DInputs 252 LSHProjectionOpTest2/Sparse1DInputs,29 253 LSHProjectionOpTest2/.+ 254 255 # Before the lstm because of clashing with matchers 256 # unidirectional_sequence_lstm_test 257 NoCifgNoPeepholeNoProjectionNoClippingUnidirectionalLstmTest/LstmBlackBoxTest,29 258 CifgPeepholeNoProjectionNoClippingUnidirectionalLstmTest/NonLayerNormLstmBlackBoxTest,29 259 # Only the two tests above, disabling all possible matches from the lstm tests 260 # coming after 261 -.+UnidirectionalLstmTest/.+ 262 263 # lstm_test 264 -LstmOpTest/InvalidTypes 265 # Float 266 Parameterized/LstmOpTest.+/0,29 267 Parameterized/LstmOpTest.+/1,29 268 Parameterized/LstmOpTest.+/2,29 269 Parameterized/LstmOpTest.+/3,29 270 # HybridUint8 271 Parameterized/LstmOpTest.+/4,29 272 Parameterized/LstmOpTest.+/5,29 273 Parameterized/LstmOpTest.+/6,29 274 Parameterized/LstmOpTest.+/7,29 275 # HybridInt8 276 -Parameterized/LstmOpTest.+/8 277 -Parameterized/LstmOpTest.+/9 278 -Parameterized/LstmOpTest.+/10 279 -Parameterized/LstmOpTest.+/11 280 281 # maximum_minimum_test 282 MaxMinOpTest/.+nt8Test,29 283 MaximumOpTest/.+,29 284 285 # mirror_pad_test 286 MirrorPadTest/.+,1000007 287 288 # mul_test 289 FloatMulOpTest/.+ 290 291 # neg_test 292 -NegOpModel/.+Int64 293 NegOpModel/.+,29 294 295 # pack_test 296 # int32 and uint8 are supported since NNAPI FL6 297 PackOpTest/Int32.+,1000006 298 PackOpTestInt/1/.+,1000006 299 # PACK along last axis is supported since NNAPI FL6 300 PackOpTest/FloatThreeInputsDifferentAxis,1000006 301 PackOpTest/FloatThreeInputsNegativeAxis,1000006 302 PackOpTestInt/0/ThreeInputsDifferentAxis,1000006 303 PackOpTestInt/0/ThreeInputsNegativeAxis,1000006 304 # f32 and int8 are supported since NNAPI 1.3 by decomposition 305 PackOpTest/Float.+,30 306 PackOpTestInt/0/.+,30 307 308 # pad_test 309 -PadOpTest/TooManyDimensions 310 -PadOpTest/UnequalDimensions 311 -PadOpTest/InvalidPadValue 312 # Zero height or width is not supported 313 -PadOpTest/Zero.+ConstImageStyleTest 314 # Dynamic tensors are not supported 315 -.*Pad.*OpTest/.+Dynamic.*Test 316 -QuantizedPad.*OpTest/.+ZeroNotInQuantizationRange 317 # 16-bit tests are not supported 318 -QuantizedPadOpTest/.*Int16.* 319 QuantizedPadOpTest/.+,29 320 QuantizedPadV2OpTest/.+,29 321 PadOpTest/.+,29 322 323 # pooling_test 324 FloatPoolingOpTest/L2PoolActivationRelu.*,29 325 FloatPoolingOpTest/.+ 326 # Image is too big 327 -QuantizedPoolingOpTest/AveragePoolImageSize17 328 # Int16 unsupported 329 -QuantizedPoolingOpTest/SymmetricAveragePool16 330 QuantizedPoolingOpTest/.+ 331 QuantizedUInt8PoolingOpTest/.+ 332 333 # pow_test 334 -PowOpModel/Simple 335 -PowOpModel/NegativeAndZeroValue 336 -PowOpModel/BroadcastTest 337 -PowOpModel/IntSingleIntegerExponentTest 338 PowOpModel/.+,29 339 340 # quant_basic_lstm_test 341 QuantizedLstmTest/BasicQuantizedLstmTest/29 342 343 # quantized_lstm op test 344 # Temporary disabled due to b/188515203 345 //IntegerLstmOpTest/NoCifg_NoPeephole_Projection_LayerNorm,30 346 347 # quantize_test 348 QuantizeOpTest/UINT8,29 349 QuantizeOpTest/UInt8UInt8.+,29 350 QuantizeOpTest/Int8Int8.+,30 351 QuantizeOpTest/INT8,30 352 353 # rank 354 355 # reduce_test 356 -Dynamic.+(Mean|Sum|Prod|Max|Min)OpTest/.+ 357 -ConstUint8SumOpTest/.+ 358 ConstUint8(Max|Min)OpTest/.+,29 359 ConstUint8(Mean)OpTest/.+,29 360 -ConstInt8(Max|Min)OpTest/.+,29 361 ConstInt8MeanOpTest/.+,29 362 -ConstMeanOpTest.*/.+Int16 363 ConstMeanOpTest.*/.+,29 364 -MeanOpTestQuantized.*/.+Int16 365 MeanOpTestQuantized.*/.+,29 366 ConstFloat(Sum|Prod|Max|Min)OpTest/NotKeepDims,29 367 ConstFloat(Sum|Prod|Max|Min)OpTest/KeepDims,29 368 ConstFloat(Mean|Any)OpTest/NotKeepDims 369 ConstFloat(Mean|Any)OpTest/KeepDims 370 ConstFloat(Sum|Prod|Max|Min)OpTest/ScalarAxis,29 371 372 # reshape_test 373 # Acceleration would be only for the test with shape being a constant tensor or 374 # as hardcoded options. 375 ReshapeOpTest/[01]/InvalidShape 376 ReshapeOpTest/[01]/RegularShapes 377 ReshapeOpTest/[01]/WithStretchDimension 378 # int32 is supported since NNAPI FL6 379 ReshapeOpTest/3/InvalidShape,1000006 380 ReshapeOpTest/3/RegularShapes,1000006 381 ReshapeOpTest/3/WithStretchDimension,1000006 382 383 # resize_bilinear_test 384 // align_corners & half_pixel_centers are not implemented in NNAPI before API 30 385 ResizeBilinearOpTest/ResizeBilinearOpTest.+HalfPixelCenters.*/0,30 386 // Only models with constant size tensor are accelerated 387 ResizeBilinearOpTest/ResizeBilinearOpTest/.+/0,29 388 389 # resize_nearest_neighbor_test 390 // align_corners & half_pixel_centers are not implemented in NNAPI before API 30 391 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+AlignCorners.*/0,30 392 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest.+HalfPixelCenters.*/0,30 393 // 16-bit tests are not supported 394 -ResizeNearestNeighborOpTest.+Int16/.+ 395 // Only models with constant size tensor are accelerated 396 ResizeNearestNeighborOpTest/ResizeNearestNeighborOpTest/.+/0,29 397 398 # reverse_test 399 -ReverseOpTest/Int64.+ 400 -ReverseOpTest/Int16.+ 401 ReverseOpTest/.+,1000007 402 403 # select_test 404 -SelectOpTest/SelectBool 405 -SelectOpTest.SelectInt16 406 -SelectOpTest/RankZero.+ 407 -SelectOpTest/RankOne.+ 408 SelectOpTest/.+,29 409 410 # slice_test 411 -SliceOpTest/SliceOpTest/IndexInt64/.+ 412 -SliceOpTest/SliceOpTest/SliceString/.+ 413 -SliceOpTest/SliceOpTest/SliceInt64/.+ 414 -SliceOpTest/SliceOpTest/SliceBool/.+ 415 -SliceOpTest/SliceOpTest/SliceInt16/.+ 416 -SliceOpTest/SliceOpTest/SliceInt64StaticOutput/.* 417 # Only constant tensors 418 SliceOpTest/SliceOpTest/.+/0,29 419 420 # softmax_test 421 SoftmaxOpTest/CompareWithTFminiBetaEq1 422 SoftmaxOpTest/CompareWithTFminiBetaNotEq1 423 424 # space_to_depth_test 425 SpaceToDepthOpModel/Float32 426 SpaceToDepthOpModel/Uint8 427 SpaceToDepthOpModel/int8 428 429 # split_test 430 -SplitOpTest/SplitOpTest/.+Int8/.+ 431 # Only accelerated when axis is a constant tensor 432 SplitOpTest/SplitOpTest/.+/0,29 433 434 # split_v_test 435 # NNAPI does not support int16 436 -SplitVOpTypedTest/3/.+ 437 # NNAPI does not support zero-sized slice 438 -SplitVOpTypedTest/.+OneDimensional2 439 # Only accelerated when both split_sizes and axis are constant 440 SplitVOpTypedTest/.+/ConstSplits.+,30 441 442 # squared_difference_test 443 FloatSquaredDifferenceOpTest/.+,28 444 (Integer|Quantized)SquaredDifferenceOpTest/.+,30 445 446 # squeeze_test 447 FloatSqueezeOpTest/.+,29 448 449 # sub_test 450 -FloatSubOpModel/WithBroadcast5D 451 FloatSubOpModel/.+ 452 -QuantizedSubOpModel/.+Int16 453 -QuantizedSubOpModel/.+Int8 454 QuantizedSubOpModel/.+ 455 456 # svdf_test 457 SVDFOpTest/BlackBoxTestRank1 458 SVDFOpTest/BlackBoxTestRank2 459 460 # tile_test 461 -TileTest/TileTest/Int64.+/.+ 462 -TileTest/TileTest/Boolean.+/.+ 463 -TileTest/TileTest/String.+/.+ 464 # Const tensor only 465 TileTest/TileTest/.+/0,29 466 467 # topk_v2_test 468 -TopKV2OpTest/TopKV2OpTest/.+Int64/.+ 469 # Const tensor only 470 TopKV2OpTest/TopKV2OpTest/.+/0,29 471 472 # transpose_test 473 # death test 474 -TransposeTest/Test6DInputTensor 475 -TransposeTest/5DDividedIntoTwo2Ds.* 476 -TransposeTest/Complex5DTest.* 477 -TransposeTest/.+DynamicTensor 478 -TransposeTest/TestRefOps4DInt16 479 TransposeTest/.+ 480 481 # transpose_conv_test 482 -TransposeConvOpTest/TransposeConvOpTest.SimpleTestQuantizedPerChannel16x8/.+ 483 TransposeConvOpTest/TransposeConvOpTest..*Bias.*/0,29 484 # Const tensor only 485 TransposeConvOpTest/TransposeConvOpTest/.+/0,29 486 487 # unidirectional_sequence_rnn_test 488 UnidirectionalRNNOpTest/BlackBoxTest,29 489 UnidirectionalRNNOpTest.TimeMajorBlackBoxTest,29 490 491 # unpack_test 492 # Unpacking along the last axis is not supported 493 -UnpackOpTest/.+/ThreeOutputsAxisOne 494 -UnpackOpTest/.+/ThreeOutputsNegativeAxisOne 495 -UnpackOpTest/.+/ThreeDimensionsOutputs 496 # Unpacking 5D tensor is not supported 497 -UnpackOpTest/.+/FiveDimensionsOutputs 498 # Unpacking a vector to scalar is not supported 499 -UnpackOpTest/.+/VectorToScalar 500 # float, int8, uint8 only 501 UnpackOpTest/(0|2|3)/.+,30 502 )"; 503 504 } // namespace tflite 505