1# Lint as: python2, python3 2# Copyright 2017 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5import os 6import unittest 7 8import common 9 10from autotest_lib.server.cros.tradefed import tradefed_utils 11 12 13def _load_data(filename): 14 """Loads the test data of the given file name.""" 15 with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 16 'tradefed_utils_unittest_data', filename), 'r') as f: 17 return f.read() 18 19 20class TradefedTestTest(unittest.TestCase): 21 """Unittest for tradefed_utils.""" 22 23 def test_parse_tradefed_result(self): 24 """Test for parse_tradefed_result.""" 25 26 waivers = set([ 27 'android.app.cts.SystemFeaturesTest#testUsbAccessory', 28 'android.widget.cts.GridViewTest#testSetNumColumns', 29 ]) 30 31 # b/35605415 and b/36520623 32 # http://pantheon/storage/browser/chromeos-autotest-results/108103986-chromeos-test/ 33 # CTS: Tradefed may split a module to multiple chunks. 34 # Besides, the module name may not end with "TestCases". 35 waived, _ = tradefed_utils.parse_tradefed_result( 36 _load_data('CtsHostsideNetworkTests.txt'), 37 waivers=waivers) 38 self.assertEquals(0, len(waived)) 39 40 # b/35530394 41 # http://pantheon/storage/browser/chromeos-autotest-results/108291418-chromeos-test/ 42 # Crashed, but the automatic retry by tradefed executed the rest. 43 waived, _ = tradefed_utils.parse_tradefed_result( 44 _load_data('CtsMediaTestCases.txt'), 45 waivers=waivers) 46 self.assertEquals(0, len(waived)) 47 48 # b/35530394 49 # http://pantheon/storage/browser/chromeos-autotest-results/106540705-chromeos-test/ 50 # Crashed in the middle, and the device didn't came back. 51 waived, _ = tradefed_utils.parse_tradefed_result( 52 _load_data('CtsSecurityTestCases.txt'), 53 waivers=waivers) 54 self.assertEquals(0, len(waived)) 55 56 # b/36629187 57 # http://pantheon/storage/browser/chromeos-autotest-results/108855595-chromeos-test/ 58 # Crashed in the middle. Tradefed decided not to continue. 59 waived, _ = tradefed_utils.parse_tradefed_result( 60 _load_data('CtsViewTestCases.txt'), 61 waivers=waivers) 62 self.assertEquals(0, len(waived)) 63 64 # b/36375690 65 # http://pantheon/storage/browser/chromeos-autotest-results/109040174-chromeos-test/ 66 # Mixture of real failures and waivers. 67 waived, _ = tradefed_utils.parse_tradefed_result( 68 _load_data('CtsAppTestCases.txt'), 69 waivers=waivers) 70 self.assertEquals(1, len(waived)) 71 # ... and the retry of the above failing iteration. 72 waived, _ = tradefed_utils.parse_tradefed_result( 73 _load_data('CtsAppTestCases-retry.txt'), 74 waivers=waivers) 75 self.assertEquals(1, len(waived)) 76 77 # http://pantheon/storage/browser/chromeos-autotest-results/116875512-chromeos-test/ 78 # When a test case crashed during teardown, tradefed prints the "fail" 79 # message twice. Tolerate it and still return an (inconsistent) count. 80 waived, _ = tradefed_utils.parse_tradefed_result( 81 _load_data('CtsWidgetTestCases.txt'), 82 waivers=waivers) 83 self.assertEquals(1, len(waived)) 84 85 # http://pantheon/storage/browser/chromeos-autotest-results/117914707-chromeos-test/ 86 # When a test case unrecoverably crashed during teardown, tradefed 87 # prints the "fail" and failure summary message twice. Tolerate it. 88 waived, _ = tradefed_utils.parse_tradefed_result( 89 _load_data('CtsPrintTestCases.txt'), 90 waivers=waivers) 91 self.assertEquals(0, len(waived)) 92 93 gts_waivers = set([ 94 ('com.google.android.placement.gts.CoreGmsAppsTest#' + 95 'testCoreGmsAppsPreloaded'), 96 ('com.google.android.placement.gts.CoreGmsAppsTest#' + 97 'testGoogleDuoPreloaded'), 98 'com.google.android.placement.gts.UiPlacementTest#testPlayStore' 99 ]) 100 101 # crbug.com/748116 102 # http://pantheon/storage/browser/chromeos-autotest-results/130080763-chromeos-test/ 103 # 3 ABIS: x86, x86_64, and armeabi-v7a 104 waived, _ = tradefed_utils.parse_tradefed_result( 105 _load_data('GtsPlacementTestCases.txt'), 106 waivers=gts_waivers) 107 self.assertEquals(9, len(waived)) 108 109 # b/64095702 110 # http://pantheon/storage/browser/chromeos-autotest-results/130211812-chromeos-test/ 111 # The result of the last chunk not reported by tradefed. 112 # The actual dEQP log is too big, hence the test data here is trimmed. 113 waived, _ = tradefed_utils.parse_tradefed_result( 114 _load_data('CtsDeqpTestCases-trimmed.txt'), 115 waivers=waivers) 116 self.assertEquals(0, len(waived)) 117 118 # b/80160772 119 # http://pantheon/storage/browser/chromeos-autotest-results/201962931-kkanchi/ 120 # The newer tradefed requires different parsing to count waivers. 121 waived, _ = tradefed_utils.parse_tradefed_result( 122 _load_data('CtsAppTestCases_P_simplified.txt'), 123 waivers=waivers) 124 self.assertEquals(1, len(waived)) 125 126 # b/66899135, tradefed may reported inaccuratly with `list results`. 127 # Check if summary section shows that the result is inacurrate. 128 _, accurate = tradefed_utils.parse_tradefed_result( 129 _load_data('CtsAppTestCases_P_simplified.txt'), 130 waivers=waivers) 131 self.assertTrue(accurate) 132 133 _, accurate = tradefed_utils.parse_tradefed_result( 134 _load_data('CtsDeqpTestCases-trimmed-inaccurate.txt'), 135 waivers=waivers) 136 self.assertFalse(accurate) 137 138 def test_get_test_result_xml_path(self): 139 path = tradefed_utils.get_test_result_xml_path(os.path.join( 140 os.path.dirname(os.path.realpath(__file__)), 141 'tradefed_utils_unittest_data', 'results')) 142 self.assertEqual(path, os.path.join( 143 os.path.dirname(os.path.realpath(__file__)), 144 'tradefed_utils_unittest_data', 'results', '2019.11.07_10.14.55', 145 'test_result.xml')) 146 147 # assertNoRaises 148 tradefed_utils.get_test_result_xml_path(os.path.join( 149 os.path.dirname(os.path.realpath(__file__)), 150 'tradefed_utils_unittest_data', 'not_exist')) 151 152 def test_parse_tradefed_testresults_xml_no_failure(self): 153 waived = tradefed_utils.parse_tradefed_testresults_xml( 154 os.path.join(os.path.dirname(os.path.realpath(__file__)), 155 'tradefed_utils_unittest_data', 156 'test_result.xml')) 157 self.assertEquals(0, len(waived)) 158 159 def test_parse_tradefed_testresult_xml_waivers(self): 160 waived = tradefed_utils.parse_tradefed_testresults_xml( 161 os.path.join(os.path.dirname(os.path.realpath(__file__)), 162 'tradefed_utils_unittest_data', 163 'gtsplacement_test_result.xml')) 164 self.assertEquals(0, len(waived)) 165 166 waivers = set([ 167 'com.google.android.media.gts.WidevineDashPolicyTests#testL1RenewalDelay5S', 168 'com.google.android.media.gts.MediaDrmTest#testWidevineApi28', 169 'com.google.android.media.gts.WidevineGenericOpsTests#testL3', 170 'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay5S', 171 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L3WithUHD30', 172 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL3WithUHD30', 173 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbc1L1WithUHD30', 174 'com.google.android.media.gts.WidevineDashPolicyTests#testL3RenewalDelay13S', 175 'com.google.android.gts.backup.BackupHostTest#testGmsBackupTransportIsDefault', 176 'com.google.android.placement.gts.CoreGmsAppsTest#testGoogleDuoPreloaded', 177 'com.google.android.placement.gts.CoreGmsAppsTest#testCoreGmsAppsPreloaded', 178 'com.google.android.media.gts.WidevineH264PlaybackTests#testCbcsL1WithUHD30']) 179 waived = tradefed_utils.parse_tradefed_testresults_xml(os.path.join( 180 os.path.dirname(os.path.realpath(__file__)), 181 'tradefed_utils_unittest_data', 182 'gtsplacement_test_result.xml'), 183 waivers=waivers) 184 self.assertEquals(4, len(waived)) 185 186 def test_get_perf_metrics_from_test_result_xml(self): 187 perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml( 188 os.path.join(os.path.dirname(os.path.realpath(__file__)), 189 'tradefed_utils_unittest_data', 'test_result.xml'), 190 os.path.join('/', 'resultsdir')) 191 expected_result = [ 192 {'units': 'ms', 193 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 194 'description': 'android.media.cts.AudioRecordTest' 195 '#testAudioRecordLocalMono16Bit', 196 'value': '7.1688596491228065', 'higher_is_better': False}, 197 {'units': 'ms', 198 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 199 'description': 'android.media.cts.AudioRecordTest' 200 '#testAudioRecordLocalMono16BitShort', 201 'value': '2.5416666666666665', 'higher_is_better': False}, 202 {'units': 'ms', 203 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 204 'description': 'android.media.cts.AudioRecordTest' 205 '#testAudioRecordLocalNonblockingStereoFloat', 206 'value': '1.75', 'higher_is_better': False}, 207 {'units': 'ms', 208 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 209 'description': 'android.media.cts.AudioRecordTest' 210 '#testAudioRecordMonoFloat', 211 'value': '12.958881578947368', 'higher_is_better': False}, 212 {'units': 'ms', 213 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 214 'description': 'android.media.cts.AudioRecordTest' 215 '#testAudioRecordResamplerMono8Bit', 216 'value': '0.0', 'higher_is_better': False}, 217 {'units': 'ms', 218 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 219 'description': 'android.media.cts.AudioRecordTest' 220 '#testAudioRecordResamplerStereo8Bit', 221 'value': '3.5', 'higher_is_better': False}, 222 {'units': 'ms', 223 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 224 'description': 'android.media.cts.AudioRecordTest' 225 '#testAudioRecordStereo16Bit', 226 'value': '3.5', 'higher_is_better': False}, 227 {'units': 'ms', 228 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 229 'description': 'android.media.cts.AudioTrackTest' 230 '#testFastTimestamp', 231 'value': '0.1547618955373764', 'higher_is_better': False}, 232 {'units': 'ms', 233 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 234 'description': 'android.media.cts.AudioTrackTest' 235 '#testGetTimestamp', 236 'value': '0.1490119844675064', 'higher_is_better': False}, 237 {'units': 'ms', 238 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 239 'description': 'android.media.cts.AudioTrack_ListenerTest' 240 '#testAudioTrackCallback', 241 'value': '9.347127739984884', 'higher_is_better': False}, 242 {'units': 'ms', 243 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 244 'description': 'android.media.cts.AudioTrack_ListenerTest' 245 '#testAudioTrackCallbackWithHandler', 246 'value': '7.776177955844914', 'higher_is_better': False}, 247 {'units': 'ms', 248 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 249 'description': 'android.media.cts.AudioTrack_ListenerTest' 250 '#testStaticAudioTrackCallback', 251 'value': '7.776177955844914', 'higher_is_better': False}, 252 {'units': 'ms', 253 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 254 'description': 'android.media.cts.AudioTrack_ListenerTest' 255 '#testStaticAudioTrackCallbackWithHandler', 256 'value': '9.514361300075587', 'higher_is_better': False}, 257 {'units': 'count', 258 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 259 'description': 'android.media.cts.DecoderTest' 260 '#testH264ColorAspects', 261 'value': '1.0', 'higher_is_better': True}, 262 {'units': 'count', 263 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 264 'description': 'android.media.cts.DecoderTest' 265 '#testH265ColorAspects', 266 'value': '1.0', 'higher_is_better': True}, 267 {'units': 'fps', 268 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 269 'description': 'android.media.cts.VideoDecoderPerfTest' 270 '#testAvcGoog0Perf0320x0240', 271 'value': '580.1607045151507', 'higher_is_better': True}, 272 {'units': 'fps', 273 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 274 'description': 'android.media.cts.VideoDecoderPerfTest' 275 '#testAvcGoog0Perf0720x0480', 276 'value': '244.18184010611358', 'higher_is_better': True}, 277 {'units': 'fps', 278 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 279 'description': 'android.media.cts.VideoDecoderPerfTest' 280 '#testAvcGoog0Perf1280x0720', 281 'value': '70.96290491279275', 'higher_is_better': True}, 282 {'units': 'fps', 283 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 284 'description': 'android.media.cts.VideoDecoderPerfTest' 285 '#testAvcGoog0Perf1920x1080', 286 'value': '31.299613935451564', 'higher_is_better': True}, 287 {'units': 'fps', 288 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 289 'description': 'android.media.cts.VideoDecoderPerfTest' 290 '#testAvcOther0Perf0320x0240', 291 'value': '1079.6843075197307', 'higher_is_better': True}, 292 {'units': 'fps', 293 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 294 'description': 'android.media.cts.VideoDecoderPerfTest' 295 '#testAvcOther0Perf0720x0480', 296 'value': '873.7785366761784', 'higher_is_better': True}, 297 {'units': 'fps', 298 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 299 'description': 'android.media.cts.VideoDecoderPerfTest' 300 '#testAvcOther0Perf1280x0720', 301 'value': '664.6463289568261', 'higher_is_better': True}, 302 {'units': 'fps', 303 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 304 'description': 'android.media.cts.VideoDecoderPerfTest' 305 '#testAvcOther0Perf1920x1080', 306 'value': '382.10811352923474', 'higher_is_better': True}, 307 {'units': 'fps', 308 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 309 'description': 'android.media.cts.VideoDecoderPerfTest' 310 '#testH263Goog0Perf0176x0144', 311 'value': '1511.3027429644353', 'higher_is_better': True}, 312 {'units': 'fps', 313 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 314 'description': 'android.media.cts.VideoDecoderPerfTest' 315 '#testHevcGoog0Perf0352x0288', 316 'value': '768.8737453173384', 'higher_is_better': True}, 317 {'units': 'fps', 318 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 319 'description': 'android.media.cts.VideoDecoderPerfTest' 320 '#testHevcGoog0Perf0640x0360', 321 'value': '353.7226028743237', 'higher_is_better': True}, 322 {'units': 'fps', 323 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 324 'description': 'android.media.cts.VideoDecoderPerfTest' 325 '#testHevcGoog0Perf0720x0480', 326 'value': '319.3122874170939', 'higher_is_better': True}, 327 {'units': 'fps', 328 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 329 'description': 'android.media.cts.VideoDecoderPerfTest' 330 '#testHevcGoog0Perf1280x0720', 331 'value': '120.89218432028369', 'higher_is_better': True}, 332 {'units': 'fps', 333 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 334 'description': 'android.media.cts.VideoDecoderPerfTest' 335 '#testMpeg4Goog0Perf0176x0144', 336 'value': '1851.890822618321', 'higher_is_better': True}, 337 {'units': 'fps', 338 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 339 'description': 'android.media.cts.VideoDecoderPerfTest' 340 '#testVp8Goog0Perf0320x0180', 341 'value': '1087.946513466716', 'higher_is_better': True}, 342 {'units': 'fps', 343 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 344 'description': 'android.media.cts.VideoDecoderPerfTest' 345 '#testVp8Goog0Perf0640x0360', 346 'value': '410.18461316281423', 'higher_is_better': True}, 347 {'units': 'fps', 348 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 349 'description': 'android.media.cts.VideoDecoderPerfTest' 350 '#testVp8Goog0Perf1920x1080', 351 'value': '36.26433070651982', 'higher_is_better': True}, 352 {'units': 'fps', 353 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 354 'description': 'android.media.cts.VideoDecoderPerfTest' 355 '#testVp8Other0Perf0320x0180', 356 'value': '1066.7819511702078', 'higher_is_better': True}, 357 {'units': 'fps', 358 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 359 'description': 'android.media.cts.VideoDecoderPerfTest' 360 '#testVp8Other0Perf0640x0360', 361 'value': '930.261434505189', 'higher_is_better': True}, 362 {'units': 'fps', 363 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 364 'description': 'android.media.cts.VideoDecoderPerfTest' 365 '#testVp8Other0Perf1280x0720', 366 'value': '720.4170603577236', 'higher_is_better': True}, 367 {'units': 'fps', 368 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 369 'description': 'android.media.cts.VideoDecoderPerfTest' 370 '#testVp8Other0Perf1920x1080', 371 'value': '377.55742437554915', 'higher_is_better': True}, 372 {'units': 'fps', 373 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 374 'description': 'android.media.cts.VideoDecoderPerfTest' 375 '#testVp9Goog0Perf0320x0180', 376 'value': '988.6158776121617', 'higher_is_better': True}, 377 {'units': 'fps', 378 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 379 'description': 'android.media.cts.VideoDecoderPerfTest' 380 '#testVp9Goog0Perf0640x0360', 381 'value': '409.8162085338674', 'higher_is_better': True}, 382 {'units': 'fps', 383 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 384 'description': 'android.media.cts.VideoDecoderPerfTest' 385 '#testVp9Goog0Perf1280x0720', 386 'value': '147.75847359424512', 'higher_is_better': True}, 387 {'units': 'fps', 388 'resultsdir': '/resultsdir/tests/CTS.CtsMediaTestCases', 389 'description': 'android.media.cts.VideoDecoderPerfTest' 390 '#testVp9Goog0Perf1920x1080', 391 'value': '83.95677136649255', 'higher_is_better': True} 392 ] 393 self.assertListEqual(list(perf_result), expected_result) 394 395 perf_result = tradefed_utils.get_perf_metrics_from_test_result_xml( 396 os.path.join(os.path.dirname(os.path.realpath(__file__)), 397 'tradefed_utils_unittest_data', 398 'malformed_test_result.xml'), 399 os.path.join('/', 'resultsdir')) 400 self.assertListEqual(list(perf_result), []) 401 402 # assertNoRaises 403 tradefed_utils.get_perf_metrics_from_test_result_xml( 404 os.path.join(os.path.dirname(os.path.realpath(__file__)), 405 'tradefed_utils_unittest_data', 406 'not_exist'), 407 os.path.join('/', 'resultsdir')) 408 409 410 411if __name__ == '__main__': 412 unittest.main() 413