1# Copyright 2015 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import copy 7import logging 8import os 9import re 10 11from devil.android import apk_helper 12from pylib import constants 13from pylib.base import base_test_result 14from pylib.base import test_exception 15from pylib.base import test_instance 16from pylib.constants import host_paths 17from pylib.instrumentation import instrumentation_parser 18from pylib.instrumentation import test_result 19from pylib.symbols import deobfuscator 20from pylib.symbols import stack_symbolizer 21from pylib.utils import dexdump 22from pylib.utils import gold_utils 23from pylib.utils import test_filter 24 25 26with host_paths.SysPath(host_paths.BUILD_COMMON_PATH): 27 import unittest_util # pylint: disable=import-error 28 29# Ref: http://developer.android.com/reference/android/app/Activity.html 30_ACTIVITY_RESULT_CANCELED = 0 31_ACTIVITY_RESULT_OK = -1 32 33_COMMAND_LINE_PARAMETER = 'cmdlinearg-parameter' 34_DEFAULT_ANNOTATIONS = [ 35 'SmallTest', 'MediumTest', 'LargeTest', 'EnormousTest', 'IntegrationTest'] 36# This annotation is for disabled tests that should not be run in Test Reviver. 37_DO_NOT_REVIVE_ANNOTATIONS = ['DoNotRevive', 'Manual'] 38_EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [ 39 'DisabledTest', 'FlakyTest', 'Manual'] 40_VALID_ANNOTATIONS = set(_DEFAULT_ANNOTATIONS + _DO_NOT_REVIVE_ANNOTATIONS + 41 _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS) 42 43_BASE_INSTRUMENTATION_CLASS_NAME = ( 44 'org.chromium.base.test.BaseChromiumAndroidJUnitRunner') 45 46_SKIP_PARAMETERIZATION = 'SkipCommandLineParameterization' 47_PARAMETERIZED_COMMAND_LINE_FLAGS = 'ParameterizedCommandLineFlags' 48_PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES = ( 49 'ParameterizedCommandLineFlags$Switches') 50_NATIVE_CRASH_RE = re.compile('(process|native) crash', re.IGNORECASE) 51 52# The ID of the bundle value Instrumentation uses to report which test index the 53# results are for in a collection of tests. Note that this index is 1-based. 54_BUNDLE_CURRENT_ID = 'current' 55# The ID of the bundle value Instrumentation uses to report the test class. 56_BUNDLE_CLASS_ID = 'class' 57# The ID of the bundle value Instrumentation uses to report the test name. 58_BUNDLE_TEST_ID = 'test' 59# The ID of the bundle value Instrumentation uses to report if a test was 60# skipped. 61_BUNDLE_SKIPPED_ID = 'test_skipped' 62# The ID of the bundle value Instrumentation uses to report the crash stack, if 63# the test crashed. 64_BUNDLE_STACK_ID = 'stack' 65 66# The ID of the bundle value Chrome uses to report the test duration. 67_BUNDLE_DURATION_ID = 'duration_ms' 68 69class MissingSizeAnnotationError(test_exception.TestException): 70 def __init__(self, class_name): 71 super().__init__( 72 class_name + 73 ': Test method is missing required size annotation. Add one of: ' + 74 ', '.join('@' + a for a in _VALID_ANNOTATIONS)) 75 76 77class CommandLineParameterizationException(test_exception.TestException): 78 pass 79 80 81def GenerateTestResults(result_code, result_bundle, statuses, duration_ms, 82 device_abi, symbolizer): 83 """Generate test results from |statuses|. 84 85 Args: 86 result_code: The overall status code as an integer. 87 result_bundle: The summary bundle dump as a dict. 88 statuses: A list of 2-tuples containing: 89 - the status code as an integer 90 - the bundle dump as a dict mapping string keys to string values 91 Note that this is the same as the third item in the 3-tuple returned by 92 |_ParseAmInstrumentRawOutput|. 93 duration_ms: The duration of the test in milliseconds. 94 device_abi: The device_abi, which is needed for symbolization. 95 symbolizer: The symbolizer used to symbolize stack. 96 97 Returns: 98 A list containing an instance of InstrumentationTestResult for each test 99 parsed. 100 """ 101 102 results = [] 103 104 current_result = None 105 cumulative_duration = 0 106 107 for status_code, bundle in statuses: 108 # If the last test was a failure already, don't override that failure with 109 # post-test failures that could be caused by the original failure. 110 if (status_code == instrumentation_parser.STATUS_CODE_BATCH_FAILURE 111 and current_result.GetType() != base_test_result.ResultType.FAIL): 112 current_result.SetType(base_test_result.ResultType.FAIL) 113 _MaybeSetLog(bundle, current_result, symbolizer, device_abi) 114 continue 115 116 if status_code == instrumentation_parser.STATUS_CODE_TEST_DURATION: 117 # For the first result, duration will be set below to the difference 118 # between the reported and actual durations to account for overhead like 119 # starting instrumentation. 120 if results: 121 current_duration = int(bundle.get(_BUNDLE_DURATION_ID, duration_ms)) 122 current_result.SetDuration(current_duration) 123 cumulative_duration += current_duration 124 continue 125 126 test_class = bundle.get(_BUNDLE_CLASS_ID, '') 127 test_method = bundle.get(_BUNDLE_TEST_ID, '') 128 if test_class and test_method: 129 test_name = '%s#%s' % (test_class, test_method) 130 else: 131 continue 132 133 if status_code == instrumentation_parser.STATUS_CODE_START: 134 if current_result: 135 results.append(current_result) 136 current_result = test_result.InstrumentationTestResult( 137 test_name, base_test_result.ResultType.UNKNOWN, duration_ms) 138 else: 139 if status_code == instrumentation_parser.STATUS_CODE_OK: 140 if bundle.get(_BUNDLE_SKIPPED_ID, '').lower() in ('true', '1', 'yes'): 141 current_result.SetType(base_test_result.ResultType.SKIP) 142 elif current_result.GetType() == base_test_result.ResultType.UNKNOWN: 143 current_result.SetType(base_test_result.ResultType.PASS) 144 elif status_code == instrumentation_parser.STATUS_CODE_SKIP: 145 current_result.SetType(base_test_result.ResultType.SKIP) 146 elif status_code == instrumentation_parser.STATUS_CODE_ASSUMPTION_FAILURE: 147 current_result.SetType(base_test_result.ResultType.SKIP) 148 else: 149 if status_code not in (instrumentation_parser.STATUS_CODE_ERROR, 150 instrumentation_parser.STATUS_CODE_FAILURE): 151 logging.error('Unrecognized status code %d. Handling as an error.', 152 status_code) 153 current_result.SetType(base_test_result.ResultType.FAIL) 154 _MaybeSetLog(bundle, current_result, symbolizer, device_abi) 155 156 if current_result: 157 if current_result.GetType() == base_test_result.ResultType.UNKNOWN: 158 crashed = (result_code == _ACTIVITY_RESULT_CANCELED and any( 159 _NATIVE_CRASH_RE.search(l) for l in result_bundle.values())) 160 if crashed: 161 current_result.SetType(base_test_result.ResultType.CRASH) 162 163 results.append(current_result) 164 165 if results: 166 logging.info('Adding cumulative overhead to test %s: %dms', 167 results[0].GetName(), duration_ms - cumulative_duration) 168 results[0].SetDuration(duration_ms - cumulative_duration) 169 170 return results 171 172 173def _MaybeSetLog(bundle, current_result, symbolizer, device_abi): 174 if _BUNDLE_STACK_ID in bundle: 175 stack = bundle[_BUNDLE_STACK_ID] 176 if symbolizer and device_abi: 177 current_result.SetLog('%s\n%s' % (stack, '\n'.join( 178 symbolizer.ExtractAndResolveNativeStackTraces(stack, device_abi)))) 179 else: 180 current_result.SetLog(stack) 181 182 current_result.SetFailureReason(_ParseExceptionMessage(stack)) 183 184 185def _ParseExceptionMessage(stack): 186 """Extracts the exception message from the given stack trace. 187 """ 188 # This interprets stack traces reported via InstrumentationResultPrinter: 189 # https://source.chromium.org/chromium/chromium/src/+/main:third_party/android_support_test_runner/runner/src/main/java/android/support/test/internal/runner/listener/InstrumentationResultPrinter.java;l=181?q=InstrumentationResultPrinter&type=cs 190 # This is a standard Java stack trace, of the form: 191 # <Result of Exception.toString()> 192 # at SomeClass.SomeMethod(...) 193 # at ... 194 lines = stack.split('\n') 195 for i, line in enumerate(lines): 196 if line.startswith('\tat'): 197 return '\n'.join(lines[0:i]) 198 # No call stack found, so assume everything is the exception message. 199 return stack 200 201 202def FilterTests(tests, 203 filter_strs=None, 204 annotations=None, 205 excluded_annotations=None): 206 """Filter a list of tests 207 208 Args: 209 tests: a list of tests. e.g. [ 210 {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, 211 {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] 212 filter_strs: list of googletest-style filter string. 213 annotations: a dict of wanted annotations for test methods. 214 excluded_annotations: a dict of annotations to exclude. 215 216 Return: 217 A list of filtered tests 218 """ 219 220 def test_names_from_pattern(combined_pattern, test_names): 221 patterns = combined_pattern.split(':') 222 223 hashable_patterns = set() 224 filename_patterns = [] 225 for pattern in patterns: 226 if ('*' in pattern or '?' in pattern or '[' in pattern): 227 filename_patterns.append(pattern) 228 else: 229 hashable_patterns.add(pattern) 230 231 filter_test_names = set( 232 unittest_util.FilterTestNames(test_names, ':'.join( 233 filename_patterns))) if len(filename_patterns) > 0 else set() 234 235 for test_name in test_names: 236 if test_name in hashable_patterns: 237 filter_test_names.add(test_name) 238 239 return filter_test_names 240 241 def get_test_names(test): 242 test_names = set() 243 # Allow fully-qualified name as well as an omitted package. 244 unqualified_class_test = { 245 'class': test['class'].split('.')[-1], 246 'method': test['method'] 247 } 248 249 test_name = GetTestName(test, sep='.') 250 test_names.add(test_name) 251 252 unqualified_class_test_name = GetTestName(unqualified_class_test, sep='.') 253 test_names.add(unqualified_class_test_name) 254 255 unique_test_name = GetUniqueTestName(test, sep='.') 256 test_names.add(unique_test_name) 257 258 junit4_test_name = GetTestNameWithoutParameterSuffix(test, sep='.') 259 test_names.add(junit4_test_name) 260 261 unqualified_junit4_test_name = GetTestNameWithoutParameterSuffix( 262 unqualified_class_test, sep='.') 263 test_names.add(unqualified_junit4_test_name) 264 return test_names 265 266 def get_tests_from_names(tests, test_names, tests_to_names): 267 ''' Returns the tests for which the given names apply 268 269 Args: 270 tests: a list of tests. e.g. [ 271 {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, 272 {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] 273 test_names: a collection of names determining tests to return. 274 275 Return: 276 A list of tests that match the given test names 277 ''' 278 filtered_tests = [] 279 for t in tests: 280 current_test_names = tests_to_names[id(t)] 281 282 for current_test_name in current_test_names: 283 if current_test_name in test_names: 284 filtered_tests.append(t) 285 break 286 287 return filtered_tests 288 289 def remove_tests_from_names(tests, remove_test_names, tests_to_names): 290 ''' Returns the tests from the given list with given names removed 291 292 Args: 293 tests: a list of tests. e.g. [ 294 {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, 295 {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] 296 remove_test_names: a collection of names determining tests to remove. 297 tests_to_names: a dcitionary of test ids to a collection of applicable 298 names for that test 299 300 Return: 301 A list of tests that don't match the given test names 302 ''' 303 filtered_tests = [] 304 305 for t in tests: 306 for name in tests_to_names[id(t)]: 307 if name in remove_test_names: 308 break 309 else: 310 filtered_tests.append(t) 311 return filtered_tests 312 313 def gtests_filter(tests, combined_filters): 314 ''' Returns the tests after the combined_filters have been applied 315 316 Args: 317 tests: a list of tests. e.g. [ 318 {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'}, 319 {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}] 320 combined_filters: the filter string representing tests to exclude 321 322 Return: 323 A list of tests that should still be included after the combined_filters 324 are applied to their names 325 ''' 326 327 if not combined_filters: 328 return tests 329 330 # Collect all test names 331 all_test_names = set() 332 tests_to_names = {} 333 for t in tests: 334 tests_to_names[id(t)] = get_test_names(t) 335 for name in tests_to_names[id(t)]: 336 all_test_names.add(name) 337 338 for combined_filter in combined_filters: 339 pattern_groups = combined_filter.split('-') 340 negative_pattern = pattern_groups[1] if len(pattern_groups) > 1 else None 341 positive_pattern = pattern_groups[0] 342 if positive_pattern: 343 # Only use the test names that match the positive pattern 344 positive_test_names = test_names_from_pattern(positive_pattern, 345 all_test_names) 346 tests = get_tests_from_names(tests, positive_test_names, tests_to_names) 347 348 if negative_pattern: 349 # Remove any test the negative filter matches 350 remove_names = test_names_from_pattern(negative_pattern, all_test_names) 351 tests = remove_tests_from_names(tests, remove_names, tests_to_names) 352 353 return tests 354 355 def annotation_filter(all_annotations): 356 if not annotations: 357 return True 358 return any_annotation_matches(annotations, all_annotations) 359 360 def excluded_annotation_filter(all_annotations): 361 if not excluded_annotations: 362 return True 363 return not any_annotation_matches(excluded_annotations, 364 all_annotations) 365 366 def any_annotation_matches(filter_annotations, all_annotations): 367 return any( 368 ak in all_annotations 369 and annotation_value_matches(av, all_annotations[ak]) 370 for ak, av in filter_annotations) 371 372 def annotation_value_matches(filter_av, av): 373 if filter_av is None: 374 return True 375 if isinstance(av, dict): 376 tav_from_dict = av['value'] 377 # If tav_from_dict is an int, the 'in' operator breaks, so convert 378 # filter_av and manually compare. See https://crbug.com/1019707 379 if isinstance(tav_from_dict, int): 380 return int(filter_av) == tav_from_dict 381 return filter_av in tav_from_dict 382 if isinstance(av, list): 383 return filter_av in av 384 return filter_av == av 385 386 return_tests = [] 387 for t in gtests_filter(tests, filter_strs): 388 # Enforce that all tests declare their size. 389 if not any(a in _VALID_ANNOTATIONS for a in t['annotations']): 390 raise MissingSizeAnnotationError(GetTestName(t)) 391 392 if (not annotation_filter(t['annotations']) 393 or not excluded_annotation_filter(t['annotations'])): 394 continue 395 return_tests.append(t) 396 397 return return_tests 398 399 400def GetTestsFromDexdump(test_apk): 401 dex_dumps = dexdump.Dump(test_apk) 402 tests = [] 403 404 def get_test_methods(methods, annotations): 405 test_methods = [] 406 407 for method in methods: 408 if method.startswith('test'): 409 method_annotations = annotations.get(method, {}) 410 411 # Dexdump used to not return any annotation info 412 # So MediumTest annotation was added to all methods 413 # Preserving this behaviour by adding MediumTest if none of the 414 # size annotations are included in these annotations 415 if not any(valid in method_annotations for valid in _VALID_ANNOTATIONS): 416 method_annotations.update({'MediumTest': None}) 417 418 test_methods.append({ 419 'method': method, 420 'annotations': method_annotations 421 }) 422 423 return test_methods 424 425 for dump in dex_dumps: 426 for package_name, package_info in dump.items(): 427 for class_name, class_info in package_info['classes'].items(): 428 if class_name.endswith('Test') and not class_info['is_abstract']: 429 classAnnotations, methodsAnnotations = class_info['annotations'] 430 tests.append({ 431 'class': 432 '%s.%s' % (package_name, class_name), 433 'annotations': 434 classAnnotations, 435 'methods': 436 get_test_methods(class_info['methods'], methodsAnnotations), 437 }) 438 return tests 439 440 441def GetTestName(test, sep='#'): 442 """Gets the name of the given test. 443 444 Note that this may return the same name for more than one test, e.g. if a 445 test is being run multiple times with different parameters. 446 447 Args: 448 test: the instrumentation test dict. 449 sep: the character(s) that should join the class name and the method name. 450 Returns: 451 The test name as a string. 452 """ 453 test_name = '%s%s%s' % (test['class'], sep, test['method']) 454 assert not any(char in test_name for char in ' *-:'), ( 455 'The test name must not contain any of the characters in " *-:". See ' 456 'https://crbug.com/912199') 457 return test_name 458 459 460def GetTestNameWithoutParameterSuffix(test, sep='#', parameterization_sep='__'): 461 """Gets the name of the given JUnit4 test without parameter suffix. 462 463 For most WebView JUnit4 javatests, each test is parameterizatized with 464 "__sandboxed_mode" to run in both non-sandboxed mode and sandboxed mode. 465 466 This function returns the name of the test without parameterization 467 so test filters can match both parameterized and non-parameterized tests. 468 469 Args: 470 test: the instrumentation test dict. 471 sep: the character(s) that should join the class name and the method name. 472 parameterization_sep: the character(s) that separate method name and method 473 parameterization suffix. 474 Returns: 475 The test name without parameter suffix as a string. 476 """ 477 name = GetTestName(test, sep=sep) 478 return name.split(parameterization_sep)[0] 479 480 481def GetUniqueTestName(test, sep='#'): 482 """Gets the unique name of the given test. 483 484 This will include text to disambiguate between tests for which GetTestName 485 would return the same name. 486 487 Args: 488 test: the instrumentation test dict. 489 sep: the character(s) that should join the class name and the method name. 490 Returns: 491 The unique test name as a string. 492 """ 493 display_name = GetTestName(test, sep=sep) 494 if test.get('flags', [None])[0]: 495 sanitized_flags = [x.replace('-', '_') for x in test['flags']] 496 display_name = '%s_with_%s' % (display_name, '_'.join(sanitized_flags)) 497 498 assert not any(char in display_name for char in ' *-:'), ( 499 'The test name must not contain any of the characters in " *-:". See ' 500 'https://crbug.com/912199') 501 502 return display_name 503 504 505class InstrumentationTestInstance(test_instance.TestInstance): 506 507 def __init__(self, args, data_deps_delegate, error_func): 508 super().__init__() 509 510 self._additional_apks = [] 511 self._additional_apexs = [] 512 self._forced_queryable_additional_apks = [] 513 self._instant_additional_apks = [] 514 self._apk_under_test = None 515 self._apk_under_test_incremental_install_json = None 516 self._modules = None 517 self._fake_modules = None 518 self._additional_locales = None 519 self._package_info = None 520 self._suite = None 521 self._test_apk = None 522 self._test_apk_as_instant = False 523 self._test_apk_incremental_install_json = None 524 self._test_package = None 525 self._junit4_runner_class = None 526 self._uses_base_instrumentation = None 527 self._has_chromium_test_listener = None 528 self._test_support_apk = None 529 self._initializeApkAttributes(args, error_func) 530 531 self._data_deps = None 532 self._data_deps_delegate = None 533 self._runtime_deps_path = None 534 self._variations_test_seed_path = args.variations_test_seed_path 535 self._webview_variations_test_seed_path = ( 536 args.webview_variations_test_seed_path) 537 self._store_data_dependencies_in_temp = False 538 self._initializeDataDependencyAttributes(args, data_deps_delegate) 539 self._annotations = None 540 self._excluded_annotations = None 541 self._has_external_annotation_filters = None 542 self._test_filters = None 543 self._initializeTestFilterAttributes(args) 544 545 self._run_setup_commands = [] 546 self._run_teardown_commands = [] 547 self._initializeSetupTeardownCommandAttributes(args) 548 549 self._flags = None 550 self._use_apk_under_test_flags_file = False 551 self._webview_flags = args.webview_command_line_arg 552 self._initializeFlagAttributes(args) 553 554 self._screenshot_dir = None 555 self._timeout_scale = None 556 self._wait_for_java_debugger = None 557 self._initializeTestControlAttributes(args) 558 559 self._coverage_directory = None 560 self._initializeTestCoverageAttributes(args) 561 562 self._store_tombstones = False 563 self._symbolizer = None 564 self._enable_breakpad_dump = False 565 self._proguard_mapping_path = None 566 self._deobfuscator = None 567 self._initializeLogAttributes(args) 568 569 self._replace_system_package = None 570 self._initializeReplaceSystemPackageAttributes(args) 571 572 self._system_packages_to_remove = None 573 self._initializeSystemPackagesToRemoveAttributes(args) 574 575 self._use_voice_interaction_service = None 576 self._initializeUseVoiceInteractionService(args) 577 578 self._use_webview_provider = None 579 self._initializeUseWebviewProviderAttributes(args) 580 581 self._skia_gold_properties = None 582 self._initializeSkiaGoldAttributes(args) 583 584 self._test_launcher_batch_limit = None 585 self._initializeTestLauncherAttributes(args) 586 587 self._approve_app_links_domain = None 588 self._approve_app_links_package = None 589 self._initializeApproveAppLinksAttributes(args) 590 591 self._wpr_enable_record = args.wpr_enable_record 592 593 self._external_shard_index = args.test_launcher_shard_index 594 self._total_external_shards = args.test_launcher_total_shards 595 596 self._is_unit_test = False 597 self._initializeUnitTestFlag(args) 598 599 self._run_disabled = args.run_disabled 600 601 def _initializeApkAttributes(self, args, error_func): 602 if args.apk_under_test: 603 apk_under_test_path = args.apk_under_test 604 if (not args.apk_under_test.endswith('.apk') 605 and not args.apk_under_test.endswith('.apks')): 606 apk_under_test_path = os.path.join( 607 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, 608 '%s.apk' % args.apk_under_test) 609 610 # TODO(jbudorick): Move the realpath up to the argument parser once 611 # APK-by-name is no longer supported. 612 apk_under_test_path = os.path.realpath(apk_under_test_path) 613 614 if not os.path.exists(apk_under_test_path): 615 error_func('Unable to find APK under test: %s' % apk_under_test_path) 616 617 self._apk_under_test = apk_helper.ToHelper(apk_under_test_path) 618 619 test_apk_path = args.test_apk 620 if (not args.test_apk.endswith('.apk') 621 and not args.test_apk.endswith('.apks')): 622 test_apk_path = os.path.join( 623 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, 624 '%s.apk' % args.test_apk) 625 626 # TODO(jbudorick): Move the realpath up to the argument parser once 627 # APK-by-name is no longer supported. 628 test_apk_path = os.path.realpath(test_apk_path) 629 630 if not os.path.exists(test_apk_path): 631 error_func('Unable to find test APK: %s' % test_apk_path) 632 633 self._test_apk = apk_helper.ToHelper(test_apk_path) 634 self._suite = os.path.splitext(os.path.basename(args.test_apk))[0] 635 636 self._test_apk_as_instant = args.test_apk_as_instant 637 638 self._apk_under_test_incremental_install_json = ( 639 args.apk_under_test_incremental_install_json) 640 self._test_apk_incremental_install_json = ( 641 args.test_apk_incremental_install_json) 642 643 if self._test_apk_incremental_install_json: 644 assert self._suite.endswith('_incremental') 645 self._suite = self._suite[:-len('_incremental')] 646 647 self._modules = args.modules 648 self._fake_modules = args.fake_modules 649 self._additional_locales = args.additional_locales 650 651 self._test_support_apk = apk_helper.ToHelper(os.path.join( 652 constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, 653 '%sSupport.apk' % self._suite)) 654 655 self._test_package = self._test_apk.GetPackageName() 656 all_instrumentations = self._test_apk.GetAllInstrumentations() 657 658 if len(all_instrumentations) > 1: 659 logging.warning('This test apk has more than one instrumentation') 660 661 self._junit4_runner_class = (all_instrumentations[0]['android:name'] 662 if all_instrumentations else None) 663 664 test_apk_metadata = dict(self._test_apk.GetAllMetadata()) 665 self._has_chromium_test_listener = bool( 666 test_apk_metadata.get('org.chromium.hasTestRunListener')) 667 if self._junit4_runner_class: 668 if self._test_apk_incremental_install_json: 669 for name, value in test_apk_metadata.items(): 670 if (name.startswith('incremental-install-instrumentation-') 671 and value == _BASE_INSTRUMENTATION_CLASS_NAME): 672 self._uses_base_instrumentation = True 673 break 674 else: 675 self._uses_base_instrumentation = ( 676 self._junit4_runner_class == _BASE_INSTRUMENTATION_CLASS_NAME) 677 678 self._package_info = None 679 if self._apk_under_test: 680 package_under_test = self._apk_under_test.GetPackageName() 681 for package_info in constants.PACKAGE_INFO.values(): 682 if package_under_test == package_info.package: 683 self._package_info = package_info 684 break 685 if not self._package_info: 686 logging.warning( 687 'Unable to find package info for %s. ' 688 '(This may just mean that the test package is ' 689 'currently being installed.)', self._test_package) 690 691 for x in set(args.additional_apks + args.forced_queryable_additional_apks + 692 args.instant_additional_apks): 693 if not os.path.exists(x): 694 error_func('Unable to find additional APK: %s' % x) 695 696 apk = apk_helper.ToHelper(x) 697 self._additional_apks.append(apk) 698 699 if x in args.forced_queryable_additional_apks: 700 self._forced_queryable_additional_apks.append(apk) 701 702 if x in args.instant_additional_apks: 703 self._instant_additional_apks.append(apk) 704 705 self._additional_apexs = args.additional_apexs 706 707 def _initializeDataDependencyAttributes(self, args, data_deps_delegate): 708 self._data_deps = [] 709 self._data_deps_delegate = data_deps_delegate 710 self._store_data_dependencies_in_temp = args.store_data_dependencies_in_temp 711 self._runtime_deps_path = args.runtime_deps_path 712 713 if not self._runtime_deps_path: 714 logging.warning('No data dependencies will be pushed.') 715 716 def _initializeTestFilterAttributes(self, args): 717 self._test_filters = test_filter.InitializeFiltersFromArgs(args) 718 self._has_external_annotation_filters = bool(args.annotation_str 719 or args.exclude_annotation_str) 720 721 def annotation_element(a): 722 a = a.split('=', 1) 723 return (a[0], a[1] if len(a) == 2 else None) 724 725 if args.annotation_str: 726 self._annotations = [ 727 annotation_element(a) for a in args.annotation_str.split(',')] 728 elif not self._test_filters: 729 self._annotations = [ 730 annotation_element(a) for a in _DEFAULT_ANNOTATIONS] 731 else: 732 self._annotations = [] 733 734 if args.exclude_annotation_str: 735 self._excluded_annotations = [ 736 annotation_element(a) for a in args.exclude_annotation_str.split(',')] 737 else: 738 self._excluded_annotations = [] 739 740 requested_annotations = set(a[0] for a in self._annotations) 741 if args.run_disabled: 742 self._excluded_annotations.extend( 743 annotation_element(a) for a in _DO_NOT_REVIVE_ANNOTATIONS 744 if a not in requested_annotations) 745 else: 746 self._excluded_annotations.extend( 747 annotation_element(a) for a in _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS 748 if a not in requested_annotations) 749 750 def _initializeSetupTeardownCommandAttributes(self, args): 751 self._run_setup_commands = args.run_setup_commands 752 self._run_teardown_commands = args.run_teardown_commands 753 754 def _initializeFlagAttributes(self, args): 755 self._use_apk_under_test_flags_file = args.use_apk_under_test_flags_file 756 self._flags = ['--enable-test-intents'] 757 if args.command_line_flags: 758 self._flags.extend(args.command_line_flags) 759 if args.device_flags_file: 760 with open(args.device_flags_file) as device_flags_file: 761 stripped_lines = (l.strip() for l in device_flags_file) 762 self._flags.extend(flag for flag in stripped_lines if flag) 763 if args.strict_mode and args.strict_mode != 'off' and ( 764 # TODO(yliuyliu): Turn on strict mode for coverage once 765 # crbug/1006397 is fixed. 766 not args.coverage_dir): 767 self._flags.append('--strict-mode=' + args.strict_mode) 768 769 def _initializeTestControlAttributes(self, args): 770 self._screenshot_dir = args.screenshot_dir 771 self._timeout_scale = args.timeout_scale or 1 772 self._wait_for_java_debugger = args.wait_for_java_debugger 773 774 def _initializeTestCoverageAttributes(self, args): 775 self._coverage_directory = args.coverage_dir 776 777 def _initializeLogAttributes(self, args): 778 self._enable_breakpad_dump = args.enable_breakpad_dump 779 self._proguard_mapping_path = args.proguard_mapping_path 780 self._store_tombstones = args.store_tombstones 781 self._symbolizer = stack_symbolizer.Symbolizer( 782 self.apk_under_test.path if self.apk_under_test else None) 783 784 def _initializeReplaceSystemPackageAttributes(self, args): 785 if (not hasattr(args, 'replace_system_package') 786 or not args.replace_system_package): 787 return 788 self._replace_system_package = args.replace_system_package 789 790 def _initializeSystemPackagesToRemoveAttributes(self, args): 791 if (not hasattr(args, 'system_packages_to_remove') 792 or not args.system_packages_to_remove): 793 return 794 self._system_packages_to_remove = args.system_packages_to_remove 795 796 def _initializeUseVoiceInteractionService(self, args): 797 if (not hasattr(args, 'use_voice_interaction_service') 798 or not args.use_voice_interaction_service): 799 return 800 self._use_voice_interaction_service = args.use_voice_interaction_service 801 802 def _initializeUseWebviewProviderAttributes(self, args): 803 if (not hasattr(args, 'use_webview_provider') 804 or not args.use_webview_provider): 805 return 806 self._use_webview_provider = args.use_webview_provider 807 808 def _initializeSkiaGoldAttributes(self, args): 809 self._skia_gold_properties = gold_utils.AndroidSkiaGoldProperties(args) 810 811 def _initializeTestLauncherAttributes(self, args): 812 if hasattr(args, 'test_launcher_batch_limit'): 813 self._test_launcher_batch_limit = args.test_launcher_batch_limit 814 815 def _initializeApproveAppLinksAttributes(self, args): 816 if (not hasattr(args, 'approve_app_links') or not args.approve_app_links): 817 return 818 819 # The argument will be formatted as com.android.thing:www.example.com . 820 app_links = args.approve_app_links.split(':') 821 822 if (len(app_links) != 2 or not app_links[0] or not app_links[1]): 823 logging.warning('--approve_app_links option provided, but malformed.') 824 return 825 826 self._approve_app_links_package = app_links[0] 827 self._approve_app_links_domain = app_links[1] 828 829 def _initializeUnitTestFlag(self, args): 830 self._is_unit_test = args.is_unit_test 831 832 @property 833 def additional_apks(self): 834 return self._additional_apks 835 836 @property 837 def additional_apexs(self): 838 return self._additional_apexs 839 840 @property 841 def apk_under_test(self): 842 return self._apk_under_test 843 844 @property 845 def apk_under_test_incremental_install_json(self): 846 return self._apk_under_test_incremental_install_json 847 848 @property 849 def approve_app_links_package(self): 850 return self._approve_app_links_package 851 852 @property 853 def approve_app_links_domain(self): 854 return self._approve_app_links_domain 855 856 @property 857 def modules(self): 858 return self._modules 859 860 @property 861 def fake_modules(self): 862 return self._fake_modules 863 864 @property 865 def additional_locales(self): 866 return self._additional_locales 867 868 @property 869 def coverage_directory(self): 870 return self._coverage_directory 871 872 @property 873 def enable_breakpad_dump(self): 874 return self._enable_breakpad_dump 875 876 @property 877 def external_shard_index(self): 878 return self._external_shard_index 879 880 @property 881 def flags(self): 882 return self._flags[:] 883 884 @property 885 def is_unit_test(self): 886 return self._is_unit_test 887 888 @property 889 def junit4_runner_class(self): 890 return self._junit4_runner_class 891 892 @property 893 def has_chromium_test_listener(self): 894 return self._has_chromium_test_listener 895 896 @property 897 def has_external_annotation_filters(self): 898 return self._has_external_annotation_filters 899 900 @property 901 def uses_base_instrumentation(self): 902 return self._uses_base_instrumentation 903 904 @property 905 def package_info(self): 906 return self._package_info 907 908 @property 909 def replace_system_package(self): 910 return self._replace_system_package 911 912 @property 913 def run_setup_commands(self): 914 return self._run_setup_commands 915 916 @property 917 def run_teardown_commands(self): 918 return self._run_teardown_commands 919 920 @property 921 def use_voice_interaction_service(self): 922 return self._use_voice_interaction_service 923 924 @property 925 def use_webview_provider(self): 926 return self._use_webview_provider 927 928 @property 929 def webview_flags(self): 930 return self._webview_flags[:] 931 932 @property 933 def screenshot_dir(self): 934 return self._screenshot_dir 935 936 @property 937 def skia_gold_properties(self): 938 return self._skia_gold_properties 939 940 @property 941 def store_data_dependencies_in_temp(self): 942 return self._store_data_dependencies_in_temp 943 944 @property 945 def store_tombstones(self): 946 return self._store_tombstones 947 948 @property 949 def suite(self): 950 return self._suite 951 952 @property 953 def symbolizer(self): 954 return self._symbolizer 955 956 @property 957 def system_packages_to_remove(self): 958 return self._system_packages_to_remove 959 960 @property 961 def test_apk(self): 962 return self._test_apk 963 964 @property 965 def test_apk_as_instant(self): 966 return self._test_apk_as_instant 967 968 @property 969 def test_apk_incremental_install_json(self): 970 return self._test_apk_incremental_install_json 971 972 @property 973 def test_filters(self): 974 return self._test_filters 975 976 @property 977 def test_launcher_batch_limit(self): 978 return self._test_launcher_batch_limit 979 980 @property 981 def test_support_apk(self): 982 return self._test_support_apk 983 984 @property 985 def test_package(self): 986 return self._test_package 987 988 @property 989 def timeout_scale(self): 990 return self._timeout_scale 991 992 @property 993 def total_external_shards(self): 994 return self._total_external_shards 995 996 @property 997 def use_apk_under_test_flags_file(self): 998 return self._use_apk_under_test_flags_file 999 1000 @property 1001 def variations_test_seed_path(self): 1002 return self._variations_test_seed_path 1003 1004 @property 1005 def webview_variations_test_seed_path(self): 1006 return self._webview_variations_test_seed_path 1007 1008 @property 1009 def wait_for_java_debugger(self): 1010 return self._wait_for_java_debugger 1011 1012 @property 1013 def wpr_record_mode(self): 1014 return self._wpr_enable_record 1015 1016 @property 1017 def wpr_replay_mode(self): 1018 return not self._wpr_enable_record 1019 1020 #override 1021 def TestType(self): 1022 return 'instrumentation' 1023 1024 #override 1025 def GetPreferredAbis(self): 1026 # We could alternatively take the intersection of what they all support, 1027 # but it should never be the case that they support different things. 1028 apks = [self._test_apk, self._apk_under_test] + self._additional_apks 1029 for apk in apks: 1030 if apk: 1031 ret = apk.GetAbis() 1032 if ret: 1033 return ret 1034 return [] 1035 1036 #override 1037 def SetUp(self): 1038 self._data_deps.extend( 1039 self._data_deps_delegate(self._runtime_deps_path)) 1040 if self._proguard_mapping_path: 1041 self._deobfuscator = deobfuscator.DeobfuscatorPool( 1042 self._proguard_mapping_path) 1043 1044 def GetDataDependencies(self): 1045 return self._data_deps 1046 1047 def GetRunDisabledFlag(self): 1048 return self._run_disabled 1049 1050 def MaybeDeobfuscateLines(self, lines): 1051 if not self._deobfuscator: 1052 return lines 1053 return self._deobfuscator.TransformLines(lines) 1054 1055 def ProcessRawTests(self, raw_tests): 1056 inflated_tests = self._ParameterizeTestsWithFlags( 1057 self._InflateTests(raw_tests)) 1058 filtered_tests = FilterTests(inflated_tests, self._test_filters, 1059 self._annotations, self._excluded_annotations) 1060 if self._test_filters and not filtered_tests: 1061 for t in inflated_tests: 1062 logging.debug(' %s', GetUniqueTestName(t)) 1063 logging.warning('Unmatched Filters: %s', self._test_filters) 1064 return filtered_tests 1065 1066 def IsApkForceQueryable(self, apk): 1067 return apk in self._forced_queryable_additional_apks 1068 1069 def IsApkInstant(self, apk): 1070 return apk in self._instant_additional_apks 1071 1072 # pylint: disable=no-self-use 1073 def _InflateTests(self, tests): 1074 inflated_tests = [] 1075 for clazz in tests: 1076 for method in clazz['methods']: 1077 annotations = dict(clazz['annotations']) 1078 annotations.update(method['annotations']) 1079 1080 # Preserve historic default. 1081 if (not self._uses_base_instrumentation 1082 and not any(a in _VALID_ANNOTATIONS for a in annotations)): 1083 annotations['MediumTest'] = None 1084 1085 inflated_tests.append({ 1086 'class': clazz['class'], 1087 'method': method['method'], 1088 'annotations': annotations, 1089 }) 1090 return inflated_tests 1091 1092 def _ParameterizeTestsWithFlags(self, tests): 1093 1094 def _checkParameterization(annotations): 1095 types = [ 1096 _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES, 1097 _PARAMETERIZED_COMMAND_LINE_FLAGS, 1098 ] 1099 if types[0] in annotations and types[1] in annotations: 1100 raise CommandLineParameterizationException( 1101 'Multiple command-line parameterization types: {}.'.format( 1102 ', '.join(types))) 1103 1104 def _switchesToFlags(switches): 1105 return ['--{}'.format(s) for s in switches if s] 1106 1107 def _annotationToSwitches(clazz, methods): 1108 if clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES: 1109 return [methods['value']] 1110 if clazz == _PARAMETERIZED_COMMAND_LINE_FLAGS: 1111 list_of_switches = [] 1112 for annotation in methods['value']: 1113 for c, m in annotation.items(): 1114 list_of_switches += _annotationToSwitches(c, m) 1115 return list_of_switches 1116 return [] 1117 1118 def _setTestFlags(test, flags): 1119 if flags: 1120 test['flags'] = flags 1121 elif 'flags' in test: 1122 del test['flags'] 1123 1124 new_tests = [] 1125 for t in tests: 1126 annotations = t['annotations'] 1127 list_of_switches = [] 1128 _checkParameterization(annotations) 1129 if _SKIP_PARAMETERIZATION not in annotations: 1130 for clazz, methods in annotations.items(): 1131 list_of_switches += _annotationToSwitches(clazz, methods) 1132 if list_of_switches: 1133 _setTestFlags(t, _switchesToFlags(list_of_switches[0])) 1134 for p in list_of_switches[1:]: 1135 parameterized_t = copy.copy(t) 1136 _setTestFlags(parameterized_t, _switchesToFlags(p)) 1137 new_tests.append(parameterized_t) 1138 return tests + new_tests 1139 1140 @staticmethod 1141 def GenerateTestResults(result_code, result_bundle, statuses, duration_ms, 1142 device_abi, symbolizer): 1143 return GenerateTestResults(result_code, result_bundle, statuses, 1144 duration_ms, device_abi, symbolizer) 1145 1146 #override 1147 def TearDown(self): 1148 self.symbolizer.CleanUp() 1149 if self._deobfuscator: 1150 self._deobfuscator.Close() 1151 self._deobfuscator = None 1152