1*6777b538SAndroid Build Coastguard Worker# Copyright 2014 The Chromium Authors 2*6777b538SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be 3*6777b538SAndroid Build Coastguard Worker# found in the LICENSE file. 4*6777b538SAndroid Build Coastguard Worker 5*6777b538SAndroid Build Coastguard Workerfrom __future__ import print_function 6*6777b538SAndroid Build Coastguard Workerimport argparse 7*6777b538SAndroid Build Coastguard Workerimport codecs 8*6777b538SAndroid Build Coastguard Workerimport contextlib 9*6777b538SAndroid Build Coastguard Workerimport json 10*6777b538SAndroid Build Coastguard Workerimport os 11*6777b538SAndroid Build Coastguard Workerimport logging 12*6777b538SAndroid Build Coastguard Workerimport platform 13*6777b538SAndroid Build Coastguard Workerimport subprocess 14*6777b538SAndroid Build Coastguard Workerimport sys 15*6777b538SAndroid Build Coastguard Workerimport tempfile 16*6777b538SAndroid Build Coastguard Workerimport time 17*6777b538SAndroid Build Coastguard Workerimport traceback 18*6777b538SAndroid Build Coastguard Worker 19*6777b538SAndroid Build Coastguard Workerlogging.basicConfig(level=logging.INFO) 20*6777b538SAndroid Build Coastguard Worker 21*6777b538SAndroid Build Coastguard Worker# Add src/testing/ into sys.path for importing xvfb and test_env. 22*6777b538SAndroid Build Coastguard Workersys.path.append( 23*6777b538SAndroid Build Coastguard Worker os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) 24*6777b538SAndroid Build Coastguard Workerimport test_env 25*6777b538SAndroid Build Coastguard Workerif sys.platform.startswith('linux'): 26*6777b538SAndroid Build Coastguard Worker import xvfb 27*6777b538SAndroid Build Coastguard Worker 28*6777b538SAndroid Build Coastguard Worker 29*6777b538SAndroid Build Coastguard WorkerSCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) 30*6777b538SAndroid Build Coastguard WorkerSRC_DIR = os.path.abspath( 31*6777b538SAndroid Build Coastguard Worker os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir)) 32*6777b538SAndroid Build Coastguard Worker 33*6777b538SAndroid Build Coastguard Worker# Use result_sink.py in //build/util/lib/results/ for uploading the 34*6777b538SAndroid Build Coastguard Worker# results of non-isolated script tests. 35*6777b538SAndroid Build Coastguard WorkerBUILD_UTIL_DIR = os.path.join(SRC_DIR, 'build', 'util') 36*6777b538SAndroid Build Coastguard Workersys.path.insert(0, BUILD_UTIL_DIR) 37*6777b538SAndroid Build Coastguard Workertry: 38*6777b538SAndroid Build Coastguard Worker from lib.results import result_sink 39*6777b538SAndroid Build Coastguard Worker from lib.results import result_types 40*6777b538SAndroid Build Coastguard Workerexcept ImportError: 41*6777b538SAndroid Build Coastguard Worker # Some build-time scripts import this file and run into issues with 42*6777b538SAndroid Build Coastguard Worker # result_sink's dependency on requests since we can't depend on vpython 43*6777b538SAndroid Build Coastguard Worker # during build-time. So silently swallow the error in that case. 44*6777b538SAndroid Build Coastguard Worker result_sink = None 45*6777b538SAndroid Build Coastguard Worker 46*6777b538SAndroid Build Coastguard Worker# run_web_tests.py returns the number of failures as the return 47*6777b538SAndroid Build Coastguard Worker# code, but caps the return code at 101 to avoid overflow or colliding 48*6777b538SAndroid Build Coastguard Worker# with reserved values from the shell. 49*6777b538SAndroid Build Coastguard WorkerMAX_FAILURES_EXIT_STATUS = 101 50*6777b538SAndroid Build Coastguard Worker 51*6777b538SAndroid Build Coastguard Worker 52*6777b538SAndroid Build Coastguard Worker# Exit code to indicate infrastructure issue. 53*6777b538SAndroid Build Coastguard WorkerINFRA_FAILURE_EXIT_CODE = 87 54*6777b538SAndroid Build Coastguard Worker 55*6777b538SAndroid Build Coastguard Worker 56*6777b538SAndroid Build Coastguard Worker# ACL might be explicitly set or inherited. 57*6777b538SAndroid Build Coastguard WorkerCORRECT_ACL_VARIANTS = [ 58*6777b538SAndroid Build Coastguard Worker 'APPLICATION PACKAGE AUTHORITY' \ 59*6777b538SAndroid Build Coastguard Worker '\\ALL RESTRICTED APPLICATION PACKAGES:(OI)(CI)(RX)', \ 60*6777b538SAndroid Build Coastguard Worker 'APPLICATION PACKAGE AUTHORITY' \ 61*6777b538SAndroid Build Coastguard Worker '\\ALL RESTRICTED APPLICATION PACKAGES:(I)(OI)(CI)(RX)' 62*6777b538SAndroid Build Coastguard Worker] 63*6777b538SAndroid Build Coastguard Worker 64*6777b538SAndroid Build Coastguard Worker# pylint: disable=useless-object-inheritance 65*6777b538SAndroid Build Coastguard Worker 66*6777b538SAndroid Build Coastguard Worker 67*6777b538SAndroid Build Coastguard Workerdef set_lpac_acls(acl_dir, is_test_script=False): 68*6777b538SAndroid Build Coastguard Worker """Sets LPAC ACLs on a directory. Windows 10 only.""" 69*6777b538SAndroid Build Coastguard Worker if platform.release() != '10': 70*6777b538SAndroid Build Coastguard Worker return 71*6777b538SAndroid Build Coastguard Worker try: 72*6777b538SAndroid Build Coastguard Worker existing_acls = subprocess.check_output(['icacls', acl_dir], 73*6777b538SAndroid Build Coastguard Worker stderr=subprocess.STDOUT, 74*6777b538SAndroid Build Coastguard Worker universal_newlines=True) 75*6777b538SAndroid Build Coastguard Worker except subprocess.CalledProcessError as e: 76*6777b538SAndroid Build Coastguard Worker logging.error('Failed to retrieve existing ACLs for directory %s', acl_dir) 77*6777b538SAndroid Build Coastguard Worker logging.error('Command output: %s', e.output) 78*6777b538SAndroid Build Coastguard Worker sys.exit(e.returncode) 79*6777b538SAndroid Build Coastguard Worker acls_correct = False 80*6777b538SAndroid Build Coastguard Worker for acl in CORRECT_ACL_VARIANTS: 81*6777b538SAndroid Build Coastguard Worker if acl in existing_acls: 82*6777b538SAndroid Build Coastguard Worker acls_correct = True 83*6777b538SAndroid Build Coastguard Worker if not acls_correct: 84*6777b538SAndroid Build Coastguard Worker try: 85*6777b538SAndroid Build Coastguard Worker existing_acls = subprocess.check_output( 86*6777b538SAndroid Build Coastguard Worker ['icacls', acl_dir, '/grant', '*S-1-15-2-2:(OI)(CI)(RX)'], 87*6777b538SAndroid Build Coastguard Worker stderr=subprocess.STDOUT) 88*6777b538SAndroid Build Coastguard Worker except subprocess.CalledProcessError as e: 89*6777b538SAndroid Build Coastguard Worker logging.error( 90*6777b538SAndroid Build Coastguard Worker 'Failed to retrieve existing ACLs for directory %s', acl_dir) 91*6777b538SAndroid Build Coastguard Worker logging.error('Command output: %s', e.output) 92*6777b538SAndroid Build Coastguard Worker sys.exit(e.returncode) 93*6777b538SAndroid Build Coastguard Worker if not is_test_script: 94*6777b538SAndroid Build Coastguard Worker return 95*6777b538SAndroid Build Coastguard Worker # Bots running on luci use hardlinks that do not have correct ACLs so these 96*6777b538SAndroid Build Coastguard Worker # must be manually overridden here. 97*6777b538SAndroid Build Coastguard Worker with temporary_file() as tempfile_path: 98*6777b538SAndroid Build Coastguard Worker subprocess.check_output( 99*6777b538SAndroid Build Coastguard Worker ['icacls', acl_dir, '/save', tempfile_path, '/t', '/q', '/c'], 100*6777b538SAndroid Build Coastguard Worker stderr=subprocess.STDOUT) 101*6777b538SAndroid Build Coastguard Worker # ACL files look like this, e.g. for c:\a\b\c\d\Release_x64 102*6777b538SAndroid Build Coastguard Worker # 103*6777b538SAndroid Build Coastguard Worker # Release_x64 104*6777b538SAndroid Build Coastguard Worker # D:AI(A;OICI;0x1200a9;;;S-1-15-2-2)(A;OICIID;FA;;;BA) 105*6777b538SAndroid Build Coastguard Worker # Release_x64\icudtl_extra.dat 106*6777b538SAndroid Build Coastguard Worker # D:AI(A;ID;0x1200a9;;;S-1-15-2-2)(A;ID;FA;;;BA)(A;ID;0x1301bf;;;BU) 107*6777b538SAndroid Build Coastguard Worker with codecs.open(tempfile_path, encoding='utf_16_le') as aclfile: 108*6777b538SAndroid Build Coastguard Worker for filename in aclfile: 109*6777b538SAndroid Build Coastguard Worker acl = next(aclfile).strip() 110*6777b538SAndroid Build Coastguard Worker full_filename = os.path.abspath( 111*6777b538SAndroid Build Coastguard Worker os.path.join(acl_dir, os.pardir, filename.strip())) 112*6777b538SAndroid Build Coastguard Worker if 'S-1-15-2-2' in acl: 113*6777b538SAndroid Build Coastguard Worker continue 114*6777b538SAndroid Build Coastguard Worker if os.path.isdir(full_filename): 115*6777b538SAndroid Build Coastguard Worker continue 116*6777b538SAndroid Build Coastguard Worker subprocess.check_output( 117*6777b538SAndroid Build Coastguard Worker ['icacls', full_filename, '/grant', '*S-1-15-2-2:(RX)'], 118*6777b538SAndroid Build Coastguard Worker stderr=subprocess.STDOUT) 119*6777b538SAndroid Build Coastguard Worker 120*6777b538SAndroid Build Coastguard Worker 121*6777b538SAndroid Build Coastguard Workerdef run_script(argv, funcs): 122*6777b538SAndroid Build Coastguard Worker def parse_json(path): 123*6777b538SAndroid Build Coastguard Worker with open(path) as f: 124*6777b538SAndroid Build Coastguard Worker return json.load(f) 125*6777b538SAndroid Build Coastguard Worker parser = argparse.ArgumentParser() 126*6777b538SAndroid Build Coastguard Worker # TODO(phajdan.jr): Make build-config-fs required after passing it in recipe. 127*6777b538SAndroid Build Coastguard Worker parser.add_argument('--build-config-fs') 128*6777b538SAndroid Build Coastguard Worker parser.add_argument('--paths', type=parse_json, default={}) 129*6777b538SAndroid Build Coastguard Worker # Properties describe the environment of the build, and are the same per 130*6777b538SAndroid Build Coastguard Worker # script invocation. 131*6777b538SAndroid Build Coastguard Worker parser.add_argument('--properties', type=parse_json, default={}) 132*6777b538SAndroid Build Coastguard Worker # Args contains per-invocation arguments that potentially change the 133*6777b538SAndroid Build Coastguard Worker # behavior of the script. 134*6777b538SAndroid Build Coastguard Worker parser.add_argument('--args', type=parse_json, default=[]) 135*6777b538SAndroid Build Coastguard Worker 136*6777b538SAndroid Build Coastguard Worker subparsers = parser.add_subparsers() 137*6777b538SAndroid Build Coastguard Worker 138*6777b538SAndroid Build Coastguard Worker run_parser = subparsers.add_parser('run') 139*6777b538SAndroid Build Coastguard Worker run_parser.add_argument( 140*6777b538SAndroid Build Coastguard Worker '--output', type=argparse.FileType('w'), required=True) 141*6777b538SAndroid Build Coastguard Worker run_parser.add_argument('--filter-file', type=argparse.FileType('r')) 142*6777b538SAndroid Build Coastguard Worker run_parser.set_defaults(func=funcs['run']) 143*6777b538SAndroid Build Coastguard Worker 144*6777b538SAndroid Build Coastguard Worker run_parser = subparsers.add_parser('compile_targets') 145*6777b538SAndroid Build Coastguard Worker run_parser.add_argument( 146*6777b538SAndroid Build Coastguard Worker '--output', type=argparse.FileType('w'), required=True) 147*6777b538SAndroid Build Coastguard Worker run_parser.set_defaults(func=funcs['compile_targets']) 148*6777b538SAndroid Build Coastguard Worker 149*6777b538SAndroid Build Coastguard Worker args = parser.parse_args(argv) 150*6777b538SAndroid Build Coastguard Worker return args.func(args) 151*6777b538SAndroid Build Coastguard Worker 152*6777b538SAndroid Build Coastguard Worker 153*6777b538SAndroid Build Coastguard Workerdef run_command(argv, env=None, cwd=None): 154*6777b538SAndroid Build Coastguard Worker print('Running %r in %r (env: %r)' % (argv, cwd, env), file=sys.stderr) 155*6777b538SAndroid Build Coastguard Worker rc = test_env.run_command(argv, env=env, cwd=cwd) 156*6777b538SAndroid Build Coastguard Worker print('Command %r returned exit code %d' % (argv, rc), file=sys.stderr) 157*6777b538SAndroid Build Coastguard Worker return rc 158*6777b538SAndroid Build Coastguard Worker 159*6777b538SAndroid Build Coastguard Worker 160*6777b538SAndroid Build Coastguard Worker@contextlib.contextmanager 161*6777b538SAndroid Build Coastguard Workerdef temporary_file(): 162*6777b538SAndroid Build Coastguard Worker fd, path = tempfile.mkstemp() 163*6777b538SAndroid Build Coastguard Worker os.close(fd) 164*6777b538SAndroid Build Coastguard Worker try: 165*6777b538SAndroid Build Coastguard Worker yield path 166*6777b538SAndroid Build Coastguard Worker finally: 167*6777b538SAndroid Build Coastguard Worker os.remove(path) 168*6777b538SAndroid Build Coastguard Worker 169*6777b538SAndroid Build Coastguard Worker 170*6777b538SAndroid Build Coastguard Workerdef record_local_script_results(name, output_fd, failures, valid): 171*6777b538SAndroid Build Coastguard Worker """Records to a local json file and to RDB the results of the script test. 172*6777b538SAndroid Build Coastguard Worker 173*6777b538SAndroid Build Coastguard Worker For legacy reasons, local script tests (ie: script tests that run 174*6777b538SAndroid Build Coastguard Worker locally and that don't conform to the isolated-test API) are expected to 175*6777b538SAndroid Build Coastguard Worker record their results using a specific format. This method encapsulates 176*6777b538SAndroid Build Coastguard Worker that format and also uploads those results to Result DB. 177*6777b538SAndroid Build Coastguard Worker 178*6777b538SAndroid Build Coastguard Worker Args: 179*6777b538SAndroid Build Coastguard Worker name: Name of the script test. 180*6777b538SAndroid Build Coastguard Worker output_fd: A .write()-supporting file descriptor to write results to. 181*6777b538SAndroid Build Coastguard Worker failures: List of strings representing test failures. 182*6777b538SAndroid Build Coastguard Worker valid: Whether the results are valid. 183*6777b538SAndroid Build Coastguard Worker """ 184*6777b538SAndroid Build Coastguard Worker local_script_results = { 185*6777b538SAndroid Build Coastguard Worker 'valid': valid, 186*6777b538SAndroid Build Coastguard Worker 'failures': failures 187*6777b538SAndroid Build Coastguard Worker } 188*6777b538SAndroid Build Coastguard Worker json.dump(local_script_results, output_fd) 189*6777b538SAndroid Build Coastguard Worker 190*6777b538SAndroid Build Coastguard Worker if not result_sink: 191*6777b538SAndroid Build Coastguard Worker return 192*6777b538SAndroid Build Coastguard Worker result_sink_client = result_sink.TryInitClient() 193*6777b538SAndroid Build Coastguard Worker if not result_sink_client: 194*6777b538SAndroid Build Coastguard Worker return 195*6777b538SAndroid Build Coastguard Worker status = result_types.PASS 196*6777b538SAndroid Build Coastguard Worker if not valid: 197*6777b538SAndroid Build Coastguard Worker status = result_types.UNKNOWN 198*6777b538SAndroid Build Coastguard Worker elif failures: 199*6777b538SAndroid Build Coastguard Worker status = result_types.FAIL 200*6777b538SAndroid Build Coastguard Worker test_log = '\n'.join(failures) 201*6777b538SAndroid Build Coastguard Worker result_sink_client.Post(name, status, None, test_log, None) 202*6777b538SAndroid Build Coastguard Worker 203*6777b538SAndroid Build Coastguard Worker 204*6777b538SAndroid Build Coastguard Workerdef parse_common_test_results(json_results, test_separator='/'): 205*6777b538SAndroid Build Coastguard Worker def convert_trie_to_flat_paths(trie, prefix=None): 206*6777b538SAndroid Build Coastguard Worker # Also see blinkpy.web_tests.layout_package.json_results_generator 207*6777b538SAndroid Build Coastguard Worker result = {} 208*6777b538SAndroid Build Coastguard Worker for name, data in trie.items(): 209*6777b538SAndroid Build Coastguard Worker if prefix: 210*6777b538SAndroid Build Coastguard Worker name = prefix + test_separator + name 211*6777b538SAndroid Build Coastguard Worker if len(data) and not 'actual' in data and not 'expected' in data: 212*6777b538SAndroid Build Coastguard Worker result.update(convert_trie_to_flat_paths(data, name)) 213*6777b538SAndroid Build Coastguard Worker else: 214*6777b538SAndroid Build Coastguard Worker result[name] = data 215*6777b538SAndroid Build Coastguard Worker return result 216*6777b538SAndroid Build Coastguard Worker 217*6777b538SAndroid Build Coastguard Worker results = { 218*6777b538SAndroid Build Coastguard Worker 'passes': {}, 219*6777b538SAndroid Build Coastguard Worker 'unexpected_passes': {}, 220*6777b538SAndroid Build Coastguard Worker 'failures': {}, 221*6777b538SAndroid Build Coastguard Worker 'unexpected_failures': {}, 222*6777b538SAndroid Build Coastguard Worker 'flakes': {}, 223*6777b538SAndroid Build Coastguard Worker 'unexpected_flakes': {}, 224*6777b538SAndroid Build Coastguard Worker } 225*6777b538SAndroid Build Coastguard Worker 226*6777b538SAndroid Build Coastguard Worker # TODO(dpranke): crbug.com/357866 - we should simplify the handling of 227*6777b538SAndroid Build Coastguard Worker # both the return code and parsing the actual results, below. 228*6777b538SAndroid Build Coastguard Worker 229*6777b538SAndroid Build Coastguard Worker passing_statuses = ('PASS', 'SLOW', 'NEEDSREBASELINE') 230*6777b538SAndroid Build Coastguard Worker 231*6777b538SAndroid Build Coastguard Worker for test, result in convert_trie_to_flat_paths( 232*6777b538SAndroid Build Coastguard Worker json_results['tests']).items(): 233*6777b538SAndroid Build Coastguard Worker key = 'unexpected_' if result.get('is_unexpected') else '' 234*6777b538SAndroid Build Coastguard Worker data = result['actual'] 235*6777b538SAndroid Build Coastguard Worker actual_results = data.split() 236*6777b538SAndroid Build Coastguard Worker last_result = actual_results[-1] 237*6777b538SAndroid Build Coastguard Worker expected_results = result['expected'].split() 238*6777b538SAndroid Build Coastguard Worker 239*6777b538SAndroid Build Coastguard Worker if (len(actual_results) > 1 and 240*6777b538SAndroid Build Coastguard Worker (last_result in expected_results or last_result in passing_statuses)): 241*6777b538SAndroid Build Coastguard Worker key += 'flakes' 242*6777b538SAndroid Build Coastguard Worker elif last_result in passing_statuses: 243*6777b538SAndroid Build Coastguard Worker key += 'passes' 244*6777b538SAndroid Build Coastguard Worker # TODO(dpranke): crbug.com/357867 ... Why are we assigning result 245*6777b538SAndroid Build Coastguard Worker # instead of actual_result here. Do we even need these things to be 246*6777b538SAndroid Build Coastguard Worker # hashes, or just lists? 247*6777b538SAndroid Build Coastguard Worker data = result 248*6777b538SAndroid Build Coastguard Worker else: 249*6777b538SAndroid Build Coastguard Worker key += 'failures' 250*6777b538SAndroid Build Coastguard Worker results[key][test] = data 251*6777b538SAndroid Build Coastguard Worker 252*6777b538SAndroid Build Coastguard Worker return results 253*6777b538SAndroid Build Coastguard Worker 254*6777b538SAndroid Build Coastguard Worker 255*6777b538SAndroid Build Coastguard Workerdef write_interrupted_test_results_to(filepath, test_start_time): 256*6777b538SAndroid Build Coastguard Worker """Writes a test results JSON file* to filepath. 257*6777b538SAndroid Build Coastguard Worker 258*6777b538SAndroid Build Coastguard Worker This JSON file is formatted to explain that something went wrong. 259*6777b538SAndroid Build Coastguard Worker 260*6777b538SAndroid Build Coastguard Worker *src/docs/testing/json_test_results_format.md 261*6777b538SAndroid Build Coastguard Worker 262*6777b538SAndroid Build Coastguard Worker Args: 263*6777b538SAndroid Build Coastguard Worker filepath: A path to a file to write the output to. 264*6777b538SAndroid Build Coastguard Worker test_start_time: The start time of the test run expressed as a 265*6777b538SAndroid Build Coastguard Worker floating-point offset in seconds from the UNIX epoch. 266*6777b538SAndroid Build Coastguard Worker """ 267*6777b538SAndroid Build Coastguard Worker with open(filepath, 'w') as fh: 268*6777b538SAndroid Build Coastguard Worker output = { 269*6777b538SAndroid Build Coastguard Worker 'interrupted': True, 270*6777b538SAndroid Build Coastguard Worker 'num_failures_by_type': {}, 271*6777b538SAndroid Build Coastguard Worker 'seconds_since_epoch': test_start_time, 272*6777b538SAndroid Build Coastguard Worker 'tests': {}, 273*6777b538SAndroid Build Coastguard Worker 'version': 3, 274*6777b538SAndroid Build Coastguard Worker } 275*6777b538SAndroid Build Coastguard Worker json.dump(output, fh) 276*6777b538SAndroid Build Coastguard Worker 277*6777b538SAndroid Build Coastguard Worker 278*6777b538SAndroid Build Coastguard Workerdef get_gtest_summary_passes(output): 279*6777b538SAndroid Build Coastguard Worker """Returns a mapping of test to boolean indicating if the test passed. 280*6777b538SAndroid Build Coastguard Worker 281*6777b538SAndroid Build Coastguard Worker Only partially parses the format. This code is based on code in tools/build, 282*6777b538SAndroid Build Coastguard Worker specifically 283*6777b538SAndroid Build Coastguard Worker https://chromium.googlesource.com/chromium/tools/build/+/17fef98756c5f250b20bf716829a0004857235ff/scripts/slave/recipe_modules/test_utils/util.py#189 284*6777b538SAndroid Build Coastguard Worker """ 285*6777b538SAndroid Build Coastguard Worker if not output: 286*6777b538SAndroid Build Coastguard Worker return {} 287*6777b538SAndroid Build Coastguard Worker 288*6777b538SAndroid Build Coastguard Worker mapping = {} 289*6777b538SAndroid Build Coastguard Worker 290*6777b538SAndroid Build Coastguard Worker for cur_iteration_data in output.get('per_iteration_data', []): 291*6777b538SAndroid Build Coastguard Worker for test_fullname, results in cur_iteration_data.items(): 292*6777b538SAndroid Build Coastguard Worker # Results is a list with one entry per test try. Last one is the final 293*6777b538SAndroid Build Coastguard Worker # result. 294*6777b538SAndroid Build Coastguard Worker last_result = results[-1] 295*6777b538SAndroid Build Coastguard Worker 296*6777b538SAndroid Build Coastguard Worker if last_result['status'] == 'SUCCESS': 297*6777b538SAndroid Build Coastguard Worker mapping[test_fullname] = True 298*6777b538SAndroid Build Coastguard Worker elif last_result['status'] != 'SKIPPED': 299*6777b538SAndroid Build Coastguard Worker mapping[test_fullname] = False 300*6777b538SAndroid Build Coastguard Worker 301*6777b538SAndroid Build Coastguard Worker return mapping 302*6777b538SAndroid Build Coastguard Worker 303*6777b538SAndroid Build Coastguard Worker 304*6777b538SAndroid Build Coastguard Workerdef extract_filter_list(filter_list): 305*6777b538SAndroid Build Coastguard Worker """Helper for isolated script test wrappers. Parses the 306*6777b538SAndroid Build Coastguard Worker --isolated-script-test-filter command line argument. Currently, double-colon 307*6777b538SAndroid Build Coastguard Worker ('::') is used as the separator between test names, because a single colon may 308*6777b538SAndroid Build Coastguard Worker be used in the names of perf benchmarks, which contain URLs. 309*6777b538SAndroid Build Coastguard Worker """ 310*6777b538SAndroid Build Coastguard Worker return filter_list.split('::') 311*6777b538SAndroid Build Coastguard Worker 312*6777b538SAndroid Build Coastguard Worker 313*6777b538SAndroid Build Coastguard Workerdef add_emulator_args(parser): 314*6777b538SAndroid Build Coastguard Worker parser.add_argument( 315*6777b538SAndroid Build Coastguard Worker '--avd-config', 316*6777b538SAndroid Build Coastguard Worker type=os.path.realpath, 317*6777b538SAndroid Build Coastguard Worker help=('Path to the avd config. Required for Android products. ' 318*6777b538SAndroid Build Coastguard Worker '(See //tools/android/avd/proto for message definition ' 319*6777b538SAndroid Build Coastguard Worker 'and existing *.textpb files.)')) 320*6777b538SAndroid Build Coastguard Worker parser.add_argument( 321*6777b538SAndroid Build Coastguard Worker '--emulator-window', 322*6777b538SAndroid Build Coastguard Worker action='store_true', 323*6777b538SAndroid Build Coastguard Worker default=False, 324*6777b538SAndroid Build Coastguard Worker help='Enable graphical window display on the emulator.') 325*6777b538SAndroid Build Coastguard Worker 326*6777b538SAndroid Build Coastguard Worker 327*6777b538SAndroid Build Coastguard Workerclass BaseIsolatedScriptArgsAdapter: 328*6777b538SAndroid Build Coastguard Worker """The base class for all script adapters that need to translate flags 329*6777b538SAndroid Build Coastguard Worker set by isolated script test contract into the specific test script's flags. 330*6777b538SAndroid Build Coastguard Worker """ 331*6777b538SAndroid Build Coastguard Worker 332*6777b538SAndroid Build Coastguard Worker def __init__(self): 333*6777b538SAndroid Build Coastguard Worker self._parser = argparse.ArgumentParser() 334*6777b538SAndroid Build Coastguard Worker self._options = None 335*6777b538SAndroid Build Coastguard Worker self._rest_args = None 336*6777b538SAndroid Build Coastguard Worker self._script_writes_output_json = None 337*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 338*6777b538SAndroid Build Coastguard Worker '--isolated-outdir', type=str, 339*6777b538SAndroid Build Coastguard Worker required=False, 340*6777b538SAndroid Build Coastguard Worker help='value of $ISOLATED_OUTDIR from swarming task') 341*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 342*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-output', type=os.path.abspath, 343*6777b538SAndroid Build Coastguard Worker required=False, 344*6777b538SAndroid Build Coastguard Worker help='path to write test results JSON object to') 345*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 346*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-filter', type=str, 347*6777b538SAndroid Build Coastguard Worker required=False) 348*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 349*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-repeat', type=int, 350*6777b538SAndroid Build Coastguard Worker required=False) 351*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 352*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-launcher-retry-limit', type=int, 353*6777b538SAndroid Build Coastguard Worker required=False) 354*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 355*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-also-run-disabled-tests', 356*6777b538SAndroid Build Coastguard Worker default=False, action='store_true', required=False) 357*6777b538SAndroid Build Coastguard Worker 358*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 359*6777b538SAndroid Build Coastguard Worker '--xvfb', 360*6777b538SAndroid Build Coastguard Worker help='start xvfb. Ignored on unsupported platforms', 361*6777b538SAndroid Build Coastguard Worker action='store_true') 362*6777b538SAndroid Build Coastguard Worker # Used to create the correct subclass. 363*6777b538SAndroid Build Coastguard Worker self._parser.add_argument( 364*6777b538SAndroid Build Coastguard Worker '--script-type', choices=['isolated', 'typ', 'bare'], 365*6777b538SAndroid Build Coastguard Worker help='Which script adapter to use') 366*6777b538SAndroid Build Coastguard Worker 367*6777b538SAndroid Build Coastguard Worker # Arguments that are ignored, but added here because it's easier to ignore 368*6777b538SAndroid Build Coastguard Worker # them to to update bot configs to not pass them. 369*6777b538SAndroid Build Coastguard Worker self._parser.add_argument('--isolated-script-test-chartjson-output') 370*6777b538SAndroid Build Coastguard Worker self._parser.add_argument('--isolated-script-test-perf-output') 371*6777b538SAndroid Build Coastguard Worker 372*6777b538SAndroid Build Coastguard Worker def parse_args(self, args=None): 373*6777b538SAndroid Build Coastguard Worker self._options, self._rest_args = self._parser.parse_known_args(args) 374*6777b538SAndroid Build Coastguard Worker 375*6777b538SAndroid Build Coastguard Worker @property 376*6777b538SAndroid Build Coastguard Worker def parser(self): 377*6777b538SAndroid Build Coastguard Worker return self._parser 378*6777b538SAndroid Build Coastguard Worker 379*6777b538SAndroid Build Coastguard Worker @property 380*6777b538SAndroid Build Coastguard Worker def options(self): 381*6777b538SAndroid Build Coastguard Worker return self._options 382*6777b538SAndroid Build Coastguard Worker 383*6777b538SAndroid Build Coastguard Worker @property 384*6777b538SAndroid Build Coastguard Worker def rest_args(self): 385*6777b538SAndroid Build Coastguard Worker return self._rest_args 386*6777b538SAndroid Build Coastguard Worker 387*6777b538SAndroid Build Coastguard Worker def generate_test_output_args(self, output): 388*6777b538SAndroid Build Coastguard Worker del output # unused 389*6777b538SAndroid Build Coastguard Worker return [] 390*6777b538SAndroid Build Coastguard Worker 391*6777b538SAndroid Build Coastguard Worker def generate_test_filter_args(self, test_filter_str): 392*6777b538SAndroid Build Coastguard Worker del test_filter_str # unused 393*6777b538SAndroid Build Coastguard Worker raise RuntimeError('Flag not supported.') 394*6777b538SAndroid Build Coastguard Worker 395*6777b538SAndroid Build Coastguard Worker def generate_test_repeat_args(self, repeat_count): 396*6777b538SAndroid Build Coastguard Worker del repeat_count # unused 397*6777b538SAndroid Build Coastguard Worker raise RuntimeError('Flag not supported.') 398*6777b538SAndroid Build Coastguard Worker 399*6777b538SAndroid Build Coastguard Worker def generate_test_launcher_retry_limit_args(self, retry_limit): 400*6777b538SAndroid Build Coastguard Worker del retry_limit # unused 401*6777b538SAndroid Build Coastguard Worker raise RuntimeError('Flag not supported.') 402*6777b538SAndroid Build Coastguard Worker 403*6777b538SAndroid Build Coastguard Worker def generate_sharding_args(self, total_shards, shard_index): 404*6777b538SAndroid Build Coastguard Worker del total_shards, shard_index # unused 405*6777b538SAndroid Build Coastguard Worker raise RuntimeError('Flag not supported.') 406*6777b538SAndroid Build Coastguard Worker 407*6777b538SAndroid Build Coastguard Worker def generate_test_also_run_disabled_tests_args(self): 408*6777b538SAndroid Build Coastguard Worker raise RuntimeError('Flag not supported.') 409*6777b538SAndroid Build Coastguard Worker 410*6777b538SAndroid Build Coastguard Worker def select_python_executable(self): 411*6777b538SAndroid Build Coastguard Worker return sys.executable 412*6777b538SAndroid Build Coastguard Worker 413*6777b538SAndroid Build Coastguard Worker def generate_isolated_script_cmd(self): 414*6777b538SAndroid Build Coastguard Worker isolated_script_cmd = [ self.select_python_executable() ] + self.rest_args 415*6777b538SAndroid Build Coastguard Worker 416*6777b538SAndroid Build Coastguard Worker if self.options.isolated_script_test_output: 417*6777b538SAndroid Build Coastguard Worker output_args = self.generate_test_output_args( 418*6777b538SAndroid Build Coastguard Worker self.options.isolated_script_test_output) 419*6777b538SAndroid Build Coastguard Worker self._script_writes_output_json = bool(output_args) 420*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += output_args 421*6777b538SAndroid Build Coastguard Worker 422*6777b538SAndroid Build Coastguard Worker # Augment test filter args if needed 423*6777b538SAndroid Build Coastguard Worker if self.options.isolated_script_test_filter: 424*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += self.generate_test_filter_args( 425*6777b538SAndroid Build Coastguard Worker self.options.isolated_script_test_filter) 426*6777b538SAndroid Build Coastguard Worker 427*6777b538SAndroid Build Coastguard Worker # Augment test repeat if needed 428*6777b538SAndroid Build Coastguard Worker if self.options.isolated_script_test_repeat is not None: 429*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += self.generate_test_repeat_args( 430*6777b538SAndroid Build Coastguard Worker self.options.isolated_script_test_repeat) 431*6777b538SAndroid Build Coastguard Worker 432*6777b538SAndroid Build Coastguard Worker # Augment test launcher retry limit args if needed 433*6777b538SAndroid Build Coastguard Worker if self.options.isolated_script_test_launcher_retry_limit is not None: 434*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += self.generate_test_launcher_retry_limit_args( 435*6777b538SAndroid Build Coastguard Worker self.options.isolated_script_test_launcher_retry_limit) 436*6777b538SAndroid Build Coastguard Worker 437*6777b538SAndroid Build Coastguard Worker # Augment test also run disable tests args if needed 438*6777b538SAndroid Build Coastguard Worker if self.options.isolated_script_test_also_run_disabled_tests: 439*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += self.generate_test_also_run_disabled_tests_args() 440*6777b538SAndroid Build Coastguard Worker 441*6777b538SAndroid Build Coastguard Worker # Augment shard args if needed 442*6777b538SAndroid Build Coastguard Worker env = os.environ.copy() 443*6777b538SAndroid Build Coastguard Worker 444*6777b538SAndroid Build Coastguard Worker total_shards = None 445*6777b538SAndroid Build Coastguard Worker shard_index = None 446*6777b538SAndroid Build Coastguard Worker 447*6777b538SAndroid Build Coastguard Worker if 'GTEST_TOTAL_SHARDS' in env: 448*6777b538SAndroid Build Coastguard Worker total_shards = int(env['GTEST_TOTAL_SHARDS']) 449*6777b538SAndroid Build Coastguard Worker if 'GTEST_SHARD_INDEX' in env: 450*6777b538SAndroid Build Coastguard Worker shard_index = int(env['GTEST_SHARD_INDEX']) 451*6777b538SAndroid Build Coastguard Worker if total_shards is not None and shard_index is not None: 452*6777b538SAndroid Build Coastguard Worker isolated_script_cmd += self.generate_sharding_args( 453*6777b538SAndroid Build Coastguard Worker total_shards, shard_index) 454*6777b538SAndroid Build Coastguard Worker 455*6777b538SAndroid Build Coastguard Worker return isolated_script_cmd 456*6777b538SAndroid Build Coastguard Worker 457*6777b538SAndroid Build Coastguard Worker def clean_up_after_test_run(self): 458*6777b538SAndroid Build Coastguard Worker pass 459*6777b538SAndroid Build Coastguard Worker 460*6777b538SAndroid Build Coastguard Worker def do_pre_test_run_tasks(self): 461*6777b538SAndroid Build Coastguard Worker pass 462*6777b538SAndroid Build Coastguard Worker 463*6777b538SAndroid Build Coastguard Worker def do_post_test_run_tasks(self): 464*6777b538SAndroid Build Coastguard Worker pass 465*6777b538SAndroid Build Coastguard Worker 466*6777b538SAndroid Build Coastguard Worker def _write_simple_test_results(self, start_time, exit_code): 467*6777b538SAndroid Build Coastguard Worker if exit_code is None: 468*6777b538SAndroid Build Coastguard Worker failure_type = 'CRASH' 469*6777b538SAndroid Build Coastguard Worker elif exit_code == 0: 470*6777b538SAndroid Build Coastguard Worker failure_type = 'PASS' 471*6777b538SAndroid Build Coastguard Worker else: 472*6777b538SAndroid Build Coastguard Worker failure_type = 'FAIL' 473*6777b538SAndroid Build Coastguard Worker 474*6777b538SAndroid Build Coastguard Worker test_name = os.path.basename(self._rest_args[0]) 475*6777b538SAndroid Build Coastguard Worker # See //docs/testing/json_test_results_format.md 476*6777b538SAndroid Build Coastguard Worker results_json = { 477*6777b538SAndroid Build Coastguard Worker 'version': 3, 478*6777b538SAndroid Build Coastguard Worker 'interrupted': False, 479*6777b538SAndroid Build Coastguard Worker 'num_failures_by_type': { failure_type: 1 }, 480*6777b538SAndroid Build Coastguard Worker 'path_delimiter': '/', 481*6777b538SAndroid Build Coastguard Worker 'seconds_since_epoch': start_time, 482*6777b538SAndroid Build Coastguard Worker 'tests': { 483*6777b538SAndroid Build Coastguard Worker test_name: { 484*6777b538SAndroid Build Coastguard Worker 'expected': 'PASS', 485*6777b538SAndroid Build Coastguard Worker 'actual': failure_type, 486*6777b538SAndroid Build Coastguard Worker 'time': time.time() - start_time, 487*6777b538SAndroid Build Coastguard Worker }, 488*6777b538SAndroid Build Coastguard Worker }, 489*6777b538SAndroid Build Coastguard Worker } 490*6777b538SAndroid Build Coastguard Worker with open(self.options.isolated_script_test_output, 'w') as fp: 491*6777b538SAndroid Build Coastguard Worker json.dump(results_json, fp) 492*6777b538SAndroid Build Coastguard Worker 493*6777b538SAndroid Build Coastguard Worker 494*6777b538SAndroid Build Coastguard Worker def run_test(self, cwd=None): 495*6777b538SAndroid Build Coastguard Worker self.parse_args() 496*6777b538SAndroid Build Coastguard Worker cmd = self.generate_isolated_script_cmd() 497*6777b538SAndroid Build Coastguard Worker 498*6777b538SAndroid Build Coastguard Worker self.do_pre_test_run_tasks() 499*6777b538SAndroid Build Coastguard Worker 500*6777b538SAndroid Build Coastguard Worker env = os.environ.copy() 501*6777b538SAndroid Build Coastguard Worker 502*6777b538SAndroid Build Coastguard Worker env['CHROME_HEADLESS'] = '1' 503*6777b538SAndroid Build Coastguard Worker print('Running command: %s\nwith env: %r' % ( 504*6777b538SAndroid Build Coastguard Worker ' '.join(cmd), env)) 505*6777b538SAndroid Build Coastguard Worker sys.stdout.flush() 506*6777b538SAndroid Build Coastguard Worker start_time = time.time() 507*6777b538SAndroid Build Coastguard Worker try: 508*6777b538SAndroid Build Coastguard Worker if self.options.xvfb and sys.platform.startswith('linux'): 509*6777b538SAndroid Build Coastguard Worker exit_code = xvfb.run_executable(cmd, env, cwd=cwd) 510*6777b538SAndroid Build Coastguard Worker else: 511*6777b538SAndroid Build Coastguard Worker exit_code = test_env.run_command(cmd, env=env, cwd=cwd, log=False) 512*6777b538SAndroid Build Coastguard Worker print('Command returned exit code %d' % exit_code) 513*6777b538SAndroid Build Coastguard Worker sys.stdout.flush() 514*6777b538SAndroid Build Coastguard Worker self.do_post_test_run_tasks() 515*6777b538SAndroid Build Coastguard Worker except Exception: 516*6777b538SAndroid Build Coastguard Worker traceback.print_exc() 517*6777b538SAndroid Build Coastguard Worker exit_code = None 518*6777b538SAndroid Build Coastguard Worker finally: 519*6777b538SAndroid Build Coastguard Worker self.clean_up_after_test_run() 520*6777b538SAndroid Build Coastguard Worker 521*6777b538SAndroid Build Coastguard Worker if (self.options.isolated_script_test_output 522*6777b538SAndroid Build Coastguard Worker and not self._script_writes_output_json): 523*6777b538SAndroid Build Coastguard Worker self._write_simple_test_results(start_time, exit_code) 524*6777b538SAndroid Build Coastguard Worker 525*6777b538SAndroid Build Coastguard Worker return exit_code if exit_code is not None else 2 526