1# Copyright 2021 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""This is a library for working with test executables in a way that is 5Chromium-bot-friendly as specified by //docs/testing/test_executable_api.md 6 7Example usage: 8 import os 9 import sys 10 11 import main_program 12 import rust_main_program 13 14 if __name__ == '__main__': 15 cmdline_parser = argparse.ArgumentParser() 16 main_program.add_cmdline_args(cmdline_parser) 17 ... adding other cmdline parameter definitions ... 18 parsed_cmdline_args = cmdline_parser.parse_args() 19 20 test_executable_wrappers = [] 21 test_executable_wrappers.append( 22 rust_main_program.TestExecutableWrapper(...)) 23 ... 24 25 main_program.main( 26 test_executable_wrappers, parsed_cmdline_args, os.environ) 27""" 28 29import argparse 30import time 31 32import test_filtering 33import test_results 34 35 36def add_cmdline_args(argparse_parser): 37 """Adds test-filtering-specific cmdline parameter definitions to 38 `argparse_parser`. 39 40 Args: 41 argparse_parser: An object of argparse.ArgumentParser type. 42 """ 43 test_filtering.add_cmdline_args(argparse_parser) 44 test_results.add_cmdline_args(argparse_parser) 45 46 argparse_parser.add_argument( 47 '--isolated-script-test-launcher-retry-limit', 48 dest='retry_limit', 49 default=3, 50 help='Sets the limit of test retries on failures to N.', 51 metavar='N', 52 type=int) 53 argparse_parser.add_argument('--isolated-script-test-repeat', 54 dest='repetitions', 55 default=1, 56 help='Repeats each test N times.', 57 metavar='N', 58 type=int) 59 60 61def _calculate_tests_to_run(argparse_parsed_args, env, 62 test_executable_wrappers): 63 tests = [] 64 for wrapper in test_executable_wrappers: 65 extra_tests = wrapper.list_all_tests() 66 for extra_test in extra_tests: 67 assert extra_test not in tests 68 tests.extend(extra_tests) 69 return test_filtering.filter_tests(argparse_parsed_args, env, tests) 70 71 72def _run_tests_and_save_results(argparse_parsed_args, list_of_tests_to_run, 73 test_executable_wrapper): 74 start_time = time.time() 75 results = [] 76 for wrapper in test_executable_wrapper: 77 results.extend(wrapper.run_tests(list_of_tests_to_run)) 78 test_results.print_test_results(argparse_parsed_args, results, start_time) 79 80 81def main(test_executable_wrappers, argparse_parsed_args, env): 82 """Runs tests within `test_executable_wrappers` using cmdline arguments and 83 environment variables to figure out 1) which subset of tests to run, 2) 84 where to save the JSON file with test results. 85 86 Args: 87 test_executable_wrappers: A list of objects providing 88 list_all_tests(...) and run_tests(...) methods (see 89 rust_main_program._TestExecutableWrapper). 90 argparse_parsed_arg: A result of an earlier call to 91 argparse_parser.parse_args() call (where `argparse_parser` has been 92 populated via an even earlier call to add_cmdline_args). 93 env: a dictionary-like object (typically from `os.environ`). 94 """ 95 list_of_test_names_to_run = _calculate_tests_to_run( 96 argparse_parsed_args, env, test_executable_wrappers) 97 _run_tests_and_save_results(argparse_parsed_args, 98 list_of_test_names_to_run, 99 test_executable_wrappers) 100 # TODO(lukasza): Repeat tests `args.repetitions` times. 101 # TODO(lukasza): Retry failing times up to `args.retry_limit` times. 102