1#!/usr/bin/env vpython3 2# Copyright 2022 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Implements commands for standalone CFv2 test executables.""" 6 7import argparse 8import logging 9import os 10import shutil 11import subprocess 12import sys 13 14from typing import List, Optional 15 16from common import get_component_uri, get_host_arch, \ 17 register_common_args, register_device_args, \ 18 register_log_args 19from compatible_utils import map_filter_file_to_package_file 20from ffx_integration import FfxTestRunner, run_symbolizer 21from test_runner import TestRunner 22 23DEFAULT_TEST_SERVER_CONCURRENCY = 4 24 25 26def _copy_custom_output_file(test_runner: FfxTestRunner, file: str, 27 dest: str) -> None: 28 """Copy custom test output file from the device to the host.""" 29 30 artifact_dir = test_runner.get_custom_artifact_directory() 31 if not artifact_dir: 32 logging.error( 33 'Failed to parse custom artifact directory from test summary ' 34 'output files. Not copying %s from the device', file) 35 return 36 shutil.copy(os.path.join(artifact_dir, file), dest) 37 38 39def _copy_coverage_files(test_runner: FfxTestRunner, dest: str) -> None: 40 """Copy debug data file from the device to the host if it exists.""" 41 42 coverage_dir = test_runner.get_debug_data_directory() 43 if not coverage_dir: 44 logging.info( 45 'Failed to parse coverage data directory from test summary ' 46 'output files. Not copying coverage files from the device.') 47 return 48 shutil.copytree(coverage_dir, dest, dirs_exist_ok=True) 49 50 51def _get_vulkan_args(use_vulkan: Optional[str]) -> List[str]: 52 """Helper function to set vulkan related flag.""" 53 54 vulkan_args = [] 55 if not use_vulkan: 56 if get_host_arch() == 'x64': 57 # TODO(crbug.com/1261646) Remove once Vulkan is enabled by 58 # default. 59 use_vulkan = 'native' 60 else: 61 # Use swiftshader on arm64 by default because most arm64 bots 62 # currently don't support Vulkan emulation. 63 use_vulkan = 'swiftshader' 64 vulkan_args.append('--ozone-platform=headless') 65 vulkan_args.append(f'--use-vulkan={use_vulkan}') 66 return vulkan_args 67 68 69class ExecutableTestRunner(TestRunner): 70 """Test runner for running standalone test executables.""" 71 72 def __init__( # pylint: disable=too-many-arguments 73 self, out_dir: str, test_args: List[str], test_name: str, 74 target_id: Optional[str], code_coverage_dir: str, 75 logs_dir: Optional[str], package_deps: List[str], 76 test_realm: Optional[str]) -> None: 77 super().__init__(out_dir, test_args, [test_name], target_id, 78 package_deps) 79 if not self._test_args: 80 self._test_args = [] 81 self._test_name = test_name 82 self._code_coverage_dir = code_coverage_dir 83 self._custom_artifact_directory = None 84 self._isolated_script_test_output = None 85 self._isolated_script_test_perf_output = None 86 self._logs_dir = logs_dir 87 self._test_launcher_summary_output = None 88 self._test_server = None 89 self._test_realm = test_realm 90 91 def _get_args(self) -> List[str]: 92 parser = argparse.ArgumentParser() 93 parser.add_argument( 94 '--isolated-script-test-output', 95 help='If present, store test results on this path.') 96 parser.add_argument('--isolated-script-test-perf-output', 97 help='If present, store chartjson results on this ' 98 'path.') 99 parser.add_argument( 100 '--test-launcher-shard-index', 101 type=int, 102 default=os.environ.get('GTEST_SHARD_INDEX'), 103 help='Index of this instance amongst swarming shards.') 104 parser.add_argument( 105 '--test-launcher-summary-output', 106 help='Where the test launcher will output its json.') 107 parser.add_argument( 108 '--test-launcher-total-shards', 109 type=int, 110 default=os.environ.get('GTEST_TOTAL_SHARDS'), 111 help='Total number of swarming shards of this suite.') 112 parser.add_argument( 113 '--test-launcher-filter-file', 114 help='Filter file(s) passed to target test process. Use ";" to ' 115 'separate multiple filter files.') 116 parser.add_argument('--test-launcher-jobs', 117 type=int, 118 help='Sets the number of parallel test jobs.') 119 parser.add_argument('--enable-test-server', 120 action='store_true', 121 default=False, 122 help='Enable Chrome test server spawner.') 123 parser.add_argument('--test-arg', 124 dest='test_args', 125 action='append', 126 help='Legacy flag to pass in arguments for ' 127 'the test process. These arguments can now be ' 128 'passed in without a preceding "--" flag.') 129 parser.add_argument('--use-vulkan', 130 help='\'native\', \'swiftshader\' or \'none\'.') 131 args, child_args = parser.parse_known_args(self._test_args) 132 if args.isolated_script_test_output: 133 self._isolated_script_test_output = args.isolated_script_test_output 134 child_args.append( 135 '--isolated-script-test-output=/custom_artifacts/%s' % 136 os.path.basename(self._isolated_script_test_output)) 137 if args.isolated_script_test_perf_output: 138 self._isolated_script_test_perf_output = \ 139 args.isolated_script_test_perf_output 140 child_args.append( 141 '--isolated-script-test-perf-output=/custom_artifacts/%s' % 142 os.path.basename(self._isolated_script_test_perf_output)) 143 if args.test_launcher_shard_index is not None: 144 child_args.append('--test-launcher-shard-index=%d' % 145 args.test_launcher_shard_index) 146 if args.test_launcher_total_shards is not None: 147 child_args.append('--test-launcher-total-shards=%d' % 148 args.test_launcher_total_shards) 149 if args.test_launcher_summary_output: 150 self._test_launcher_summary_output = \ 151 args.test_launcher_summary_output 152 child_args.append( 153 '--test-launcher-summary-output=/custom_artifacts/%s' % 154 os.path.basename(self._test_launcher_summary_output)) 155 if args.test_launcher_filter_file: 156 test_launcher_filter_files = map( 157 map_filter_file_to_package_file, 158 args.test_launcher_filter_file.split(';')) 159 child_args.append('--test-launcher-filter-file=' + 160 ';'.join(test_launcher_filter_files)) 161 if args.test_launcher_jobs is not None: 162 test_concurrency = args.test_launcher_jobs 163 else: 164 test_concurrency = DEFAULT_TEST_SERVER_CONCURRENCY 165 if args.enable_test_server: 166 # Repos other than chromium may not have chrome_test_server_spawner, 167 # and they may not run server at all, so only import the test_server 168 # when it's really necessary. 169 170 # pylint: disable=import-outside-toplevel 171 from test_server import setup_test_server 172 # pylint: enable=import-outside-toplevel 173 self._test_server, spawner_url_base = setup_test_server( 174 self._target_id, test_concurrency) 175 child_args.append('--remote-test-server-spawner-url-base=%s' % 176 spawner_url_base) 177 child_args.extend(_get_vulkan_args(args.use_vulkan)) 178 if args.test_args: 179 child_args.extend(args.test_args) 180 return child_args 181 182 def _postprocess(self, test_runner: FfxTestRunner) -> None: 183 if self._test_server: 184 self._test_server.Stop() 185 if self._test_launcher_summary_output: 186 _copy_custom_output_file( 187 test_runner, 188 os.path.basename(self._test_launcher_summary_output), 189 self._test_launcher_summary_output) 190 if self._isolated_script_test_output: 191 _copy_custom_output_file( 192 test_runner, 193 os.path.basename(self._isolated_script_test_output), 194 self._isolated_script_test_output) 195 if self._isolated_script_test_perf_output: 196 _copy_custom_output_file( 197 test_runner, 198 os.path.basename(self._isolated_script_test_perf_output), 199 self._isolated_script_test_perf_output) 200 if self._code_coverage_dir: 201 _copy_coverage_files(test_runner, 202 os.path.basename(self._code_coverage_dir)) 203 204 def run_test(self) -> subprocess.Popen: 205 test_args = self._get_args() 206 with FfxTestRunner(self._logs_dir) as test_runner: 207 test_proc = test_runner.run_test( 208 get_component_uri(self._test_name), test_args, self._target_id, 209 self._test_realm) 210 211 symbol_paths = [] 212 for pkg_path in self.package_deps.values(): 213 symbol_paths.append( 214 os.path.join(os.path.dirname(pkg_path), 'ids.txt')) 215 # Symbolize output from test process and print to terminal. 216 symbolizer_proc = run_symbolizer(symbol_paths, test_proc.stdout, 217 sys.stdout) 218 symbolizer_proc.communicate() 219 220 if test_proc.wait() == 0: 221 logging.info('Process exited normally with status code 0.') 222 else: 223 # The test runner returns an error status code if *any* 224 # tests fail, so we should proceed anyway. 225 logging.warning('Process exited with status code %d.', 226 test_proc.returncode) 227 self._postprocess(test_runner) 228 return test_proc 229 230 231def create_executable_test_runner(runner_args: argparse.Namespace, 232 test_args: List[str]): 233 """Helper for creating an ExecutableTestRunner.""" 234 235 return ExecutableTestRunner(runner_args.out_dir, test_args, 236 runner_args.test_type, runner_args.target_id, 237 runner_args.code_coverage_dir, 238 runner_args.logs_dir, runner_args.package_deps, 239 runner_args.test_realm) 240 241 242def register_executable_test_args(parser: argparse.ArgumentParser) -> None: 243 """Register common arguments for ExecutableTestRunner.""" 244 245 test_args = parser.add_argument_group('test', 'arguments for test running') 246 test_args.add_argument('--code-coverage-dir', 247 default=None, 248 help='Directory to place code coverage ' 249 'information. Only relevant when the target was ' 250 'built with |fuchsia_code_coverage| set to true.') 251 test_args.add_argument('--test-name', 252 dest='test_type', 253 help='Name of the test package (e.g. ' 254 'unit_tests).') 255 test_args.add_argument( 256 '--test-realm', 257 default=None, 258 help='The realm to run the test in. This field is optional and takes ' 259 'the form: /path/to/realm:test_collection. See ' 260 'https://fuchsia.dev/go/components/non-hermetic-tests') 261 test_args.add_argument('--package-deps', 262 action='append', 263 help='A list of the full path of the dependencies ' 264 'to retrieve the symbol ids. Keeping it empty to ' 265 'automatically generates from package_metadata.') 266 267 268def main(): 269 """Stand-alone function for running executable tests.""" 270 271 parser = argparse.ArgumentParser() 272 register_common_args(parser) 273 register_device_args(parser) 274 register_log_args(parser) 275 register_executable_test_args(parser) 276 runner_args, test_args = parser.parse_known_args() 277 runner = create_executable_test_runner(runner_args, test_args) 278 return runner.run_test().returncode 279 280 281if __name__ == '__main__': 282 sys.exit(main()) 283