1# Copyright 2021 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""This is a library for handling of --isolated-script-test-filter and 5--isolated-script-test-filter-file cmdline arguments, as specified by 6//docs/testing/test_executable_api.md 7 8Typical usage: 9 import argparse 10 import test_filtering 11 12 cmdline_parser = argparse.ArgumentParser() 13 test_filtering.add_cmdline_args(cmdline_parser) 14 ... adding other cmdline parameter definitions ... 15 parsed_cmdline_args = cmdline_parser.parse_args() 16 17 list_of_all_test_names = ... queried from the wrapped test executable ... 18 list_of_test_names_to_run = test_filtering.filter_tests( 19 parsed_cmdline_args, list_of_all_test_names) 20""" 21 22import argparse 23import os 24import re 25import sys 26 27 28class _TestFilter: 29 """_TestFilter represents a single test filter pattern like foo (including 30 'foo' test in the test run), bar* (including all tests with a name starting 31 with 'bar'), or -baz (excluding 'baz' test from the test run). 32 """ 33 34 def __init__(self, filter_text): 35 assert '::' not in filter_text 36 if '*' in filter_text[:-1]: 37 raise ValueError('* is only allowed at the end (as documented ' \ 38 'in //docs/testing/test_executable_api.md).') 39 40 if filter_text.startswith('-'): 41 self._is_exclusion_filter = True 42 filter_text = filter_text[1:] 43 else: 44 self._is_exclusion_filter = False 45 46 if filter_text.endswith('*'): 47 self._is_prefix_match = True 48 filter_text = filter_text[:-1] 49 else: 50 self._is_prefix_match = False 51 52 self._filter_text = filter_text 53 54 def is_match(self, test_name): 55 """Returns whether the test filter should apply to `test_name`. 56 """ 57 if self._is_prefix_match: 58 return test_name.startswith(self._filter_text) 59 return test_name == self._filter_text 60 61 def is_exclusion_filter(self): 62 """Rreturns whether this filter excludes (rather than includes) matching 63 test names. 64 """ 65 return self._is_exclusion_filter 66 67 def get_specificity_key(self): 68 """Returns a key that can be used to sort the TestFilter objects by 69 their specificity. From //docs/testing/test_executable_api.md: 70 If multiple filters [...] match a given test name, the longest match 71 takes priority (longest match wins). [...] It is an error to have 72 multiple expressions of the same length that conflict (e.g., a*::-a*). 73 """ 74 return (len(self._filter_text), self._filter_text) 75 76 def __str__(self): 77 result = self._filter_text 78 if self._is_exclusion_filter: 79 result = "-" + result 80 if self._is_prefix_match: 81 result += "*" 82 return result 83 84 85class _TestFiltersGroup: 86 """_TestFiltersGroup represents an individual group of test filters 87 (corresponding to a single --isolated-script-test-filter or 88 --isolated-script-test-filter-file cmdline argument). 89 """ 90 91 def __init__(self, list_of_test_filters): 92 """Internal implementation detail - please use from_string and/or 93 from_filter_file static methods instead.""" 94 self._list_of_test_filters = sorted( 95 list_of_test_filters, 96 key=lambda x: x.get_specificity_key(), 97 reverse=True) 98 99 if all(f.is_exclusion_filter() for f in self._list_of_test_filters): 100 self._list_of_test_filters.append(_TestFilter('*')) 101 assert len(list_of_test_filters) 102 103 for i in range(len(self._list_of_test_filters) - 1): 104 prev = self._list_of_test_filters[i] 105 curr = self._list_of_test_filters[i + 1] 106 if prev.get_specificity_key() == curr.get_specificity_key(): 107 raise ValueError( 108 'It is an error to have multiple test filters of the ' \ 109 'same length that conflict (e.g., a*::-a*). Conflicting ' \ 110 'filters: {} and {}'.format(prev, curr)) 111 112 @staticmethod 113 def from_string(cmdline_arg): 114 """Constructs a _TestFiltersGroup from a string that follows the format 115 of --isolated-script-test-filter cmdline argument as described in 116 Chromium's //docs/testing/test_executable_api.md 117 """ 118 list_of_test_filters = [] 119 for filter_text in cmdline_arg.split('::'): 120 list_of_test_filters.append(_TestFilter(filter_text)) 121 return _TestFiltersGroup(list_of_test_filters) 122 123 @staticmethod 124 def from_filter_file(filepath): 125 """Constructs a _TestFiltersGroup from an input file that can be passed 126 to the --isolated-script-test-filter-file cmdline argument as described 127 Chromium's //docs/testing/test_executable_api.md. The file format is 128 described in bit.ly/chromium-test-list-format (aka go/test-list-format). 129 """ 130 list_of_test_filters = [] 131 regex = r' \[ [^]]* \]' # [ foo ] 132 regex += r'| Bug \( [^)]* \)' # Bug(12345) 133 regex += r'| crbug.com/\S*' # crbug.com/12345 134 regex += r'| skbug.com/\S*' # skbug.com/12345 135 regex += r'| webkit.org/\S*' # webkit.org/12345 136 compiled_regex = re.compile(regex, re.VERBOSE) 137 with open(filepath, mode='r', encoding='utf-8') as f: 138 for line in f.readlines(): 139 filter_text = line.split('#')[0] 140 filter_text = compiled_regex.sub('', filter_text) 141 filter_text = filter_text.strip() 142 if filter_text: 143 list_of_test_filters.append(_TestFilter(filter_text)) 144 return _TestFiltersGroup(list_of_test_filters) 145 146 def is_test_name_included(self, test_name): 147 for test_filter in self._list_of_test_filters: 148 if test_filter.is_match(test_name): 149 return not test_filter.is_exclusion_filter() 150 return False 151 152 153class _SetOfTestFiltersGroups: 154 def __init__(self, list_of_test_filter_groups): 155 """Constructs _SetOfTestFiltersGroups from `list_of_test_filter_groups`. 156 157 Args: 158 list_of_test_filter_groups: A list of _TestFiltersGroup objects. 159 """ 160 self._test_filters_groups = list_of_test_filter_groups 161 162 def filter_test_names(self, list_of_test_names): 163 return [ 164 t for t in list_of_test_names if self._is_test_name_included(t) 165 ] 166 167 def _is_test_name_included(self, test_name): 168 for test_filters_group in self._test_filters_groups: 169 if not test_filters_group.is_test_name_included(test_name): 170 return False 171 return True 172 173 174def _shard_tests(list_of_test_names, env): 175 # Defaulting to 0 for `shard_index` and to 1 for `total_shards`. 176 shard_index = int(env.get('GTEST_SHARD_INDEX', 0)) 177 total_shards = int(env.get('GTEST_TOTAL_SHARDS', 1)) 178 assert shard_index < total_shards 179 180 result = [] 181 for i in range(len(list_of_test_names)): 182 if (i % total_shards) == shard_index: 183 result.append(list_of_test_names[i]) 184 185 return result 186 187 188def _filter_test_names(list_of_test_names, argparse_parsed_args): 189 inline_filter_groups = [ 190 _TestFiltersGroup.from_string(s) 191 for s in argparse_parsed_args.test_filters 192 ] 193 filter_file_groups = [ 194 _TestFiltersGroup.from_filter_file(f) 195 for f in argparse_parsed_args.test_filter_files 196 ] 197 set_of_filter_groups = _SetOfTestFiltersGroups(inline_filter_groups + 198 filter_file_groups) 199 return set_of_filter_groups.filter_test_names(list_of_test_names) 200 201 202def add_cmdline_args(argparse_parser): 203 """Adds test-filtering-specific cmdline parameter definitions to 204 `argparse_parser`. 205 206 Args: 207 argparse_parser: An object of argparse.ArgumentParser type. 208 """ 209 filter_help = 'A double-colon-separated list of strings, where each ' \ 210 'string either uniquely identifies a full test name or is ' \ 211 'a prefix plus a "*" on the end (to form a glob). If the ' \ 212 'string has a "-" at the front, the test (or glob of ' \ 213 'tests) will be skipped, not run.' 214 argparse_parser.add_argument('--test-filter', 215 '--isolated-script-test-filter', 216 action='append', 217 default=[], 218 dest='test_filters', 219 help=filter_help, 220 metavar='TEST-NAME-PATTERNS') 221 file_help = 'Path to a file with test filters in Chromium Test List ' \ 222 'Format. See also //testing/buildbot/filters/README.md and ' \ 223 'bit.ly/chromium-test-list-format' 224 argparse_parser.add_argument('--test-filter-file', 225 '--isolated-script-test-filter-file', 226 action='append', 227 default=[], 228 dest='test_filter_files', 229 help=file_help, 230 metavar='FILEPATH') 231 232 233def filter_tests(argparse_parsed_args, env, list_of_test_names): 234 """Filters `list_of_test_names` as requested by the cmdline arguments 235 and sharding-related environment variables. 236 237 Args: 238 argparse_parsed_arg: A result of an earlier call to 239 argparse_parser.parse_args() call (where `argparse_parser` has been 240 populated via an even earlier call to add_cmdline_args). 241 env: a dictionary-like object (typically from `os.environ`). 242 list_of_test_name: A list of strings (a list of test names). 243 """ 244 filtered_names = _filter_test_names(list_of_test_names, 245 argparse_parsed_args) 246 sharded_names = _shard_tests(filtered_names, env) 247 return sharded_names 248