1# Lint as: python3 2# Copyright 2014 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import logging 7import os 8import re 9import shutil 10from autotest_lib.client.bin import utils 11from autotest_lib.client.common_lib import error 12from autotest_lib.client.cros import cros_logging, service_stopper 13from autotest_lib.client.cros.graphics import graphics_utils 14 15 16class graphics_parallel_dEQP(graphics_utils.GraphicsTest): 17 """Run the drawElements Quality Program test suite.""" 18 version = 1 19 _services = None 20 _shard_number = 0 21 _shard_count = 1 22 _board = None 23 _cpu_type = None 24 _gpu_type = None 25 _surface = None 26 _filter = None 27 _width = 256 # Use smallest width for which all tests run/pass. 28 _height = 256 # Use smallest height for which all tests7 run/pass. 29 _caselist = None 30 _log_path = None # Location for detailed test output logs 31 _debug = False # Analyze kernel messages. 32 _log_reader = None # Reader to analyze (kernel) messages log. 33 _log_filter = re.compile('.* .* kernel:') # kernel messages filter. 34 _env = None # environment for test processes 35 _skips = [] 36 _fails = [] 37 _flakes = [] 38 _api_helper = None 39 # We do not consider these results as failures. 40 TEST_RESULT_FILTER = [ 41 'pass', 'notsupported', 'internalerror', 'qualitywarning', 42 'compatibilitywarning', 'skipped' 43 ] 44 _expectations_dir = '/usr/local/graphics/expectations/deqp/' 45 46 def initialize(self): 47 """Initialize the test.""" 48 super().initialize() 49 self._api_helper = graphics_utils.GraphicsApiHelper() 50 self._board = utils.get_board() 51 self._cpu_type = utils.get_cpu_soc_family() 52 self._gpu_type = utils.get_gpu_family() 53 54 # deqp may depend on libraries that are present only on test images. 55 # Those libraries are installed in /usr/local. 56 self._env = os.environ.copy() 57 old_ld_path = self._env.get('LD_LIBRARY_PATH', '') 58 if old_ld_path: 59 self._env[ 60 'LD_LIBRARY_PATH'] = '/usr/local/lib:/usr/local/lib64:' + old_ld_path 61 else: 62 self._env['LD_LIBRARY_PATH'] = '/usr/local/lib:/usr/local/lib64' 63 64 self._services = service_stopper.ServiceStopper(['ui', 'powerd']) 65 # Valid choices are fbo and pbuffer. The latter avoids dEQP assumptions. 66 self._surface = 'pbuffer' 67 self._services.stop_services() 68 69 def cleanup(self): 70 """Clean up the test state from initialize().""" 71 if self._services: 72 self._services.restore_services() 73 super().cleanup() 74 75 def _get_executable(self, api): 76 """Return the executable path of the api.""" 77 return self._api_helper.get_deqp_executable(api) 78 79 def _can_run(self, api): 80 """Check if specific api is supported in this board.""" 81 return api in self._api_helper.get_supported_apis() 82 83 def read_file(self, filename): 84 """Board/GPU expectation file read helper.""" 85 expects_path = os.path.join(self._expectations_dir, filename) 86 try: 87 with open(expects_path, encoding='utf-8') as file: 88 logging.debug(f'Reading board test list from {expects_path}') 89 return file.readlines() 90 except IOError as _: 91 logging.debug('No file found at %s', format(expects_path)) 92 return [] 93 94 def read_expectations(self, name): 95 """Appends the skips, fails and flakes files if they exist.""" 96 self._skips += self.read_file(name + '-skips.txt') 97 self._fails += self.read_file(name + '-fails.txt') 98 self._flakes += self.read_file(name + '-flakes.txt') 99 100 def setup_case_list_filters(self): 101 """Set up the skip/flake/fails filter lists. 102 103 The expected fails list will be entries like 104 'dEQP-SUITE.test.name,Crash', such as you find in a failures.csv, 105 results.csv, or the "Some failures found:" stdout output of a previous 106 run. Enter a test here when it has an expected state other than Pass or 107 Skip. 108 109 The skips list is a list of regexs to match test names to not run at 110 all. This is good for tests that are too slow or uninteresting to ever 111 see status for. 112 113 The flakes list is a list of regexes to match test names that may have 114 unreliable status. Any unexpected result of that test will be marked 115 with the Flake status and not cause the test run to fail. The runner 116 does automatic flake detection on its own to try to mitigate 117 intermittent failures, but even with that we can see too many spurious 118 failures in CI when run across many boards and many builds, so this lets 119 you run those tests while avoiding having them fail out CI runs until 120 the source of the flakiness can be resolved. 121 122 The primary source of board skip/flake/fails will be files in this test 123 directory under boards/, but we also list some common entries directly 124 in the code here to save repetition of the explanations. The files may 125 contain empty lines or comments starting with '#'. 126 127 We could avoid adding filters for other apis than the one being tested, 128 but it's harmless to have unused tests in the lists and makes 129 copy-and-paste mistakes less likely. 130 """ 131 # Add expectations common for all boards/chipsets. 132 self.read_expectations('all-chipsets') 133 134 # Add any chipset specific expectations. Most issues should be here. 135 self.read_expectations(self._gpu_type) 136 137 # Add any board-specific expectations. Lets hope we never need models. 138 self.read_expectations(self._board) 139 140 def add_filter_arg(self, command, tests, arg, filename): 141 """Adds an arg for xfail/skip/flake filtering if we made the file for it.""" 142 if not tests: 143 return 144 145 path = os.path.join(self._log_path, filename) 146 with open(path, 'w', encoding='utf-8') as file: 147 for test in tests: 148 file.write(test + '\n') 149 command.append(arg + '=' + path) 150 151 def run_once(self, opts=None): 152 """Invokes deqp-runner to run a deqp test group.""" 153 options = dict( 154 api=None, 155 caselist=None, 156 filter='', 157 subset_to_run='Pass', # Pass, Fail, Timeout, NotPass... 158 shard_number='0', 159 shard_count='1', 160 debug='False', 161 perf_failure_description=None) 162 if opts is None: 163 opts = [] 164 options.update(utils.args_to_dict(opts)) 165 logging.info('Test Options: %s', options) 166 167 self._caselist = options['caselist'] 168 self._shard_number = int(options['shard_number']) 169 self._shard_count = int(options['shard_count']) 170 self._debug = (options['debug'] == 'True') 171 172 api = options['api'] 173 174 if not self._can_run(api): 175 logging.info('Skipping on %s due to lack of %s API support', 176 self._gpu_type, api) 177 return 178 179 # Some information to help post-process logs. 180 logging.info('ChromeOS BOARD = %s', self._board) 181 logging.info('ChromeOS CPU family = %s', self._cpu_type) 182 logging.info('ChromeOS GPU family = %s', self._gpu_type) 183 184 self.setup_case_list_filters() 185 186 # Create a place to put detailed test output logs. 187 filter_name = self._filter or os.path.basename(self._caselist) 188 logging.info('dEQP test filter = %s', filter_name) 189 self._log_path = os.path.join(os.getcwd(), 'deqp-runner') 190 shutil.rmtree(self._log_path, ignore_errors=True) 191 os.mkdir(self._log_path) 192 193 if self._debug: 194 # LogReader works on /var/log/messages by default. 195 self._log_reader = cros_logging.LogReader() 196 self._log_reader.set_start_by_current() 197 198 executable = self._get_executable(api) 199 # Must be in the executable directory when running for it to find it's 200 # test data files! 201 os.chdir(os.path.dirname(executable)) 202 203 command = ['deqp-runner', 'run'] 204 command.append(f'--output={self._log_path}') 205 command.append(f'--deqp={executable}') 206 command.append('--testlog-to-xml=%s' % os.path.join( 207 self._api_helper.get_deqp_dir(), 'executor', 'testlog-to-xml')) 208 command.append(f'--caselist={self._caselist}') 209 if self._shard_number != 0: 210 command.append(f'--fraction-start={self._shard_number + 1}') 211 if self._shard_count != 1: 212 command.append(f'--fraction={self._shard_count}') 213 214 self.add_filter_arg(command, self._flakes, '--flakes', 215 'known_flakes.txt') 216 self.add_filter_arg(command, self._skips, '--skips', 'skips.txt') 217 self.add_filter_arg(command, self._fails, '--baseline', 218 'expected-fails.txt') 219 220 command.append('--') 221 command.append(f'--deqp-surface-type={self._surface}') 222 command.append(f'--deqp-surface-width={self._width}') 223 command.append(f'--deqp-surface-height={self._height}') 224 command.append('--deqp-gl-config-name=rgba8888d24s8ms0') 225 226 # Must initialize because some errors don't repopulate 227 # run_result, leaving old results. 228 run_result = {} 229 try: 230 logging.info(command) 231 run_result = utils.run( 232 command, 233 env=self._env, 234 ignore_status=True, 235 stdout_tee=utils.TEE_TO_LOGS, 236 stdout_level=logging.INFO, 237 stderr_tee=utils.TEE_TO_LOGS) 238 except error.CmdError: 239 raise error.TestFail("Failed starting '%s'" % command) 240 241 # Update failing tests to the chrome perf dashboard records. 242 fails = [] 243 try: 244 with open( 245 os.path.join(self._log_path, 'failures.csv'), 246 encoding='utf-8') as fails_file: 247 for line in fails_file.readlines(): 248 fails.append(line) 249 self.add_failures(line) 250 except IOError: 251 # failures.csv not created if there were were no failures 252 pass 253 254 include_css = False 255 for path in os.listdir(self._log_path): 256 path = os.path.join(self._log_path, path) 257 # Remove the large (~15Mb) temporary .shader_cache files generated by the dEQP runs 258 # so we don't upload them with the logs to stainless. 259 if path.endswith('.shader_cache') and os.path.isfile(path): 260 os.remove(os.path.join(self._log_path, path)) 261 262 if path.endswith('.xml'): 263 include_css = True 264 265 # If we have any QPA XML files, then we'll want to include the CSS 266 # in the logs so you can view them. 267 if include_css: 268 stylesheet = os.path.join(self._api_helper.get_deqp_dir(), 269 'testlog-stylesheet') 270 for file in ['testlog.css', 'testlog.xsl']: 271 shutil.copy(os.path.join(stylesheet, file), self._log_path) 272 273 if fails: 274 if len(fails) == 1: 275 raise error.TestFail(f'Failed test: {fails[0]}') 276 # We format the failure message so it is not too long and reasonably 277 # stable even if there are a few flaky tests to simplify triaging 278 # on stainless and testmon. We sort the failing tests and report 279 # first and last failure. 280 fails.sort() 281 fail_msg = f'Failed {len(fails)} tests: ' 282 fail_msg += fails[0].rstrip() + ', ..., ' + fails[-1].rstrip() 283 fail_msg += ' (see failures.csv)' 284 raise error.TestFail(fail_msg) 285 if run_result.exit_status != 0: 286 raise error.TestFail( 287 f'dEQP run failed with status code {run_result.exit_status}') 288