1# Copyright 2013 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Module containing base test results classes.""" 6 7 8import functools 9import re 10import sys 11import threading 12 13from lib.results import result_types 14 15# This must match the source adding the suffix: bit.ly/3Zmwwyx 16MULTIPROCESS_SUFFIX = '__multiprocess_mode' 17 18# This must match the source adding the suffix: bit.ly/3Qt0Ww4 19_NULL_MUTATION_SUFFIX = '__null_' 20_MUTATION_SUFFIX_PATTERN = re.compile(r'^(.*)__([a-zA-Z]+)\.\.([a-zA-Z]+)_$') 21 22 23class ResultType: 24 """Class enumerating test types. 25 26 Wraps the results defined in //build/util/lib/results/. 27 """ 28 PASS = result_types.PASS 29 SKIP = result_types.SKIP 30 FAIL = result_types.FAIL 31 CRASH = result_types.CRASH 32 TIMEOUT = result_types.TIMEOUT 33 UNKNOWN = result_types.UNKNOWN 34 NOTRUN = result_types.NOTRUN 35 36 @staticmethod 37 def GetTypes(): 38 """Get a list of all test types.""" 39 return [ResultType.PASS, ResultType.SKIP, ResultType.FAIL, 40 ResultType.CRASH, ResultType.TIMEOUT, ResultType.UNKNOWN, 41 ResultType.NOTRUN] 42 43 44@functools.total_ordering 45class BaseTestResult: 46 """Base class for a single test result.""" 47 48 def __init__(self, name, test_type, duration=0, log='', failure_reason=None): 49 """Construct a BaseTestResult. 50 51 Args: 52 name: Name of the test which defines uniqueness. 53 test_type: Type of the test result as defined in ResultType. 54 duration: Time it took for the test to run in milliseconds. 55 log: An optional string listing any errors. 56 """ 57 assert name 58 assert test_type in ResultType.GetTypes() 59 self._name = name 60 self._test_type = test_type 61 self._duration = duration 62 self._log = log 63 self._failure_reason = failure_reason 64 self._links = {} 65 self._webview_multiprocess_mode = MULTIPROCESS_SUFFIX in name 66 67 def __str__(self): 68 return self._name 69 70 def __repr__(self): 71 return self._name 72 73 def __eq__(self, other): 74 return self.GetName() == other.GetName() 75 76 def __lt__(self, other): 77 return self.GetName() < other.GetName() 78 79 def __hash__(self): 80 return hash(self._name) 81 82 def SetName(self, name): 83 """Set the test name. 84 85 Because we're putting this into a set, this should only be used if moving 86 this test result into another set. 87 """ 88 self._name = name 89 90 def GetName(self): 91 """Get the test name.""" 92 return self._name 93 94 def GetNameForResultSink(self): 95 """Get the test name to be reported to resultsink.""" 96 raw_name = self.GetName() 97 98 # The name can include suffixes encoding Webview variant data: 99 # a Webview multiprocess mode suffix and an AwSettings mutation suffix. 100 # If both are present, the mutation suffix will come after the multiprocess 101 # suffix. The mutation suffix can either be "__null_" or "__{key}..{value}_" 102 # 103 # Examples: 104 # (...)AwSettingsTest#testAssetUrl__multiprocess_mode__allMutations..true_ 105 # (...)AwSettingsTest#testAssetUrl__multiprocess_mode__null_ 106 # (...)AwSettingsTest#testAssetUrl__allMutations..true_ 107 # org.chromium.android_webview.test.AwSettingsTest#testAssetUrl__null_ 108 109 # first, strip any AwSettings mutation parameter information 110 # from the RHS of the raw_name 111 if raw_name.endswith(_NULL_MUTATION_SUFFIX): 112 raw_name = raw_name[:-len(_NULL_MUTATION_SUFFIX)] 113 elif match := _MUTATION_SUFFIX_PATTERN.search(raw_name): 114 raw_name = match.group(1) 115 116 # At this stage, the name will only have the multiprocess suffix appended, 117 # if applicable. 118 # 119 # Examples: 120 # (...)AwSettingsTest#testAssetUrl__multiprocess_mode 121 # org.chromium.android_webview.test.AwSettingsTest#testAssetUrl 122 123 # then check for multiprocess mode suffix and strip it, if present 124 if self._webview_multiprocess_mode: 125 assert raw_name.endswith( 126 MULTIPROCESS_SUFFIX 127 ), 'multiprocess mode test raw name should have the corresponding suffix' 128 return raw_name[:-len(MULTIPROCESS_SUFFIX)] 129 return raw_name 130 131 def SetType(self, test_type): 132 """Set the test result type.""" 133 assert test_type in ResultType.GetTypes() 134 self._test_type = test_type 135 136 def GetType(self): 137 """Get the test result type.""" 138 return self._test_type 139 140 def GetDuration(self): 141 """Get the test duration.""" 142 return self._duration 143 144 def SetLog(self, log): 145 """Set the test log.""" 146 self._log = log 147 148 def GetLog(self): 149 """Get the test log.""" 150 return self._log 151 152 def SetFailureReason(self, failure_reason): 153 """Set the reason the test failed. 154 155 This should be the first failure the test encounters and exclude any stack 156 trace. 157 """ 158 self._failure_reason = failure_reason 159 160 def GetFailureReason(self): 161 """Get the reason the test failed. 162 163 Returns None if the test did not fail or if the reason the test failed is 164 unknown. 165 """ 166 return self._failure_reason 167 168 def SetLink(self, name, link_url): 169 """Set link with test result data.""" 170 self._links[name] = link_url 171 172 def GetLinks(self): 173 """Get dict containing links to test result data.""" 174 return self._links 175 176 def GetVariantForResultSink(self): 177 """Get the variant dict to be reported to result sink.""" 178 variants = {} 179 if match := _MUTATION_SUFFIX_PATTERN.search(self.GetName()): 180 # variant keys need to be lowercase 181 variants[match.group(2).lower()] = match.group(3) 182 if self._webview_multiprocess_mode: 183 variants['webview_multiprocess_mode'] = 'Yes' 184 return variants or None 185 186 187class TestRunResults: 188 """Set of results for a test run.""" 189 190 def __init__(self): 191 self._links = {} 192 self._results = set() 193 self._results_lock = threading.RLock() 194 195 def SetLink(self, name, link_url): 196 """Add link with test run results data.""" 197 self._links[name] = link_url 198 199 def GetLinks(self): 200 """Get dict containing links to test run result data.""" 201 return self._links 202 203 def GetLogs(self): 204 """Get the string representation of all test logs.""" 205 with self._results_lock: 206 s = [] 207 for test_type in ResultType.GetTypes(): 208 if test_type != ResultType.PASS: 209 for t in sorted(self._GetType(test_type)): 210 log = t.GetLog() 211 if log: 212 s.append('[%s] %s:' % (test_type, t)) 213 s.append(log) 214 if sys.version_info.major == 2: 215 decoded = [u.decode(encoding='utf-8', errors='ignore') for u in s] 216 return '\n'.join(decoded) 217 return '\n'.join(s) 218 219 def GetGtestForm(self): 220 """Get the gtest string representation of this object.""" 221 with self._results_lock: 222 s = [] 223 plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s) 224 tests = lambda n: plural(n, 'test', 'tests') 225 226 s.append('[==========] %s ran.' % (tests(len(self.GetAll())))) 227 s.append('[ PASSED ] %s.' % (tests(len(self.GetPass())))) 228 229 skipped = self.GetSkip() 230 if skipped: 231 s.append('[ SKIPPED ] Skipped %s, listed below:' % tests(len(skipped))) 232 for t in sorted(skipped): 233 s.append('[ SKIPPED ] %s' % str(t)) 234 235 all_failures = self.GetFail().union(self.GetCrash(), self.GetTimeout(), 236 self.GetUnknown()) 237 if all_failures: 238 s.append('[ FAILED ] %s, listed below:' % tests(len(all_failures))) 239 for t in sorted(self.GetFail()): 240 s.append('[ FAILED ] %s' % str(t)) 241 for t in sorted(self.GetCrash()): 242 s.append('[ FAILED ] %s (CRASHED)' % str(t)) 243 for t in sorted(self.GetTimeout()): 244 s.append('[ FAILED ] %s (TIMEOUT)' % str(t)) 245 for t in sorted(self.GetUnknown()): 246 s.append('[ FAILED ] %s (UNKNOWN)' % str(t)) 247 s.append('') 248 s.append(plural(len(all_failures), 'FAILED TEST', 'FAILED TESTS')) 249 return '\n'.join(s) 250 251 def GetShortForm(self): 252 """Get the short string representation of this object.""" 253 with self._results_lock: 254 s = [] 255 s.append('ALL: %d' % len(self._results)) 256 for test_type in ResultType.GetTypes(): 257 s.append('%s: %d' % (test_type, len(self._GetType(test_type)))) 258 return ''.join([x.ljust(15) for x in s]) 259 260 def __str__(self): 261 return self.GetGtestForm() 262 263 def AddResult(self, result): 264 """Add |result| to the set. 265 266 Args: 267 result: An instance of BaseTestResult. 268 """ 269 assert isinstance(result, BaseTestResult) 270 with self._results_lock: 271 self._results.discard(result) 272 self._results.add(result) 273 274 def AddResults(self, results): 275 """Add |results| to the set. 276 277 Args: 278 results: An iterable of BaseTestResult objects. 279 """ 280 with self._results_lock: 281 for t in results: 282 self.AddResult(t) 283 284 def AddTestRunResults(self, results): 285 """Add the set of test results from |results|. 286 287 Args: 288 results: An instance of TestRunResults. 289 """ 290 assert isinstance(results, TestRunResults), ( 291 'Expected TestRunResult object: %s' % type(results)) 292 with self._results_lock: 293 # pylint: disable=W0212 294 self._results.update(results._results) 295 296 def GetAll(self): 297 """Get the set of all test results.""" 298 with self._results_lock: 299 return self._results.copy() 300 301 def _GetType(self, test_type): 302 """Get the set of test results with the given test type.""" 303 with self._results_lock: 304 return set(t for t in self._results if t.GetType() == test_type) 305 306 def GetPass(self): 307 """Get the set of all passed test results.""" 308 return self._GetType(ResultType.PASS) 309 310 def GetSkip(self): 311 """Get the set of all skipped test results.""" 312 return self._GetType(ResultType.SKIP) 313 314 def GetFail(self): 315 """Get the set of all failed test results.""" 316 return self._GetType(ResultType.FAIL) 317 318 def GetCrash(self): 319 """Get the set of all crashed test results.""" 320 return self._GetType(ResultType.CRASH) 321 322 def GetTimeout(self): 323 """Get the set of all timed out test results.""" 324 return self._GetType(ResultType.TIMEOUT) 325 326 def GetUnknown(self): 327 """Get the set of all unknown test results.""" 328 return self._GetType(ResultType.UNKNOWN) 329 330 def GetNotPass(self): 331 """Get the set of all non-passed test results.""" 332 return self.GetAll() - self.GetPass() 333 334 def DidRunPass(self): 335 """Return whether the test run was successful.""" 336 return not self.GetNotPass() - self.GetSkip() 337