1# Lint as: python2, python3 2# Copyright 2022 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5""" 6This class provides base wrapper functions for Bluetooth quick test 7""" 8 9import functools 10import logging 11 12from autotest_lib.client.common_lib import error 13 14 15class BluetoothQuickTestsBase(object): 16 """Provides base helper functions for Bluetooth quick test batches/packages. 17 18 The Bluetooth quick test infrastructure provides a way to quickly run a set 19 of tests. As for today, auto-test ramp up time per test is about 90-120 20 seconds, where a typical Bluetooth test may take ~30-60 seconds to run. 21 22 The quick test infra, implemented in this class, saves this huge overhead 23 by running only the minimal reset and cleanup operations required between 24 each set of tests (takes a few seconds). 25 26 This class provides wrapper functions to start and end a test, a batch or a 27 package. A batch is defined as a set of tests, preferably with a common 28 subject. A package is a set of batches. 29 This class takes care of tests, batches, and packages test results, and 30 prints out summaries to results. The class also resets and cleans up 31 required states between tests, batches and packages. 32 33 A batch can also run as a separate auto-test. There is a place holder to 34 add a way to run a specific test of a batch autonomously. 35 36 A batch can be implemented by inheriting from this class, and using its 37 wrapper functions. A package can be implemented by inheriting from a set of 38 batches. 39 40 Adding a test to one of the batches is as easy as adding a method to the 41 class of the batch. 42 """ 43 44 # Some delay is needed between tests. TODO(yshavit): investigate and remove 45 TEST_SLEEP_SECS = 3 46 47 def _print_delimiter(self): 48 logging.info('=======================================================') 49 50 def quick_test_init(self, flag='Quick Health'): 51 """Inits the quick test.""" 52 53 self.flag = flag 54 self.test_iter = None 55 56 self.bat_tests_results = [] 57 self.bat_pass_count = 0 58 self.bat_fail_count = 0 59 self.bat_testna_count = 0 60 self.bat_warn_count = 0 61 self.bat_name = None 62 self.bat_iter = None 63 64 self.pkg_tests_results = [] 65 self.pkg_pass_count = 0 66 self.pkg_fail_count = 0 67 self.pkg_testna_count = 0 68 self.pkg_warn_count = 0 69 self.pkg_name = None 70 self.pkg_iter = None 71 self.pkg_is_running = False 72 73 def quick_test_get_model_name(self): 74 """This method should be implemented by children classes. 75 76 The ways to get the model names are different between server and client 77 sides. The derived class should provide the method to get the info. 78 """ 79 raise NotImplementedError 80 81 def quick_test_get_chipset_name(self): 82 """This method should be implemented by children classes. 83 84 The ways to get the chipset names are different between server and 85 client sides. The derived class should provide the method to get the 86 info. 87 """ 88 raise NotImplementedError 89 90 @staticmethod 91 def quick_test_test_decorator(test_name, 92 flags=None, 93 check_runnable_func=None, 94 pretest_func=None, 95 posttest_func=None, 96 model_testNA=None, 97 model_testWarn=None, 98 skip_models=None, 99 skip_chipsets=None, 100 skip_common_errors=False): 101 """A decorator providing a wrapper to a quick test. 102 103 Using the decorator a test method can implement only the core 104 test and let the decorator handle the quick test wrapper methods 105 (reset/cleanup/logging). 106 107 @param test_name: The name of the test to log. 108 @param flags: List of string to describe who should run the test. The 109 string could be one of the following: 110 ['AVL', 'Quick Health', 'All']. 111 @check_runnable_func: A function that accepts a bluetooth quick test 112 instance as argument. If not None and returns 113 False, the test exits early without failure. 114 @pretest_func: A function that accepts a bluetooth quick test instance 115 as argument. If not None, the function is run right 116 before the test method. 117 @posttest_func: A function that accepts a bluetooth quick test instance 118 as argument. If not None, the function is run after the 119 test summary is logged. 120 Note that the exception raised from this function is NOT 121 caught by the decorator. 122 @param model_testNA: If the current platform is in this list, failures 123 are emitted as TestNAError. 124 @param model_testWarn: If the current platform is in this list, failures 125 are emitted as TestWarn. 126 @param skip_models: Raises TestNA on these models and doesn't attempt to 127 run the tests. 128 @param skip_chipsets: Raises TestNA on these chipset and doesn't attempt 129 to run the tests. 130 @param skip_common_errors: If the test encounters a common error (such 131 as USB disconnect or daemon crash), mark the 132 test as TESTNA instead. USE THIS SPARINGLY, 133 it may mask bugs. This is available for tests 134 that require state to be properly retained 135 throughout the whole test (i.e. advertising) 136 and any outside failure will cause the test 137 to fail. 138 """ 139 140 if flags is None: 141 flags = ['All'] 142 if model_testNA is None: 143 model_testNA = [] 144 if model_testWarn is None: 145 model_testWarn = [] 146 if skip_models is None: 147 skip_models = [] 148 if skip_chipsets is None: 149 skip_chipsets = [] 150 151 def decorator(test_method): 152 """A decorator wrapper of the decorated test_method. 153 154 @param test_method: The test method being decorated. 155 156 @return: The wrapper of the test method. 157 """ 158 159 @functools.wraps(test_method) 160 def wrapper(self): 161 """A wrapper of the decorated method.""" 162 163 # Set test name before exiting so batches correctly identify 164 # failing tests 165 self.test_name = test_name 166 167 # Reset failure info before running any check, so 168 # quick_test_test_log_results() can judge the result correctly. 169 self.fails = [] 170 self.had_known_common_failure = False 171 172 # Check that the test is runnable in current setting 173 if not (self.flag in flags or 'All' in flags): 174 logging.info('SKIPPING TEST %s', test_name) 175 logging.info('flag %s not in %s', self.flag, flags) 176 self._print_delimiter() 177 return 178 179 if check_runnable_func and not check_runnable_func(self): 180 return 181 182 try: 183 model = self.quick_test_get_model_name() 184 if model in skip_models: 185 logging.info('SKIPPING TEST %s', test_name) 186 raise error.TestNAError( 187 'Test not supported on this model') 188 189 chipset = self.quick_test_get_chipset_name() 190 logging.debug('Bluetooth module name is %s', chipset) 191 if chipset in skip_chipsets: 192 logging.info('SKIPPING TEST %s on chipset %s', 193 test_name, chipset) 194 raise error.TestNAError( 195 'Test not supported on this chipset') 196 197 if pretest_func: 198 pretest_func(self) 199 200 self._print_delimiter() 201 logging.info('Starting test: %s', test_name) 202 203 test_method(self) 204 except error.TestError as e: 205 fail_msg = '[--- error {} ({})]'.format( 206 test_method.__name__, str(e)) 207 logging.error(fail_msg) 208 self.fails.append(fail_msg) 209 except error.TestFail as e: 210 fail_msg = '[--- failed {} ({})]'.format( 211 test_method.__name__, str(e)) 212 logging.error(fail_msg) 213 self.fails.append(fail_msg) 214 except error.TestNAError as e: 215 fail_msg = '[--- SKIPPED {} ({})]'.format( 216 test_method.__name__, str(e)) 217 logging.error(fail_msg) 218 self.fails.append(fail_msg) 219 except Exception as e: 220 fail_msg = '[--- unknown error {} ({})]'.format( 221 test_method.__name__, str(e)) 222 logging.exception(fail_msg) 223 self.fails.append(fail_msg) 224 225 self.quick_test_test_log_results( 226 model_testNA=model_testNA, 227 model_testWarn=model_testWarn, 228 skip_common_errors=skip_common_errors) 229 230 if posttest_func: 231 posttest_func(self) 232 233 return wrapper 234 235 return decorator 236 237 def quick_test_test_log_results(self, 238 model_testNA=None, 239 model_testWarn=None, 240 skip_common_errors=False): 241 """Logs and tracks the test results.""" 242 243 if model_testNA is None: 244 model_testNA = [] 245 if model_testWarn is None: 246 model_testWarn = [] 247 248 result_msgs = [] 249 model = self.quick_test_get_model_name() 250 251 if self.test_iter is not None: 252 result_msgs += ['Test Iter: ' + str(self.test_iter)] 253 254 if self.bat_iter is not None: 255 result_msgs += ['Batch Iter: ' + str(self.bat_iter)] 256 257 if self.pkg_is_running is True: 258 result_msgs += ['Package iter: ' + str(self.pkg_iter)] 259 260 if self.bat_name is not None: 261 result_msgs += ['Batch Name: ' + self.bat_name] 262 263 if self.test_name is not None: 264 result_msgs += ['Test Name: ' + self.test_name] 265 266 result_msg = ", ".join(result_msgs) 267 268 if not bool(self.fails): 269 result_msg = 'PASSED | ' + result_msg 270 self.bat_pass_count += 1 271 self.pkg_pass_count += 1 272 # The test should be marked as TESTNA if any of the test expressions 273 # were SKIPPED (they threw their own TESTNA error) or the model is in 274 # the list of NA models (so any failure is considered NA instead) 275 elif model in model_testNA or any(['SKIPPED' in x 276 for x in self.fails]): 277 result_msg = 'TESTNA | ' + result_msg 278 self.bat_testna_count += 1 279 self.pkg_testna_count += 1 280 elif model in model_testWarn: 281 result_msg = 'WARN | ' + result_msg 282 self.bat_warn_count += 1 283 self.pkg_warn_count += 1 284 # Some tests may fail due to known common failure reasons (like usb 285 # disconnect during suspend, bluetoothd crashes, etc). Skip those tests 286 # with TESTNA when that happens. 287 # 288 # This should be used sparingly because it may hide legitimate errors. 289 elif bool(self.had_known_common_failure) and skip_common_errors: 290 result_msg = 'TESTNA | ' + result_msg 291 self.bat_testna_count += 1 292 self.pkg_testna_count += 1 293 else: 294 result_msg = 'FAIL | ' + result_msg 295 self.bat_fail_count += 1 296 self.pkg_fail_count += 1 297 298 logging.info(result_msg) 299 self._print_delimiter() 300 self.bat_tests_results.append(result_msg) 301 self.pkg_tests_results.append(result_msg) 302 303 @staticmethod 304 def quick_test_batch_decorator(batch_name): 305 """A decorator providing a wrapper to a batch. 306 307 Using the decorator a test batch method can implement only its core 308 tests invocations and let the decorator handle the wrapper, which is 309 taking care for whether to run a specific test or the batch as a whole 310 and and running the batch in iterations 311 312 @param batch_name: The name of the batch to log. 313 """ 314 315 def decorator(batch_method): 316 """A decorator wrapper of the decorated test_method. 317 318 @param test_method: The test method being decorated. 319 @return: The wrapper of the test method. 320 """ 321 322 @functools.wraps(batch_method) 323 def wrapper(self, num_iterations=1, test_name=None): 324 """A wrapper of the decorated method. 325 326 @param num_iterations: How many iterations to run. 327 @param test_name: Specific test to run otherwise None to run the 328 whole batch. 329 """ 330 331 if test_name is not None: 332 single_test_method = getattr(self, test_name) 333 for iter in range(1, num_iterations + 1): 334 self.test_iter = iter 335 single_test_method() 336 337 if self.fails: 338 # If failure is marked as TESTNA, prioritize that over 339 # a failure. Same with WARN. 340 if self.bat_testna_count > 0: 341 raise error.TestNAError(self.fails) 342 elif self.bat_warn_count > 0: 343 raise error.TestWarn(self.fails) 344 else: 345 raise error.TestFail(self.fails) 346 else: 347 for iter in range(1, num_iterations + 1): 348 self.quick_test_batch_start(batch_name, iter) 349 batch_method(self, num_iterations, test_name) 350 self.quick_test_batch_end() 351 352 return wrapper 353 354 return decorator 355 356 def quick_test_batch_start(self, bat_name, iteration=1): 357 """Clears and sets test batch variables.""" 358 359 self.bat_tests_results = [] 360 self.bat_pass_count = 0 361 self.bat_fail_count = 0 362 self.bat_testna_count = 0 363 self.bat_warn_count = 0 364 self.bat_name = bat_name 365 self.bat_iter = iteration 366 367 def quick_test_batch_end(self): 368 """Prints results summary of a test batch.""" 369 370 logging.info( 371 '%s Test Batch Summary: total pass %d, total fail %d, ' 372 'warn %d, NA %d', self.bat_name, self.bat_pass_count, 373 self.bat_fail_count, self.bat_warn_count, 374 self.bat_testna_count) 375 for result in self.bat_tests_results: 376 logging.info(result) 377 self._print_delimiter() 378 if self.bat_fail_count > 0: 379 logging.error('===> Test Batch Failed! More than one failure') 380 self._print_delimiter() 381 if self.pkg_is_running is False: 382 raise error.TestFail(self.bat_tests_results) 383 elif self.bat_testna_count > 0: 384 logging.error('===> Test Batch Passed! Some TestNA results') 385 self._print_delimiter() 386 if self.pkg_is_running is False: 387 raise error.TestNAError(self.bat_tests_results) 388 elif self.bat_warn_count > 0: 389 logging.error('===> Test Batch Passed! Some WARN results') 390 self._print_delimiter() 391 if self.pkg_is_running is False: 392 raise error.TestWarn(self.bat_tests_results) 393 else: 394 logging.info('===> Test Batch Passed! zero failures') 395 self._print_delimiter() 396 397 def quick_test_package_start(self, pkg_name): 398 """Clears and sets test package variables.""" 399 400 self.pkg_tests_results = [] 401 self.pkg_pass_count = 0 402 self.pkg_fail_count = 0 403 self.pkg_name = pkg_name 404 self.pkg_is_running = True 405 406 def quick_test_print_summary(self): 407 """Prints results summary of a batch.""" 408 409 logging.info( 410 '%s Test Package Summary: total pass %d, total fail %d, ' 411 'Warn %d, NA %d', self.pkg_name, self.pkg_pass_count, 412 self.pkg_fail_count, self.pkg_warn_count, 413 self.pkg_testna_count) 414 for result in self.pkg_tests_results: 415 logging.info(result) 416 self._print_delimiter() 417 418 def quick_test_package_update_iteration(self, iteration): 419 """Updates state and prints log per package iteration. 420 421 Must be called to have a proper package test result tracking. 422 """ 423 424 self.pkg_iter = iteration 425 if self.pkg_name is None: 426 logging.error('Error: no quick package is running') 427 raise error.TestFail('Error: no quick package is running') 428 logging.info('Starting %s Test Package iteration %d', self.pkg_name, 429 iteration) 430 431 def quick_test_package_end(self): 432 """Prints final result of a test package.""" 433 434 if self.pkg_fail_count > 0: 435 logging.error('===> Test Package Failed! More than one failure') 436 self._print_delimiter() 437 raise error.TestFail(self.bat_tests_results) 438 elif self.pkg_testna_count > 0: 439 logging.error('===> Test Package Passed! Some TestNA results') 440 self._print_delimiter() 441 raise error.TestNAError(self.bat_tests_results) 442 elif self.pkg_warn_count > 0: 443 logging.error('===> Test Package Passed! Some WARN results') 444 self._print_delimiter() 445 raise error.TestWarn(self.bat_tests_results) 446 else: 447 logging.info('===> Test Package Passed! zero failures') 448 self._print_delimiter() 449 self.pkg_is_running = False 450