1# Lint as: python2, python3 2# pylint: disable-msg=C0111 3# Copyright 2008 Google Inc. Released under the GPL v2 4 5from __future__ import absolute_import 6from __future__ import division 7from __future__ import print_function 8 9import ast 10import logging 11import textwrap 12import re 13import six 14 15from autotest_lib.client.common_lib import autotest_enum 16from autotest_lib.client.common_lib import global_config 17from autotest_lib.client.common_lib import priorities 18 19 20REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type']) 21OBSOLETE_VARS = set(['experimental']) 22 23CONTROL_TYPE = autotest_enum.AutotestEnum('Server', 'Client', start_value=1) 24CONTROL_TYPE_NAMES = autotest_enum.AutotestEnum(*CONTROL_TYPE.names, 25 string_values=True) 26 27_SUITE_ATTRIBUTE_PREFIX = 'suite:' 28 29CONFIG = global_config.global_config 30 31# Default maximum test result size in kB. 32DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value( 33 'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000) 34 35 36class ControlVariableException(Exception): 37 pass 38 39def _validate_control_file_fields(control_file_path, control_file_vars, 40 raise_warnings): 41 """Validate the given set of variables from a control file. 42 43 @param control_file_path: string path of the control file these were 44 loaded from. 45 @param control_file_vars: dict of variables set in a control file. 46 @param raise_warnings: True iff we should raise on invalid variables. 47 48 """ 49 diff = REQUIRED_VARS - set(control_file_vars) 50 if diff: 51 warning = ('WARNING: Not all required control ' 52 'variables were specified in %s. Please define ' 53 '%s.') % (control_file_path, ', '.join(diff)) 54 if raise_warnings: 55 raise ControlVariableException(warning) 56 print(textwrap.wrap(warning, 80)) 57 58 obsolete = OBSOLETE_VARS & set(control_file_vars) 59 if obsolete: 60 warning = ('WARNING: Obsolete variables were ' 61 'specified in %s. Please remove ' 62 '%s.') % (control_file_path, ', '.join(obsolete)) 63 if raise_warnings: 64 raise ControlVariableException(warning) 65 print(textwrap.wrap(warning, 80)) 66 67 68class ControlData(object): 69 # Available TIME settings in control file, the list must be in lower case 70 # and in ascending order, test running faster comes first. 71 TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy'] 72 TEST_TIME = autotest_enum.AutotestEnum(*TEST_TIME_LIST, 73 string_values=False) 74 75 @staticmethod 76 def get_test_time_index(time): 77 """ 78 Get the order of estimated test time, based on the TIME setting in 79 Control file. Faster test gets a lower index number. 80 """ 81 try: 82 return ControlData.TEST_TIME.get_value(time.lower()) 83 except AttributeError: 84 # Raise exception if time value is not a valid TIME setting. 85 error_msg = '%s is not a valid TIME.' % time 86 logging.error(error_msg) 87 raise ControlVariableException(error_msg) 88 89 90 def __init__(self, vars, path, raise_warnings=False): 91 # Defaults 92 self.path = path 93 self.dependencies = set() 94 # TODO(jrbarnette): This should be removed once outside 95 # code that uses can be changed. 96 self.experimental = False 97 self.run_verify = True 98 self.sync_count = 1 99 self.test_parameters = set() 100 self.test_category = '' 101 self.test_class = '' 102 self.job_retries = 0 103 # Default to require server-side package. Unless require_ssp is 104 # explicitly set to False, server-side package will be used for the 105 # job. 106 self.require_ssp = None 107 self.attributes = set() 108 self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB 109 self.priority = priorities.Priority.DEFAULT 110 self.extended_timeout = None 111 self.fast = True 112 # This will only be honored via `test_that`, and not in lab (for now). 113 self.py_version = None 114 115 _validate_control_file_fields(self.path, vars, raise_warnings) 116 117 for key, val in six.iteritems(vars): 118 try: 119 self.set_attr(key, val, raise_warnings) 120 except Exception as e: 121 if raise_warnings: 122 raise 123 print('WARNING: %s; skipping' % e) 124 125 self._patch_up_suites_from_attributes() 126 127 128 @property 129 def suite_tag_parts(self): 130 """Return the part strings of the test's suite tag.""" 131 if hasattr(self, 'suite'): 132 return [part.strip() for part in self.suite.split(',')] 133 else: 134 return [] 135 136 137 def set_attr(self, attr, val, raise_warnings=False): 138 attr = attr.lower() 139 try: 140 set_fn = getattr(self, 'set_%s' % attr) 141 set_fn(val) 142 except AttributeError: 143 # This must not be a variable we care about 144 pass 145 146 147 def _patch_up_suites_from_attributes(self): 148 """Patch up the set of suites this test is part of. 149 150 Legacy builds will not have an appropriate ATTRIBUTES field set. 151 Take the union of suites specified via ATTRIBUTES and suites specified 152 via SUITE. 153 154 SUITE used to be its own variable, but now suites are taken only from 155 the attributes. 156 157 """ 158 159 suite_names = set() 160 # Extract any suites we know ourselves to be in based on the SUITE 161 # line. This line is deprecated, but control files in old builds will 162 # still have it. 163 if hasattr(self, 'suite'): 164 existing_suites = self.suite.split(',') 165 existing_suites = [name.strip() for name in existing_suites] 166 existing_suites = [name for name in existing_suites if name] 167 suite_names.update(existing_suites) 168 169 # Figure out if our attributes mention any suites. 170 for attribute in self.attributes: 171 if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX): 172 continue 173 suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):] 174 suite_names.add(suite_name) 175 176 # Rebuild the suite field if necessary. 177 if suite_names: 178 self.set_suite(','.join(sorted(list(suite_names)))) 179 180 181 def _set_string(self, attr, val): 182 val = str(val) 183 setattr(self, attr, val) 184 185 186 def _set_option(self, attr, val, options): 187 val = str(val) 188 if val.lower() not in [x.lower() for x in options]: 189 raise ValueError("%s must be one of the following " 190 "options: %s" % (attr, 191 ', '.join(options))) 192 setattr(self, attr, val) 193 194 195 def _set_bool(self, attr, val): 196 val = str(val).lower() 197 if val == "false": 198 val = False 199 elif val == "true": 200 val = True 201 else: 202 msg = "%s must be either true or false" % attr 203 raise ValueError(msg) 204 setattr(self, attr, val) 205 206 207 def _set_int(self, attr, val, min=None, max=None): 208 val = int(val) 209 if min is not None and min > val: 210 raise ValueError("%s is %d, which is below the " 211 "minimum of %d" % (attr, val, min)) 212 if max is not None and max < val: 213 raise ValueError("%s is %d, which is above the " 214 "maximum of %d" % (attr, val, max)) 215 setattr(self, attr, val) 216 217 218 def _set_set(self, attr, val): 219 val = str(val) 220 items = [x.strip() for x in val.split(',') if x.strip()] 221 setattr(self, attr, set(items)) 222 223 224 def set_author(self, val): 225 self._set_string('author', val) 226 227 228 def set_dependencies(self, val): 229 self._set_set('dependencies', val) 230 231 232 def set_doc(self, val): 233 self._set_string('doc', val) 234 235 236 def set_name(self, val): 237 self._set_string('name', val) 238 239 240 def set_run_verify(self, val): 241 self._set_bool('run_verify', val) 242 243 244 def set_sync_count(self, val): 245 self._set_int('sync_count', val, min=1) 246 247 248 def set_suite(self, val): 249 self._set_string('suite', val) 250 251 252 def set_time(self, val): 253 self._set_option('time', val, ControlData.TEST_TIME_LIST) 254 255 256 def set_test_class(self, val): 257 self._set_string('test_class', val.lower()) 258 259 260 def set_test_category(self, val): 261 self._set_string('test_category', val.lower()) 262 263 264 def set_test_type(self, val): 265 self._set_option('test_type', val, list(CONTROL_TYPE.names)) 266 267 268 def set_test_parameters(self, val): 269 self._set_set('test_parameters', val) 270 271 272 def set_job_retries(self, val): 273 self._set_int('job_retries', val) 274 275 276 def set_bug_template(self, val): 277 if type(val) == dict: 278 setattr(self, 'bug_template', val) 279 280 281 def set_require_ssp(self, val): 282 self._set_bool('require_ssp', val) 283 284 285 def set_build(self, val): 286 self._set_string('build', val) 287 288 289 def set_builds(self, val): 290 if type(val) == dict: 291 setattr(self, 'builds', val) 292 293 def set_max_result_size_kb(self, val): 294 self._set_int('max_result_size_KB', val) 295 296 def set_priority(self, val): 297 self._set_int('priority', val) 298 299 def set_fast(self, val): 300 self._set_bool('fast', val) 301 302 def set_update_type(self, val): 303 self._set_string('update_type', val) 304 305 def set_source_release(self, val): 306 self._set_string('source_release', val) 307 308 def set_target_release(self, val): 309 self._set_string('target_release', val) 310 311 def set_target_payload_uri(self, val): 312 self._set_string('target_payload_uri', val) 313 314 def set_source_payload_uri(self, val): 315 self._set_string('source_payload_uri', val) 316 317 def set_source_archive_uri(self, val): 318 self._set_string('source_archive_uri', val) 319 320 def set_attributes(self, val): 321 self._set_set('attributes', val) 322 323 def set_extended_timeout(self, val): 324 """In seconds.""" 325 self._set_int('extended_timeout', val) 326 327 def set_py_version(self, val): 328 """In majors, ie: 2 or 3.""" 329 self._set_int('py_version', val) 330 331 332def _extract_const(expr): 333 assert (expr.__class__ == ast.Str) 334 if six.PY2: 335 assert (expr.s.__class__ in (str, int, float, unicode)) 336 else: 337 assert (expr.s.__class__ in (str, int, float)) 338 return str(expr.s).strip() 339 340 341def _extract_dict(expr): 342 assert (expr.__class__ == ast.Dict) 343 assert (expr.keys.__class__ == list) 344 cf_dict = {} 345 for key, value in zip(expr.keys, expr.values): 346 try: 347 key = _extract_const(key) 348 val = _extract_expression(value) 349 except (AssertionError, ValueError): 350 pass 351 else: 352 cf_dict[key] = val 353 return cf_dict 354 355 356def _extract_list(expr): 357 assert (expr.__class__ == ast.List) 358 list_values = [] 359 for value in expr.elts: 360 try: 361 list_values.append(_extract_expression(value)) 362 except (AssertionError, ValueError): 363 pass 364 return list_values 365 366 367def _extract_name(expr): 368 assert (expr.__class__ == ast.Name) 369 assert (expr.id in ('False', 'True', 'None')) 370 return str(expr.id) 371 372 373def _extract_expression(expr): 374 if expr.__class__ == ast.Str: 375 return _extract_const(expr) 376 if expr.__class__ == ast.Name: 377 return _extract_name(expr) 378 if expr.__class__ == ast.Dict: 379 return _extract_dict(expr) 380 if expr.__class__ == ast.List: 381 return _extract_list(expr) 382 if expr.__class__ == ast.Num: 383 return expr.n 384 if six.PY3 and expr.__class__ == ast.NameConstant: 385 return expr.value 386 if six.PY3 and expr.__class__ == ast.Constant: 387 try: 388 return expr.value.strip() 389 except Exception: 390 return expr.value 391 raise ValueError('Unknown rval %s' % expr) 392 393 394def _extract_assignment(n): 395 assert (n.__class__ == ast.Assign) 396 assert (len(n.targets) == 1) 397 assert (n.targets[0].__class__ == ast.Name) 398 val = _extract_expression(n.value) 399 key = n.targets[0].id.lower() 400 return (key, val) 401 402 403def parse_control_string(control, raise_warnings=False, path=''): 404 """Parse a control file from a string. 405 406 @param control: string containing the text of a control file. 407 @param raise_warnings: True iff ControlData should raise an error on 408 warnings about control file contents. 409 @param path: string path to the control file. 410 411 """ 412 try: 413 mod = ast.parse(control) 414 except SyntaxError as e: 415 logging.error('Syntax error (%s) while parsing control string:', e) 416 lines = control.split('\n') 417 for n, l in enumerate(lines): 418 logging.error('Line %d: %s', n + 1, l) 419 raise ControlVariableException("Error parsing data because %s" % e) 420 return finish_parse(mod, path, raise_warnings) 421 422 423def parse_control(path, raise_warnings=False): 424 try: 425 with open(path, 'r') as r: 426 mod = ast.parse(r.read()) 427 except SyntaxError as e: 428 raise ControlVariableException("Error parsing %s because %s" % 429 (path, e)) 430 return finish_parse(mod, path, raise_warnings) 431 432 433def _try_extract_assignment(node, variables): 434 """Try to extract assignment from the given node. 435 436 @param node: An Assign object. 437 @param variables: Dictionary to store the parsed assignments. 438 """ 439 try: 440 key, val = _extract_assignment(node) 441 variables[key] = val 442 except (AssertionError, ValueError) as e: 443 pass 444 445 446def finish_parse(mod, path, raise_warnings): 447 assert (mod.__class__ == ast.Module) 448 assert (mod.body.__class__ == list) 449 450 variables = {} 451 injection_variables = {} 452 for n in mod.body: 453 if (n.__class__ == ast.FunctionDef and re.match('step\d+', n.name)): 454 vars_in_step = {} 455 for sub_node in n.body: 456 _try_extract_assignment(sub_node, vars_in_step) 457 if vars_in_step: 458 # Empty the vars collection so assignments from multiple steps 459 # won't be mixed. 460 variables.clear() 461 variables.update(vars_in_step) 462 else: 463 _try_extract_assignment(n, injection_variables) 464 465 variables.update(injection_variables) 466 return ControlData(variables, path, raise_warnings) 467