1# Lint as: python3 2# Copyright 2020 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import ast 7import collections 8import glob 9import logging 10import os 11import shutil 12from typing import Generator, List, DefaultDict, NoReturn, Optional, Tuple 13 14import control_files 15 16_ROOT_DIR = os.path.realpath( 17 os.path.join(os.path.realpath(__file__), '../../..')) 18_METADATA_DIR = os.path.join(_ROOT_DIR, 'metadata') 19_CATEGORY_DEFS_DIR = os.path.join(_METADATA_DIR, 'tests') 20_CATEGORY_DEFS_PACKAGE = '//metadata/tests' 21_ALL_TESTS_FILE = os.path.join(_METADATA_DIR, 'tests.star') 22 23 24class ExtractionError(Exception): 25 """Generic error from this package.""" 26 27 28def main(): 29 """Extract starlark metadata for all tests in this project. 30 31 This script generates the starlark config files by parsing control files. 32 The intent is to backfill majority of the required starlark configs this way, 33 followed by hand-editing. 34 """ 35 logging.basicConfig(level=logging.INFO) 36 controls = control_files.load_all() 37 logging.info('Loaded %d control files', len(controls)) 38 categorized = dict(_categorize_control_files(controls)) 39 logging.info('Categorized control files into %d categories', len(categorized)) 40 41 _delete_existing_defs() 42 for category, control in categorized.items(): 43 with open(_category_def_file(category), 'w') as f: 44 f.write(_generate_category_def(control)) 45 logging.info('Wrote starlark files to %s', _CATEGORY_DEFS_DIR) 46 with open(_ALL_TESTS_FILE, 'w') as f: 47 f.write(_generate_all_tests_def(categorized.keys())) 48 logging.info('Included all categories in %s', _ALL_TESTS_FILE) 49 50 51def _delete_existing_defs() -> NoReturn: 52 if os.path.exists(_CATEGORY_DEFS_DIR): 53 shutil.rmtree(_CATEGORY_DEFS_DIR) 54 os.mkdir(_CATEGORY_DEFS_DIR) 55 if os.path.exists(_ALL_TESTS_FILE): 56 os.unlink(_ALL_TESTS_FILE) 57 58 59def _categorize_control_files(controls: List[control_files.Control] 60 ) -> DefaultDict[str, control_files.Control]: 61 categorized = collections.defaultdict(list) 62 for c in controls: 63 categorized[c.category].append(c) 64 if 'uncategorized' in categorized: 65 raise ExtractionError('"uncategorized" is reserved') 66 if '' in categorized: 67 categorized['uncategorized'] = categorized[''] 68 del categorized[''] 69 return categorized 70 71 72def _category_def_file(category: str) -> str: 73 return os.path.join(_CATEGORY_DEFS_DIR, '%s.star' % category) 74 75 76_CATEGORY_TEMPLATE = """ 77# Copyright 2020 The Chromium OS Authors. All rights reserved. 78# Use of this source code is governed by a BSD-style license that can be 79# found in the LICENSE file. 80 81# !!! GENERATED FILE. DO NOT EDIT !!! 82 83load('//metadata/test_common.star', 'test_common') 84 85def define_tests(): 86 return [%(test_list)s 87 ] 88""" 89 90_TEST_TEMPLATE = """ 91 test_common.define_test( 92 %(name)s, 93 suites = %(suites)s, 94 main_package = %(main_package)s, 95 )""" 96 97 98def _generate_category_def(controls: List[control_files.Control]) -> str: 99 controls = sorted(controls, key=lambda c: c.name) 100 test_list = ','.join([ 101 _TEST_TEMPLATE % { 102 'name': "'%s/%s'" % (c.category, c.name), 103 'suites': sorted(c.suites), 104 'main_package': "'%s'" % c.main_package, 105 } for c in controls 106 ]) 107 return _CATEGORY_TEMPLATE % {'test_list': test_list} 108 109 110_ALL_CATEGORIES_TEMPLATE = """ 111# Copyright 2020 The Chromium OS Authors. All rights reserved. 112# Use of this source code is governed by a BSD-style license that can be 113# found in the LICENSE file. 114 115# !!! GENERATED FILE. DO NOT EDIT !!! 116 117%(load_inclusions)s 118 119def define_tests(): 120 tests = [] 121 %(append_inclusions)s 122 return tests 123""" 124 125_LOAD_CATEGORY_TEMPLATE = """ 126load('%(path)s', define_%(name)s = 'define_tests')""" 127 128_APPEND_CATEGORY_TEMPLATE = """ 129 tests += define_%(name)s()""" 130 131 132def _generate_all_tests_def(categories: List[str]) -> str: 133 load_inclusions = [ 134 _LOAD_CATEGORY_TEMPLATE % { 135 'name': c, 136 'path': '%s/%s.star' % (_CATEGORY_DEFS_PACKAGE, c), 137 } for c in categories 138 ] 139 append_inclusions = [ 140 _APPEND_CATEGORY_TEMPLATE % { 141 'name': c 142 } for c in categories 143 ] 144 return _ALL_CATEGORIES_TEMPLATE % { 145 'load_inclusions': ''.join(load_inclusions), 146 'append_inclusions': ''.join(append_inclusions), 147 } 148 149 150if __name__ == '__main__': 151 main() 152