1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Manages the list of Pigweed modules. 15 16Used by modules.gni to generate: 17 18- a build arg for each module, 19- a list of module paths (pw_modules), 20- a list of module tests (pw_module_tests), and 21- a list of module docs (pw_module_docs). 22""" 23 24import argparse 25import difflib 26import enum 27import io 28import os 29from pathlib import Path 30import sys 31import subprocess 32from typing import Iterator, Sequence 33 34_COPYRIGHT_NOTICE = '''\ 35# Copyright 2022 The Pigweed Authors 36# 37# Licensed under the Apache License, Version 2.0 (the "License"); you may not 38# use this file except in compliance with the License. You may obtain a copy of 39# the License at 40# 41# https://www.apache.org/licenses/LICENSE-2.0 42# 43# Unless required by applicable law or agreed to in writing, software 44# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 45# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 46# License for the specific language governing permissions and limitations under 47# the License.''' 48 49_WARNING = '\033[31m\033[1mWARNING:\033[0m ' # Red WARNING: prefix 50_ERROR = '\033[41m\033[37m\033[1mERROR:\033[0m ' # Red background ERROR: prefix 51 52_MISSING_MODULES_WARNING = ( 53 _WARNING 54 + '''\ 55The PIGWEED_MODULES list is missing the following modules: 56{modules} 57 58If the listed modules are Pigweed modules, add them to PIGWEED_MODULES. 59 60If the listed modules are not actual Pigweed modules, remove any stray pw_* 61directories in the Pigweed repository (git clean -fd). 62''' 63) 64 65_OUT_OF_DATE_WARNING = ( 66 _ERROR 67 + '''\ 68The generated Pigweed modules list .gni file is out of date! 69 70Regenerate the modules lists and commit it to fix this: 71 72 ninja -C {out_dir} update_modules 73 74 git add {file} 75''' 76) 77 78_FORMAT_FAILED_WARNING = ( 79 _ERROR 80 + '''\ 81Failed to generate a valid .gni from PIGWEED_MODULES! 82 83This may be a Pigweed bug; please report this to the Pigweed team. 84''' 85) 86 87_DO_NOT_SET = 'DO NOT SET THIS BUILD ARGUMENT!' 88 89 90def _module_list_warnings(root: Path, modules: Sequence[str]) -> Iterator[str]: 91 missing = _missing_modules(root, modules) 92 if missing: 93 yield _MISSING_MODULES_WARNING.format( 94 modules=''.join(f'\n - {module}' for module in missing) 95 ) 96 97 if any(modules[i] > modules[i + 1] for i in range(len(modules) - 1)): 98 yield _WARNING + 'The PIGWEED_MODULES list is not sorted!' 99 yield '' 100 yield 'Apply the following diff to fix the order:' 101 yield '' 102 yield from difflib.unified_diff( 103 modules, 104 sorted(modules), 105 lineterm='', 106 n=1, 107 fromfile='PIGWEED_MODULES', 108 tofile='PIGWEED_MODULES', 109 ) 110 111 yield '' 112 113 114def _generate_modules_gni( 115 prefix: Path, modules: Sequence[str] 116) -> Iterator[str]: 117 """Generates a .gni file with variables and lists for Pigweed modules.""" 118 script_path = Path(__file__).resolve() 119 script = script_path.relative_to(script_path.parent.parent).as_posix() 120 121 module_paths: Sequence[Path] = list(Path(module) for module in modules) 122 123 yield _COPYRIGHT_NOTICE 124 yield '' 125 yield '# Build args and lists for all modules in Pigweed.' 126 yield '#' 127 yield f'# DO NOT EDIT! Generated by {script}.' 128 yield '#' 129 yield '# To add modules here, list them in PIGWEED_MODULES and build the' 130 yield '# update_modules target and commit the updated version of this file:' 131 yield '#' 132 yield '# ninja -C out update_modules' 133 yield '#' 134 yield '# DO NOT IMPORT THIS FILE DIRECTLY!' 135 yield '#' 136 yield '# Import it through //build_overrides/pigweed.gni instead.' 137 yield '' 138 yield '# Declare a build arg for each module.' 139 yield 'declare_args() {' 140 141 for module in module_paths: 142 final_path = (prefix / module).as_posix() 143 yield f'dir_{module.name} = get_path_info("{final_path}", "abspath")' 144 145 yield '}' 146 yield '' 147 yield '# Declare these as GN args in case this is imported in args.gni.' 148 yield '# Use a separate block so variables in the prior block can be used.' 149 yield 'declare_args() {' 150 yield f'# A list with paths to all Pigweed module. {_DO_NOT_SET}' 151 yield 'pw_modules = [' 152 153 for module in module_paths: 154 yield f'dir_{module.name},' 155 156 yield ']' 157 yield '' 158 159 yield f'# A list with all Pigweed module test groups. {_DO_NOT_SET}' 160 yield 'pw_module_tests = [' 161 162 for module in module_paths: 163 yield f'"$dir_{module.name}:tests",' 164 165 yield ']' 166 yield '' 167 yield f'# A list with all Pigweed modules docs groups. {_DO_NOT_SET}' 168 yield 'pw_module_docs = [' 169 170 for module in module_paths: 171 yield f'"$dir_{module.name}:docs",' 172 173 yield ']' 174 yield '' 175 yield '}' 176 177 178def _ignore_module_entry(entry: Path): 179 # If there are only empty directories we ignore the "module". 180 if entry.is_dir(): 181 return True 182 183 # Likewise, ignore compiled Python files. 184 if entry.name.endswith('.pyc'): 185 return True 186 187 return False 188 189 190def _missing_modules(root: Path, modules: Sequence[str]) -> Sequence[str]: 191 found_modules = set() 192 for path in root.glob('pw_*'): 193 if not path.is_dir(): 194 continue 195 196 for entry in path.rglob('*'): 197 if not _ignore_module_entry(entry): 198 found_modules.add(path) 199 break 200 201 return sorted( 202 frozenset(str(p.relative_to(root)) for p in found_modules) 203 - frozenset(modules) 204 ) 205 206 207class Mode(enum.Enum): 208 WARN = 0 # Warn if anything is out of date 209 CHECK = 1 # Fail if anything is out of date 210 UPDATE = 2 # Update the generated modules lists 211 212 213def _parse_args() -> dict: 214 parser = argparse.ArgumentParser( 215 description=__doc__, 216 formatter_class=argparse.RawDescriptionHelpFormatter, 217 ) 218 parser.add_argument('root', type=Path, help='Root build dir') 219 parser.add_argument('modules_list', type=Path, help='Input modules list') 220 parser.add_argument('modules_gni_file', type=Path, help='Output .gni file') 221 parser.add_argument( 222 '--mode', type=Mode.__getitem__, choices=Mode, required=True 223 ) 224 parser.add_argument( 225 '--stamp', 226 type=Path, 227 help='Stamp file for operations that should only run once (warn)', 228 ) 229 return vars(parser.parse_args()) 230 231 232def main( 233 root: Path, 234 modules_list: Path, 235 modules_gni_file: Path, 236 mode: Mode, 237 stamp: Path | None = None, 238) -> int: 239 """Manages the list of Pigweed modules.""" 240 prefix = Path(os.path.relpath(root, modules_gni_file.parent)) 241 modules = modules_list.read_text().splitlines() 242 243 # Detect any problems with the modules list. 244 warnings = list(_module_list_warnings(root, modules)) 245 errors = [] 246 247 modules.sort() # Sort in case the modules list in case it wasn't sorted. 248 249 # Check if the contents of the .gni file are out of date. 250 if mode in (Mode.WARN, Mode.CHECK): 251 text = io.StringIO() 252 for line in _generate_modules_gni(prefix, modules): 253 print(line, file=text) 254 255 process = subprocess.run( 256 ['gn', 'format', '--stdin'], 257 input=text.getvalue().encode('utf-8'), 258 stdout=subprocess.PIPE, 259 ) 260 if process.returncode != 0: 261 errors.append(_FORMAT_FAILED_WARNING) 262 263 # Make a diff of required changes 264 modules_gni_relpath = os.path.relpath(modules_gni_file, root) 265 diff = list( 266 difflib.unified_diff( 267 modules_gni_file.read_text().splitlines(), 268 process.stdout.decode('utf-8', errors='replace').splitlines(), 269 fromfile=os.path.join('a', modules_gni_relpath), 270 tofile=os.path.join('b', modules_gni_relpath), 271 lineterm='', 272 n=1, 273 ) 274 ) 275 # If any differences were found, print the error and the diff. 276 if diff: 277 errors.append( 278 _OUT_OF_DATE_WARNING.format( 279 out_dir=os.path.relpath(os.curdir, root), 280 file=os.path.relpath(modules_gni_file, root), 281 ) 282 ) 283 errors.append('Expected Diff:\n') 284 errors.append('\n'.join(diff)) 285 errors.append('\n') 286 287 elif mode is Mode.UPDATE: # Update the modules .gni file 288 with modules_gni_file.open('w', encoding='utf-8') as file: 289 for line in _generate_modules_gni(prefix, modules): 290 print(line, file=file) 291 292 process = subprocess.run( 293 ['gn', 'format', modules_gni_file], stdout=subprocess.DEVNULL 294 ) 295 if process.returncode != 0: 296 errors.append(_FORMAT_FAILED_WARNING) 297 298 # If there are errors, display them and abort. 299 if warnings or errors: 300 for line in warnings + errors: 301 print(line, file=sys.stderr) 302 303 # Delete the stamp so this always reruns. Deleting is necessary since 304 # some of the checks do not depend on input files. 305 if stamp and stamp.exists(): 306 stamp.unlink() 307 308 if mode is Mode.WARN: 309 return 0 310 311 if mode is Mode.CHECK: 312 return 1 313 314 return 1 if errors else 0 # Allow warnings but not errors when updating 315 316 if stamp: 317 stamp.touch() 318 319 return 0 320 321 322if __name__ == '__main__': 323 sys.exit(main(**_parse_args())) 324