1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Argument parsing code for presubmit checks.""" 15 16import argparse 17import fnmatch 18import logging 19import os 20from pathlib import Path 21import re 22import shutil 23import textwrap 24from typing import Callable, Collection, Sequence 25 26from pw_presubmit import git_repo, presubmit 27 28_LOG = logging.getLogger(__name__) 29DEFAULT_PATH = Path('out', 'presubmit') 30 31_OUTPUT_PATH_README = '''\ 32This directory was created by pw_presubmit to run presubmit checks for the 33{repo} repository. This directory is not used by the regular GN or CMake Ninja 34builds. It may be deleted safely. 35''' 36 37 38def add_path_arguments(parser) -> None: 39 """Adds common presubmit check options to an argument parser.""" 40 41 parser.add_argument( 42 'paths', 43 metavar='pathspec', 44 nargs='*', 45 help=( 46 'Paths or patterns to which to restrict the checks. These are ' 47 'interpreted as Git pathspecs. If --base is provided, only ' 48 'paths changed since that commit are checked.' 49 ), 50 ) 51 52 base = parser.add_mutually_exclusive_group() 53 base.add_argument( 54 '-b', 55 '--base', 56 metavar='commit', 57 default=git_repo.TRACKING_BRANCH_ALIAS, 58 help=( 59 'Git revision against which to diff for changed files. ' 60 'Default is the tracking branch of the current branch: ' 61 f'{git_repo.TRACKING_BRANCH_ALIAS}' 62 ), 63 ) 64 65 base.add_argument( 66 '--all', 67 '--full', 68 dest='base', 69 action='store_const', 70 const=None, 71 help='Run actions for all files, not just changed files.', 72 ) 73 74 parser.add_argument( 75 '-e', 76 '--exclude', 77 metavar='regular_expression', 78 default=[], 79 action='append', 80 type=re.compile, 81 help=( 82 'Exclude paths matching any of these regular expressions, ' 83 "which are interpreted relative to each Git repository's root." 84 ), 85 ) 86 87 88def _add_programs_arguments( 89 parser: argparse.ArgumentParser, programs: presubmit.Programs, default: str 90): 91 def presubmit_program(arg: str) -> presubmit.Program: 92 if arg not in programs: 93 all_program_names = ', '.join(sorted(programs.keys())) 94 raise argparse.ArgumentTypeError( 95 f'{arg} is not the name of a presubmit program\n\n' 96 f'Valid Programs:\n{all_program_names}' 97 ) 98 99 return programs[arg] 100 101 # This argument is used to copy the default program into the argparse 102 # namespace argument. It's not intended to be set by users. 103 parser.add_argument( 104 '--default-program', 105 default=[presubmit_program(default)], 106 help=argparse.SUPPRESS, 107 ) 108 109 parser.add_argument( 110 '-p', 111 '--program', 112 choices=programs.values(), 113 type=presubmit_program, 114 action='append', 115 default=[], 116 help='Which presubmit program to run', 117 ) 118 119 parser.add_argument( 120 '--list-steps-file', 121 dest='list_steps_file', 122 type=Path, 123 help=argparse.SUPPRESS, 124 ) 125 126 all_steps = programs.all_steps() 127 128 def list_steps() -> None: 129 """List all available presubmit steps and their docstrings.""" 130 for step in sorted(all_steps.values(), key=str): 131 _LOG.info('%s', step) 132 if step.doc: 133 first, *rest = step.doc.split('\n', 1) 134 _LOG.info(' %s', first) 135 if rest and _LOG.isEnabledFor(logging.DEBUG): 136 for line in textwrap.dedent(*rest).splitlines(): 137 _LOG.debug(' %s', line) 138 139 parser.add_argument( 140 '--list-steps', 141 action='store_const', 142 const=list_steps, 143 default=None, 144 help='List all the available steps.', 145 ) 146 147 def presubmit_step(arg: str) -> list[presubmit.Check]: 148 """Return a list of matching presubmit steps.""" 149 filtered_step_names = fnmatch.filter(all_steps.keys(), arg) 150 151 if not filtered_step_names: 152 all_step_names = ', '.join(sorted(all_steps.keys())) 153 raise argparse.ArgumentTypeError( 154 f'"{arg}" does not match the name of a presubmit step.\n\n' 155 f'Valid Steps:\n{all_step_names}' 156 ) 157 158 return list(all_steps[name] for name in filtered_step_names) 159 160 parser.add_argument( 161 '--step', 162 action='extend', 163 default=[], 164 help=( 165 'Run specific steps instead of running a full program. Include an ' 166 'asterix to match more than one step name. For example: --step ' 167 "'*_format'" 168 ), 169 type=presubmit_step, 170 ) 171 172 parser.add_argument( 173 '--substep', 174 action='store', 175 help=( 176 "Run a specific substep of a step. Only supported if there's only " 177 'one --step argument and no --program arguments.' 178 ), 179 ) 180 181 def gn_arg(argument): 182 key, value = argument.split('=', 1) 183 return (key, value) 184 185 # Recipe code for handling builds with pre-release toolchains requires the 186 # ability to pass through GN args. This ability is not expected to be used 187 # directly outside of this case, so the option is hidden. Values passed in 188 # to this argument should be of the form 'key=value'. 189 parser.add_argument( 190 '--override-gn-arg', 191 dest='override_gn_args', 192 action='append', 193 type=gn_arg, 194 help=argparse.SUPPRESS, 195 ) 196 197 198def add_arguments( 199 parser: argparse.ArgumentParser, 200 programs: presubmit.Programs | None = None, 201 default: str = '', 202) -> None: 203 """Adds common presubmit check options to an argument parser.""" 204 205 add_path_arguments(parser) 206 207 parser.add_argument( 208 '--dry-run', 209 action='store_true', 210 help=( 211 'Execute the presubits with in dry-run mode. System commands that' 212 'pw_presubmit would run are instead printed to the terminal.' 213 ), 214 ) 215 parser.add_argument( 216 '-k', 217 '--keep-going', 218 action='store_true', 219 help='Continue running presubmit steps after a failure.', 220 ) 221 222 parser.add_argument( 223 '--continue-after-build-error', 224 action='store_true', 225 help=( 226 'Within presubmit steps, continue running build steps after a ' 227 'failure.' 228 ), 229 ) 230 231 parser.add_argument( 232 '--rng-seed', 233 type=int, 234 default=1, 235 help='Seed for random number generators.', 236 ) 237 238 parser.add_argument( 239 '--output-directory', 240 type=Path, 241 help=f'Output directory (default: {"<repo root>" / DEFAULT_PATH})', 242 ) 243 244 parser.add_argument( 245 '--package-root', 246 type=Path, 247 help='Package root directory (default: <env directory>/packages)', 248 ) 249 250 exclusive = parser.add_mutually_exclusive_group() 251 exclusive.add_argument( 252 '--clear', 253 '--clean', 254 action='store_true', 255 help='Delete the presubmit output directory and exit.', 256 ) 257 258 if programs: 259 if not default: 260 raise ValueError('A default must be provided with programs') 261 262 _add_programs_arguments(parser, programs, default) 263 264 # LUCI builders extract the list of steps from the program and run them 265 # individually for a better UX in MILO. 266 parser.add_argument( 267 '--only-list-steps', 268 action='store_true', 269 help=argparse.SUPPRESS, 270 ) 271 272 273def _get_default_parser() -> argparse.ArgumentParser: 274 """Return all common pw presubmit args for sphinx documentation.""" 275 parser = argparse.ArgumentParser(description="Runs local presubmit checks.") 276 add_arguments(parser) 277 return parser 278 279 280def run( # pylint: disable=too-many-arguments 281 default_program: presubmit.Program | None, 282 program: Sequence[presubmit.Program], 283 step: Sequence[presubmit.Check], 284 substep: str, 285 output_directory: Path | None, 286 package_root: Path, 287 clear: bool, 288 root: Path | None = None, 289 repositories: Collection[Path] = (), 290 only_list_steps=False, 291 list_steps: Callable[[], None] | None = None, 292 dry_run: bool = False, 293 **other_args, 294) -> int: 295 """Processes arguments from add_arguments and runs the presubmit. 296 297 Args: 298 default_program: program to use if neither --program nor --step is used 299 program: from the --program option 300 step: from the --step option 301 substep: from the --substep option 302 output_directory: from --output-directory option 303 package_root: from --package-root option 304 clear: from the --clear option 305 root: base path from which to run presubmit checks; defaults to the root 306 of the current directory's repository 307 repositories: roots of Git repositories on which to run presubmit checks; 308 defaults to the root of the current directory's repository 309 only_list_steps: list the steps that would be executed, one per line, 310 instead of executing them 311 list_steps: list the steps that would be executed with their docstrings 312 **other_args: remaining arguments defined by by add_arguments 313 314 Returns: 315 exit code for sys.exit; 0 if successful, 1 if an error occurred 316 """ 317 if root is None: 318 root = git_repo.root() 319 320 if not repositories: 321 repositories = [root] 322 323 if output_directory is None: 324 output_directory = root / DEFAULT_PATH 325 326 output_directory.mkdir(parents=True, exist_ok=True) 327 output_directory.joinpath('README.txt').write_text( 328 _OUTPUT_PATH_README.format(repo=root) 329 ) 330 331 if not package_root: 332 package_root = Path(os.environ['PW_PACKAGE_ROOT']) 333 334 _LOG.debug('Using environment at %s', output_directory) 335 336 if clear: 337 _LOG.info('Clearing presubmit output directory') 338 339 if output_directory.exists(): 340 shutil.rmtree(output_directory) 341 _LOG.info('Deleted %s', output_directory) 342 343 return 0 344 345 if list_steps: 346 list_steps() 347 return 0 348 349 final_program: presubmit.Program | None = None 350 if not program and not step: 351 assert default_program # Cast away "| None". 352 final_program = default_program 353 elif len(program) == 1 and not step: 354 final_program = program[0] 355 else: 356 steps: list[presubmit.Check] = [] 357 steps.extend(step) 358 for prog in program: 359 steps.extend(prog) 360 final_program = presubmit.Program('', steps) 361 362 if substep and len(final_program) > 1: 363 _LOG.error('--substep not supported if there are multiple steps') 364 return 1 365 366 if presubmit.run( 367 final_program, 368 root, 369 repositories, 370 only_list_steps=only_list_steps, 371 output_directory=output_directory, 372 package_root=package_root, 373 substep=substep, 374 dry_run=dry_run, 375 **other_args, 376 ): 377 return 0 378 379 # Check if this failed presumbit was run as a Git hook by looking for GIT_* 380 # environment variables. Mention using --no-verify to skip if so. 381 for env_var in os.environ: 382 if env_var.startswith('GIT'): 383 _LOG.info( 384 'To skip these checks and continue with this push, ' 385 'add --no-verify to the git command' 386 ) 387 break 388 389 return 1 390