1#!/usr/bin/env python3 2# Copyright 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Repo pre-upload hook. 17 18Normally this is loaded indirectly by repo itself, but it can be run directly 19when developing. 20""" 21 22import argparse 23import concurrent.futures 24import datetime 25import os 26import signal 27import sys 28from typing import List, Optional 29 30 31# Assert some minimum Python versions as we don't test or support any others. 32if sys.version_info < (3, 6): 33 print('repohooks: error: Python-3.6+ is required', file=sys.stderr) 34 sys.exit(1) 35 36 37_path = os.path.dirname(os.path.realpath(__file__)) 38if sys.path[0] != _path: 39 sys.path.insert(0, _path) 40del _path 41 42# We have to import our local modules after the sys.path tweak. We can't use 43# relative imports because this is an executable program, not a module. 44# pylint: disable=wrong-import-position 45import rh 46import rh.results 47import rh.config 48import rh.git 49import rh.hooks 50import rh.terminal 51import rh.utils 52 53 54# Repohooks homepage. 55REPOHOOKS_URL = 'https://android.googlesource.com/platform/tools/repohooks/' 56 57 58class Output(object): 59 """Class for reporting hook status.""" 60 61 COLOR = rh.terminal.Color() 62 COMMIT = COLOR.color(COLOR.CYAN, 'COMMIT') 63 RUNNING = COLOR.color(COLOR.YELLOW, 'RUNNING') 64 PASSED = COLOR.color(COLOR.GREEN, 'PASSED') 65 FAILED = COLOR.color(COLOR.RED, 'FAILED') 66 WARNING = COLOR.color(COLOR.YELLOW, 'WARNING') 67 FIXUP = COLOR.color(COLOR.MAGENTA, 'FIXUP') 68 69 # How long a hook is allowed to run before we warn that it is "too slow". 70 _SLOW_HOOK_DURATION = datetime.timedelta(seconds=30) 71 72 def __init__(self, project_name): 73 """Create a new Output object for a specified project. 74 75 Args: 76 project_name: name of project. 77 """ 78 self.project_name = project_name 79 self.hooks = None 80 self.num_hooks = None 81 self.num_commits = None 82 self.commit_index = 0 83 self.success = True 84 self.start_time = datetime.datetime.now() 85 self.hook_start_time = None 86 # Cache number of invisible characters in our banner. 87 self._banner_esc_chars = len(self.COLOR.color(self.COLOR.YELLOW, '')) 88 89 def set_num_commits(self, num_commits: int) -> None: 90 """Keep track of how many commits we'll be running. 91 92 Args: 93 num_commits: Number of commits to be run. 94 """ 95 self.num_commits = num_commits 96 self.commit_index = 1 97 98 def commit_start(self, hooks, commit, commit_summary): 99 """Emit status for new commit. 100 101 Args: 102 hooks: All the hooks to be run for this commit. 103 commit: commit hash. 104 commit_summary: commit summary. 105 """ 106 status_line = ( 107 f'[{self.COMMIT} ' 108 f'{self.commit_index}/{self.num_commits} ' 109 f'{commit[0:12]}] {commit_summary}' 110 ) 111 rh.terminal.print_status_line(status_line, print_newline=True) 112 self.commit_index += 1 113 114 # Initialize the pending hooks line too. 115 self.hooks = set(hooks) 116 self.num_hooks = len(hooks) 117 self.hook_banner() 118 119 def hook_banner(self): 120 """Display the banner for current set of hooks.""" 121 pending = ', '.join(x.name for x in self.hooks) 122 status_line = ( 123 f'[{self.RUNNING} ' 124 f'{self.num_hooks - len(self.hooks)}/{self.num_hooks}] ' 125 f'{pending}' 126 ) 127 if self._banner_esc_chars and sys.stderr.isatty(): 128 cols = os.get_terminal_size(sys.stderr.fileno()).columns 129 status_line = status_line[0:cols + self._banner_esc_chars] 130 rh.terminal.print_status_line(status_line) 131 132 def hook_finish(self, hook, duration): 133 """Finish processing any per-hook state.""" 134 self.hooks.remove(hook) 135 if duration >= self._SLOW_HOOK_DURATION: 136 d = rh.utils.timedelta_str(duration) 137 self.hook_warning( 138 hook, 139 f'This hook took {d} to finish which is fairly slow for ' 140 'developers.\nPlease consider moving the check to the ' 141 'server/CI system instead.') 142 143 # Show any hooks still pending. 144 if self.hooks: 145 self.hook_banner() 146 147 def hook_error(self, hook, error): 148 """Print an error for a single hook. 149 150 Args: 151 hook: The hook that generated the output. 152 error: error string. 153 """ 154 self.error(f'{hook.name} hook', error) 155 156 def hook_warning(self, hook, warning): 157 """Print a warning for a single hook. 158 159 Args: 160 hook: The hook that generated the output. 161 warning: warning string. 162 """ 163 status_line = f'[{self.WARNING}] {hook.name}' 164 rh.terminal.print_status_line(status_line, print_newline=True) 165 print(warning, file=sys.stderr) 166 167 def error(self, header, error): 168 """Print a general error. 169 170 Args: 171 header: A unique identifier for the source of this error. 172 error: error string. 173 """ 174 status_line = f'[{self.FAILED}] {header}' 175 rh.terminal.print_status_line(status_line, print_newline=True) 176 print(error, file=sys.stderr) 177 self.success = False 178 179 def hook_fixups( 180 self, 181 project_results: rh.results.ProjectResults, 182 hook_results: List[rh.results.HookResult], 183 ) -> None: 184 """Display summary of possible fixups for a single hook.""" 185 for result in (x for x in hook_results if x.fixup_cmd): 186 cmd = result.fixup_cmd + list(result.files) 187 for line in ( 188 f'[{self.FIXUP}] {result.hook} has automated fixups available', 189 f' cd {rh.shell.quote(project_results.workdir)} && \\', 190 f' {rh.shell.cmd_to_str(cmd)}', 191 ): 192 rh.terminal.print_status_line(line, print_newline=True) 193 194 def finish(self): 195 """Print summary for all the hooks.""" 196 header = self.PASSED if self.success else self.FAILED 197 status = 'passed' if self.success else 'failed' 198 d = rh.utils.timedelta_str(datetime.datetime.now() - self.start_time) 199 rh.terminal.print_status_line( 200 f'[{header}] repohooks for {self.project_name} {status} in {d}', 201 print_newline=True) 202 203 204def _process_hook_results(results): 205 """Returns an error string if an error occurred. 206 207 Args: 208 results: A list of HookResult objects, or None. 209 210 Returns: 211 error output if an error occurred, otherwise None 212 warning output if an error occurred, otherwise None 213 """ 214 if not results: 215 return (None, None) 216 217 # We track these as dedicated fields in case a hook doesn't output anything. 218 # We want to treat silent non-zero exits as failures too. 219 has_error = False 220 has_warning = False 221 222 error_ret = '' 223 warning_ret = '' 224 for result in results: 225 if result or result.is_warning(): 226 ret = '' 227 if result.files: 228 ret += f' FILES: {rh.shell.cmd_to_str(result.files)}\n' 229 lines = result.error.splitlines() 230 ret += '\n'.join(f' {x}' for x in lines) 231 if result.is_warning(): 232 has_warning = True 233 warning_ret += ret 234 else: 235 has_error = True 236 error_ret += ret 237 238 return (error_ret if has_error else None, 239 warning_ret if has_warning else None) 240 241 242def _get_project_config(from_git=False): 243 """Returns the configuration for a project. 244 245 Args: 246 from_git: If true, we are called from git directly and repo should not be 247 used. 248 Expects to be called from within the project root. 249 """ 250 if from_git: 251 global_paths = (rh.git.find_repo_root(),) 252 else: 253 global_paths = ( 254 # Load the global config found in the manifest repo. 255 (os.path.join(rh.git.find_repo_root(), '.repo', 'manifests')), 256 # Load the global config found in the root of the repo checkout. 257 rh.git.find_repo_root(), 258 ) 259 260 paths = ( 261 # Load the config for this git repo. 262 '.', 263 ) 264 return rh.config.PreUploadSettings(paths=paths, global_paths=global_paths) 265 266 267def _attempt_fixes(projects_results: List[rh.results.ProjectResults]) -> None: 268 """Attempts to fix fixable results.""" 269 # Filter out any result that has a fixup. 270 fixups = [] 271 for project_results in projects_results: 272 fixups.extend((project_results.workdir, x) 273 for x in project_results.fixups) 274 if not fixups: 275 return 276 277 if len(fixups) > 1: 278 banner = f'Multiple fixups ({len(fixups)}) are available.' 279 else: 280 banner = 'Automated fixups are available.' 281 print(Output.COLOR.color(Output.COLOR.MAGENTA, banner), file=sys.stderr) 282 283 # If there's more than one fixup available, ask if they want to blindly run 284 # them all, or prompt for them one-by-one. 285 mode = 'some' 286 if len(fixups) > 1: 287 while True: 288 response = rh.terminal.str_prompt( 289 'What would you like to do', 290 ('Run (A)ll', 'Run (S)ome', '(D)ry-run', '(N)othing [default]')) 291 if not response: 292 print('', file=sys.stderr) 293 return 294 if response.startswith('a') or response.startswith('y'): 295 mode = 'all' 296 break 297 elif response.startswith('s'): 298 mode = 'some' 299 break 300 elif response.startswith('d'): 301 mode = 'dry-run' 302 break 303 elif response.startswith('n'): 304 print('', file=sys.stderr) 305 return 306 307 # Walk all the fixups and run them one-by-one. 308 for workdir, result in fixups: 309 if mode == 'some': 310 if not rh.terminal.boolean_prompt( 311 f'Run {result.hook} fixup for {result.commit}' 312 ): 313 continue 314 315 cmd = tuple(result.fixup_cmd) + tuple(result.files) 316 print( 317 f'\n[{Output.RUNNING}] cd {rh.shell.quote(workdir)} && ' 318 f'{rh.shell.cmd_to_str(cmd)}', file=sys.stderr) 319 if mode == 'dry-run': 320 continue 321 322 cmd_result = rh.utils.run(cmd, cwd=workdir, check=False) 323 if cmd_result.returncode: 324 print(f'[{Output.WARNING}] command exited {cmd_result.returncode}', 325 file=sys.stderr) 326 else: 327 print(f'[{Output.PASSED}] great success', file=sys.stderr) 328 329 print(f'\n[{Output.FIXUP}] Please amend & rebase your tree before ' 330 'attempting to upload again.\n', file=sys.stderr) 331 332def _run_project_hooks_in_cwd( 333 project_name: str, 334 proj_dir: str, 335 output: Output, 336 jobs: Optional[int] = None, 337 from_git: bool = False, 338 commit_list: Optional[List[str]] = None, 339) -> rh.results.ProjectResults: 340 """Run the project-specific hooks in the cwd. 341 342 Args: 343 project_name: The name of this project. 344 proj_dir: The directory for this project (for passing on in metadata). 345 output: Helper for summarizing output/errors to the user. 346 jobs: How many hooks to run in parallel. 347 from_git: If true, we are called from git directly and repo should not be 348 used. 349 commit_list: A list of commits to run hooks against. If None or empty 350 list then we'll automatically get the list of commits that would be 351 uploaded. 352 353 Returns: 354 All the results for this project. 355 """ 356 ret = rh.results.ProjectResults(project_name, proj_dir) 357 358 try: 359 config = _get_project_config(from_git) 360 except rh.config.ValidationError as e: 361 output.error('Loading config files', str(e)) 362 return ret._replace(internal_failure=True) 363 364 builtin_hooks = list(config.callable_builtin_hooks()) 365 custom_hooks = list(config.callable_custom_hooks()) 366 367 # If the repo has no pre-upload hooks enabled, then just return. 368 if not builtin_hooks and not custom_hooks: 369 return ret 370 371 # Set up the environment like repo would with the forall command. 372 try: 373 remote = rh.git.get_upstream_remote() 374 upstream_branch = rh.git.get_upstream_branch() 375 except rh.utils.CalledProcessError as e: 376 output.error('Upstream remote/tracking branch lookup', 377 f'{e}\nDid you run repo start? Is your HEAD detached?') 378 return ret._replace(internal_failure=True) 379 380 project = rh.Project(name=project_name, dir=proj_dir) 381 rel_proj_dir = os.path.relpath(proj_dir, rh.git.find_repo_root()) 382 383 # Filter out the hooks to process. 384 builtin_hooks = [x for x in builtin_hooks if rel_proj_dir not in x.scope] 385 custom_hooks = [x for x in custom_hooks if rel_proj_dir not in x.scope] 386 387 if not builtin_hooks and not custom_hooks: 388 return ret 389 390 os.environ.update({ 391 'REPO_LREV': rh.git.get_commit_for_ref(upstream_branch), 392 'REPO_PATH': rel_proj_dir, 393 'REPO_PROJECT': project_name, 394 'REPO_REMOTE': remote, 395 'REPO_RREV': rh.git.get_remote_revision(upstream_branch, remote), 396 }) 397 398 if not commit_list: 399 commit_list = rh.git.get_commits( 400 ignore_merged_commits=config.ignore_merged_commits) 401 output.set_num_commits(len(commit_list)) 402 403 def _run_hook(hook, project, commit, desc, diff): 404 """Run a hook, gather stats, and process its results.""" 405 start = datetime.datetime.now() 406 results = hook.hook(project, commit, desc, diff) 407 (error, warning) = _process_hook_results(results) 408 duration = datetime.datetime.now() - start 409 return (hook, results, error, warning, duration) 410 411 with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as executor: 412 for commit in commit_list: 413 # Mix in some settings for our hooks. 414 os.environ['PREUPLOAD_COMMIT'] = commit 415 diff = rh.git.get_affected_files(commit) 416 desc = rh.git.get_commit_desc(commit) 417 os.environ['PREUPLOAD_COMMIT_MESSAGE'] = desc 418 419 commit_summary = desc.split('\n', 1)[0] 420 output.commit_start(builtin_hooks + custom_hooks, commit, commit_summary) 421 422 def run_hooks(hooks): 423 futures = ( 424 executor.submit(_run_hook, hook, project, commit, desc, diff) 425 for hook in hooks 426 ) 427 future_results = ( 428 x.result() for x in concurrent.futures.as_completed(futures) 429 ) 430 for hook, hook_results, error, warning, duration in future_results: 431 ret.add_results(hook_results) 432 if error is not None or warning is not None: 433 if warning is not None: 434 output.hook_warning(hook, warning) 435 if error is not None: 436 output.hook_error(hook, error) 437 output.hook_fixups(ret, hook_results) 438 output.hook_finish(hook, duration) 439 440 run_hooks(builtin_hooks) 441 run_hooks(custom_hooks) 442 443 return ret 444 445 446def _run_project_hooks( 447 project_name: str, 448 proj_dir: Optional[str] = None, 449 jobs: Optional[int] = None, 450 from_git: bool = False, 451 commit_list: Optional[List[str]] = None, 452) -> rh.results.ProjectResults: 453 """Run the project-specific hooks in |proj_dir|. 454 455 Args: 456 project_name: The name of project to run hooks for. 457 proj_dir: If non-None, this is the directory the project is in. If None, 458 we'll ask repo. 459 jobs: How many hooks to run in parallel. 460 from_git: If true, we are called from git directly and repo should not be 461 used. 462 commit_list: A list of commits to run hooks against. If None or empty 463 list then we'll automatically get the list of commits that would be 464 uploaded. 465 466 Returns: 467 All the results for this project. 468 """ 469 output = Output(project_name) 470 471 if proj_dir is None: 472 cmd = ['repo', 'forall', project_name, '-c', 'pwd'] 473 result = rh.utils.run(cmd, capture_output=True) 474 proj_dirs = result.stdout.split() 475 if not proj_dirs: 476 print(f'{project_name} cannot be found.', file=sys.stderr) 477 print('Please specify a valid project.', file=sys.stderr) 478 return False 479 if len(proj_dirs) > 1: 480 print(f'{project_name} is associated with multiple directories.', 481 file=sys.stderr) 482 print('Please specify a directory to help disambiguate.', 483 file=sys.stderr) 484 return False 485 proj_dir = proj_dirs[0] 486 487 pwd = os.getcwd() 488 try: 489 # Hooks assume they are run from the root of the project. 490 os.chdir(proj_dir) 491 return _run_project_hooks_in_cwd( 492 project_name, proj_dir, output, jobs=jobs, from_git=from_git, 493 commit_list=commit_list) 494 finally: 495 output.finish() 496 os.chdir(pwd) 497 498 499def _run_projects_hooks( 500 project_list: List[str], 501 worktree_list: List[Optional[str]], 502 jobs: Optional[int] = None, 503 from_git: bool = False, 504 commit_list: Optional[List[str]] = None, 505) -> bool: 506 """Run all the hooks 507 508 Args: 509 project_list: List of project names. 510 worktree_list: List of project checkouts. 511 jobs: How many hooks to run in parallel. 512 from_git: If true, we are called from git directly and repo should not be 513 used. 514 commit_list: A list of commits to run hooks against. If None or empty 515 list then we'll automatically get the list of commits that would be 516 uploaded. 517 518 Returns: 519 True if everything passed, else False. 520 """ 521 results = [] 522 for project, worktree in zip(project_list, worktree_list): 523 result = _run_project_hooks( 524 project, 525 proj_dir=worktree, 526 jobs=jobs, 527 from_git=from_git, 528 commit_list=commit_list, 529 ) 530 results.append(result) 531 if result: 532 # If a repo had failures, add a blank line to help break up the 533 # output. If there were no failures, then the output should be 534 # very minimal, so we don't add it then. 535 print('', file=sys.stderr) 536 537 _attempt_fixes(results) 538 return not any(results) 539 540 541def main(project_list, worktree_list=None, **_kwargs): 542 """Main function invoked directly by repo. 543 544 We must use the name "main" as that is what repo requires. 545 546 This function will exit directly upon error so that repo doesn't print some 547 obscure error message. 548 549 Args: 550 project_list: List of projects to run on. 551 worktree_list: A list of directories. It should be the same length as 552 project_list, so that each entry in project_list matches with a 553 directory in worktree_list. If None, we will attempt to calculate 554 the directories automatically. 555 kwargs: Leave this here for forward-compatibility. 556 """ 557 if not worktree_list: 558 worktree_list = [None] * len(project_list) 559 if not _run_projects_hooks(project_list, worktree_list): 560 color = rh.terminal.Color() 561 print(color.color(color.RED, 'FATAL') + 562 ': Preupload failed due to above error(s).\n' 563 f'For more info, see: {REPOHOOKS_URL}', 564 file=sys.stderr) 565 sys.exit(1) 566 567 568def _identify_project(path, from_git=False): 569 """Identify the repo project associated with the given path. 570 571 Returns: 572 A string indicating what project is associated with the path passed in or 573 a blank string upon failure. 574 """ 575 if from_git: 576 cmd = ['git', 'rev-parse', '--show-toplevel'] 577 project_path = rh.utils.run(cmd, capture_output=True).stdout.strip() 578 cmd = ['git', 'rev-parse', '--show-superproject-working-tree'] 579 superproject_path = rh.utils.run( 580 cmd, capture_output=True).stdout.strip() 581 module_path = project_path[len(superproject_path) + 1:] 582 cmd = ['git', 'config', '-f', '.gitmodules', 583 '--name-only', '--get-regexp', r'^submodule\..*\.path$', 584 f"^{module_path}$"] 585 module_name = rh.utils.run(cmd, cwd=superproject_path, 586 capture_output=True).stdout.strip() 587 return module_name[len('submodule.'):-len(".path")] 588 else: 589 cmd = ['repo', 'forall', '.', '-c', 'echo ${REPO_PROJECT}'] 590 return rh.utils.run(cmd, capture_output=True, cwd=path).stdout.strip() 591 592 593def direct_main(argv): 594 """Run hooks directly (outside of the context of repo). 595 596 Args: 597 argv: The command line args to process. 598 599 Returns: 600 0 if no pre-upload failures, 1 if failures. 601 602 Raises: 603 BadInvocation: On some types of invocation errors. 604 """ 605 parser = argparse.ArgumentParser(description=__doc__) 606 parser.add_argument('--git', action='store_true', 607 help='This hook is called from git instead of repo') 608 parser.add_argument('--dir', default=None, 609 help='The directory that the project lives in. If not ' 610 'specified, use the git project root based on the cwd.') 611 parser.add_argument('--project', default=None, 612 help='The project repo path; this can affect how the ' 613 'hooks get run, since some hooks are project-specific.' 614 'If not specified, `repo` will be used to figure this ' 615 'out based on the dir.') 616 parser.add_argument('-j', '--jobs', type=int, 617 help='Run up to this many hooks in parallel. Setting ' 618 'to 1 forces serial execution, and the default ' 619 'automatically chooses an appropriate number for the ' 620 'current system.') 621 parser.add_argument('commits', nargs='*', 622 help='Check specific commits') 623 opts = parser.parse_args(argv) 624 625 # Check/normalize git dir; if unspecified, we'll use the root of the git 626 # project from CWD. 627 if opts.dir is None: 628 cmd = ['git', 'rev-parse', '--git-dir'] 629 git_dir = rh.utils.run(cmd, capture_output=True).stdout.strip() 630 if not git_dir: 631 parser.error('The current directory is not part of a git project.') 632 opts.dir = os.path.dirname(os.path.abspath(git_dir)) 633 elif not os.path.isdir(opts.dir): 634 parser.error(f'Invalid dir: {opts.dir}') 635 elif not rh.git.is_git_repository(opts.dir): 636 parser.error(f'Not a git repository: {opts.dir}') 637 638 # Identify the project if it wasn't specified; this _requires_ the repo 639 # tool to be installed and for the project to be part of a repo checkout. 640 if not opts.project: 641 opts.project = _identify_project(opts.dir, opts.git) 642 if not opts.project: 643 parser.error(f"Couldn't identify the project of {opts.dir}") 644 645 try: 646 if _run_projects_hooks([opts.project], [opts.dir], jobs=opts.jobs, 647 from_git=opts.git, commit_list=opts.commits): 648 return 0 649 except KeyboardInterrupt: 650 print('Aborting execution early due to user interrupt', file=sys.stderr) 651 return 128 + signal.SIGINT 652 return 1 653 654 655if __name__ == '__main__': 656 sys.exit(direct_main(sys.argv[1:])) 657