1#!/usr/bin/env python3 2 3# Copyright 2020 The Pigweed Authors 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may not 6# use this file except in compliance with the License. You may obtain a copy of 7# the License at 8# 9# https://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations under 15# the License. 16"""Create transitive CLs for requirements on internal Gerrits. 17 18This is only intended to be used by Googlers. 19 20If the current CL needs to be tested alongside internal-project:1234 on an 21internal project, but "internal-project" is something that can't be referenced 22publicly, this automates creation of a CL on the pigweed-internal Gerrit that 23references internal-project:1234 so the current commit effectively has a 24requirement on internal-project:1234. 25 26For more see http://go/pigweed-ci-cq-intro. 27""" 28 29import argparse 30import dataclasses 31import json 32import logging 33import os 34from pathlib import Path 35import re 36import subprocess 37import sys 38import tempfile 39from typing import Callable, IO, Sequence 40import uuid 41 42HELPER_GERRIT = 'pigweed-internal' 43HELPER_PROJECT = 'requires-helper' 44HELPER_REPO = 'sso://{}/{}'.format(HELPER_GERRIT, HELPER_PROJECT) 45 46# Pass checks that look for "DO NOT ..." and block submission. 47_DNS = ' '.join( 48 ( 49 'DO', 50 'NOT', 51 'SUBMIT', 52 ) 53) 54 55# Subset of the output from pushing to Gerrit. 56DEFAULT_OUTPUT = f''' 57remote: 58remote: https://{HELPER_GERRIT}-review.git.corp.google.com/c/{HELPER_PROJECT}/+/123456789 {_DNS} [NEW] 59remote: 60'''.strip() 61 62_LOG = logging.getLogger(__name__) 63 64 65@dataclasses.dataclass 66class Change: 67 gerrit_name: str 68 number: int 69 70 71class EnhancedJSONEncoder(json.JSONEncoder): 72 def default(self, o): 73 if dataclasses.is_dataclass(o): 74 return dataclasses.asdict(o) 75 return super().default(o) 76 77 78def dump_json_patches(obj: Sequence[Change], outs: IO): 79 json.dump(obj, outs, indent=2, cls=EnhancedJSONEncoder) 80 81 82def log_entry_exit(func: Callable) -> Callable: 83 def wrapper(*args, **kwargs): 84 _LOG.debug('entering %s()', func.__name__) 85 _LOG.debug('args %r', args) 86 _LOG.debug('kwargs %r', kwargs) 87 try: 88 res = func(*args, **kwargs) 89 _LOG.debug('return value %r', res) 90 return res 91 except Exception as exc: 92 _LOG.debug('exception %r', exc) 93 raise 94 finally: 95 _LOG.debug('exiting %s()', func.__name__) 96 97 return wrapper 98 99 100@log_entry_exit 101def parse_args() -> argparse.Namespace: 102 """Creates an argument parser and parses arguments.""" 103 104 parser = argparse.ArgumentParser(description=__doc__) 105 parser.add_argument( 106 'requirements', 107 nargs='+', 108 help='Requirements to be added ("<gerrit-name>:<cl-number>").', 109 ) 110 parser.add_argument( 111 '--push', 112 action=argparse.BooleanOptionalAction, 113 default=True, 114 help=argparse.SUPPRESS, # This option is only for debugging. 115 ) 116 117 return parser.parse_args() 118 119 120@log_entry_exit 121def _run_command(*args, **kwargs) -> subprocess.CompletedProcess: 122 kwargs.setdefault('capture_output', True) 123 _LOG.debug('%s', args) 124 _LOG.debug('%s', kwargs) 125 res = subprocess.run(*args, **kwargs) 126 _LOG.debug('%s', res.stdout) 127 _LOG.debug('%s', res.stderr) 128 res.check_returncode() 129 return res 130 131 132@log_entry_exit 133def check_status() -> bool: 134 res = subprocess.run(['git', 'status'], capture_output=True) 135 if res.returncode: 136 _LOG.error('repository not clean, commit to suppress this warning') 137 return False 138 return True 139 140 141@log_entry_exit 142def clone(requires_dir: Path) -> None: 143 _LOG.info('cloning helper repository into %s', requires_dir) 144 _run_command(['git', 'clone', HELPER_REPO, '.'], cwd=requires_dir) 145 146 147@log_entry_exit 148def create_commit( 149 requires_dir: Path, requirement_strings: Sequence[str] 150) -> None: 151 """Create a commit in the local tree with the given requirements.""" 152 change_id = str(uuid.uuid4()).replace('-', '00') 153 _LOG.debug('change_id %s', change_id) 154 155 requirement_objects: list[Change] = [] 156 for req in requirement_strings: 157 gerrit_name, number = req.split(':', 1) 158 requirement_objects.append(Change(gerrit_name, int(number))) 159 160 path = requires_dir / 'patches.json' 161 _LOG.debug('path %s', path) 162 with open(path, 'w') as outs: 163 dump_json_patches(requirement_objects, outs) 164 outs.write('\n') 165 166 _run_command(['git', 'add', path], cwd=requires_dir) 167 168 # TODO: b/232234662 - Don't add 'Requires:' lines to commit messages. 169 commit_message = [ 170 f'{_DNS} {change_id[0:10]}\n\n', 171 '', 172 f'Change-Id: I{change_id}', 173 ] 174 for req in requirement_strings: 175 commit_message.append(f'Requires: {req}') 176 177 _LOG.debug('message %s', commit_message) 178 _run_command( 179 ['git', 'commit', '-m', '\n'.join(commit_message)], 180 cwd=requires_dir, 181 ) 182 183 # Not strictly necessary, only used for logging. 184 _run_command(['git', 'show'], cwd=requires_dir) 185 186 187@log_entry_exit 188def push_commit(requires_dir: Path, push=True) -> Change: 189 """Push a commit to the helper repository. 190 191 Args: 192 requires_dir: Local checkout of the helper repository. 193 push: Whether to actually push or if this is a local-only test. 194 195 Returns a Change object referencing the pushed commit. 196 """ 197 198 output: str = DEFAULT_OUTPUT 199 if push: 200 res = _run_command( 201 ['git', 'push', HELPER_REPO, '+HEAD:refs/for/main'], 202 cwd=requires_dir, 203 ) 204 output = res.stderr.decode() 205 206 _LOG.debug('output: %s', output) 207 regex = re.compile( 208 f'^\\s*remote:\\s*' 209 f'https://{HELPER_GERRIT}-review.(?:git.corp.google|googlesource).com/' 210 f'c/{HELPER_PROJECT}/\\+/(?P<num>\\d+)\\s+', 211 re.MULTILINE, 212 ) 213 _LOG.debug('regex %r', regex) 214 match = regex.search(output) 215 if not match: 216 raise ValueError(f"invalid output from 'git push': {output}") 217 change_num = int(match.group('num')) 218 _LOG.info('created %s change %s', HELPER_PROJECT, change_num) 219 return Change(HELPER_GERRIT, change_num) 220 221 222@log_entry_exit 223def amend_existing_change(dependency: dict[str, str]) -> None: 224 """Amend the current change to depend on the dependency 225 226 Args: 227 dependency: The change on which the top of the current checkout now 228 depends. 229 """ 230 git_root = Path( 231 subprocess.run( 232 ['git', 'rev-parse', '--show-toplevel'], 233 capture_output=True, 234 ) 235 .stdout.decode() 236 .rstrip('\n') 237 ) 238 patches_json = git_root / 'patches.json' 239 _LOG.info('%s %d', patches_json, os.path.isfile(patches_json)) 240 241 patches = [] 242 if os.path.isfile(patches_json): 243 with open(patches_json, 'r') as ins: 244 patches = json.load(ins) 245 246 patches.append(dependency) 247 with open(patches_json, 'w') as outs: 248 dump_json_patches(patches, outs) 249 outs.write('\n') 250 _LOG.info('%s %d', patches_json, os.path.isfile(patches_json)) 251 252 _run_command(['git', 'add', patches_json]) 253 _run_command(['git', 'commit', '--amend', '--no-edit']) 254 255 256def run(requirements: Sequence[str], push: bool = True) -> int: 257 """Entry point for requires.""" 258 259 if not check_status(): 260 return -1 261 262 # Create directory for checking out helper repository. 263 with tempfile.TemporaryDirectory() as requires_dir_str: 264 requires_dir = Path(requires_dir_str) 265 # Clone into helper repository. 266 clone(requires_dir) 267 # Make commit with requirements from command line. 268 create_commit(requires_dir, requirements) 269 # Push that commit and save its number. 270 change = push_commit(requires_dir, push=push) 271 # Add dependency on newly pushed commit on current commit. 272 amend_existing_change(change) 273 274 return 0 275 276 277def main() -> int: 278 return run(**vars(parse_args())) 279 280 281if __name__ == '__main__': 282 try: 283 # If pw_cli is available, use it to initialize logs. 284 from pw_cli import log 285 286 log.install(logging.INFO) 287 except ImportError: 288 # If pw_cli isn't available, display log messages like a simple print. 289 logging.basicConfig(format='%(message)s', level=logging.INFO) 290 291 sys.exit(main()) 292