1#!/usr/bin/env vpython3 2 3# Copyright (c) 2017 The LibYUV project authors. All Rights Reserved. 4# 5# Use of this source code is governed by a BSD-style license 6# that can be found in the LICENSE file in the root of the source 7# tree. An additional intellectual property rights grant can be found 8# in the file PATENTS. All contributing project authors may 9# be found in the AUTHORS file in the root of the source tree. 10"""Script to automatically roll dependencies in the LibYUV DEPS file.""" 11 12 13import argparse 14import base64 15import collections 16import logging 17import os 18import re 19import subprocess 20import sys 21import urllib.request 22 23 24def FindSrcDirPath(): 25 """Returns the abs path to the src/ dir of the project.""" 26 src_dir = os.path.dirname(os.path.abspath(__file__)) 27 while os.path.basename(src_dir) != 'src': 28 src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) 29 return src_dir 30 31 32# Skip these dependencies (list without solution name prefix). 33DONT_AUTOROLL_THESE = [ 34 'third_party/fuchsia-gn-sdk', 35 'src/third_party/gflags/src', 36 'src/third_party/mockito/src', 37] 38 39# These dependencies are missing in chromium/src/DEPS, either unused or already 40# in-tree. For instance, src/base is a part of the Chromium source git repo, 41# but we pull it through a subtree mirror, so therefore it isn't listed in 42# Chromium's deps but it is in ours. 43LIBYUV_ONLY_DEPS = [ 44 'src/base', 45 'src/build', 46 'src/buildtools', 47 'src/ios', 48 'src/testing', 49 'src/third_party', 50 'src/third_party/android_support_test_runner', 51 'src/third_party/bazel', 52 'src/third_party/bouncycastle', 53 'src/third_party/errorprone/lib', 54 'src/third_party/findbugs', 55 'src/third_party/gson', 56 'src/third_party/gtest-parallel', 57 'src/third_party/guava', 58 'src/third_party/intellij', 59 'src/third_party/jsr-305/src', 60 'src/third_party/ow2_asm', 61 'src/third_party/proguard', 62 'src/third_party/ub-uiautomator/lib', 63 'src/tools', 64 'src/tools/clang/dsymutil', 65] 66 67LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' 68CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 69CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 70CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 71CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 72 73COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 74CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$') 75ROLL_BRANCH_NAME = 'roll_chromium_revision' 76 77SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 78CHECKOUT_SRC_DIR = FindSrcDirPath() 79CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) 80 81# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. 82ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' 83ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' 84# Location of automically gathered android deps. 85ANDROID_DEPS_PATH = 'src/third_party/android_deps/' 86 87sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 88import find_depot_tools 89 90find_depot_tools.add_depot_tools_to_path() 91 92CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' 93CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 94 'clang', 'scripts', 'update.py') 95 96DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 97ChangedDep = collections.namedtuple('ChangedDep', 98 'path url current_rev new_rev') 99CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') 100VersionEntry = collections.namedtuple('VersionEntry', 'version') 101ChangedCipdPackage = collections.namedtuple( 102 'ChangedCipdPackage', 'path package current_version new_version') 103ChangedVersionEntry = collections.namedtuple( 104 'ChangedVersionEntry', 'path current_version new_version') 105 106ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', 107 ('current_chromium_rev ' 108 'new_chromium_rev ')) 109 110 111class RollError(Exception): 112 pass 113 114 115def StrExpansion(): 116 return lambda str_value: str_value 117 118 119def VarLookup(local_scope): 120 return lambda var_name: local_scope['vars'][var_name] 121 122 123def ParseDepsDict(deps_content): 124 local_scope = {} 125 global_scope = { 126 'Str': StrExpansion(), 127 'Var': VarLookup(local_scope), 128 'deps_os': {}, 129 } 130 exec(deps_content, global_scope, local_scope) 131 return local_scope 132 133 134def ParseLocalDepsFile(filename): 135 with open(filename, 'rb') as f: 136 deps_content = f.read().decode('utf-8') 137 return ParseDepsDict(deps_content) 138 139 140def ParseCommitPosition(commit_message): 141 for line in reversed(commit_message.splitlines()): 142 m = COMMIT_POSITION_RE.match(line.strip()) 143 if m: 144 return int(m.group(1)) 145 logging.error('Failed to parse commit position id from:\n%s\n', 146 commit_message) 147 sys.exit(-1) 148 149 150def _RunCommand(command, 151 working_dir=None, 152 ignore_exit_code=False, 153 extra_env=None, 154 input_data=None): 155 """Runs a command and returns the output from that command. 156 157 If the command fails (exit code != 0), the function will exit the process. 158 159 Returns: 160 A tuple containing the stdout and stderr outputs as strings. 161 """ 162 working_dir = working_dir or CHECKOUT_SRC_DIR 163 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 164 env = os.environ.copy() 165 if extra_env: 166 assert all(isinstance(value, str) for value in extra_env.values()) 167 logging.debug('extra env: %s', extra_env) 168 env.update(extra_env) 169 p = subprocess.Popen(command, 170 stdin=subprocess.PIPE, 171 stdout=subprocess.PIPE, 172 stderr=subprocess.PIPE, 173 env=env, 174 cwd=working_dir, 175 universal_newlines=True) 176 std_output, err_output = p.communicate(input_data) 177 p.stdout.close() 178 p.stderr.close() 179 if not ignore_exit_code and p.returncode != 0: 180 logging.error('Command failed: %s\n' 181 'stdout:\n%s\n' 182 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 183 sys.exit(p.returncode) 184 return std_output, err_output 185 186 187def _GetBranches(): 188 """Returns a tuple of active,branches. 189 190 The 'active' is the name of the currently active branch and 'branches' is a 191 list of all branches. 192 """ 193 lines = _RunCommand(['git', 'branch'])[0].split('\n') 194 branches = [] 195 active = '' 196 for line in lines: 197 if '*' in line: 198 # The assumption is that the first char will always be the '*'. 199 active = line[1:].strip() 200 branches.append(active) 201 else: 202 branch = line.strip() 203 if branch: 204 branches.append(branch) 205 return active, branches 206 207 208def _ReadGitilesContent(url): 209 # Download and decode BASE64 content until 210 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 211 base64_content = ReadUrlContent(url + '?format=TEXT') 212 return base64.b64decode(base64_content[0]).decode('utf-8') 213 214 215def ReadRemoteCrFile(path_below_src, revision): 216 """Reads a remote Chromium file of a specific revision. 217 218 Args: 219 path_below_src: A path to the target file relative to src dir. 220 revision: Revision to read. 221 Returns: 222 A string with file content. 223 """ 224 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % 225 (revision, path_below_src)) 226 227 228def ReadRemoteCrCommit(revision): 229 """Reads a remote Chromium commit message. Returns a string.""" 230 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 231 232 233def ReadUrlContent(url): 234 """Connect to a remote host and read the contents. 235 236 Args: 237 url: URL to connect to. 238 Returns: 239 A list of lines. 240 """ 241 conn = urllib.request.urlopen(url) 242 try: 243 return conn.readlines() 244 except IOError as e: 245 logging.exception('Error connecting to %s. Error: %s', url, e) 246 raise 247 finally: 248 conn.close() 249 250 251def GetMatchingDepsEntries(depsentry_dict, dir_path): 252 """Gets all deps entries matching the provided path. 253 254 This list may contain more than one DepsEntry object. 255 Example: dir_path='src/testing' would give results containing both 256 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's 257 DEPS. 258 Example 2: dir_path='src/build' should return 'src/build' but not 259 'src/buildtools'. 260 261 Returns: 262 A list of DepsEntry objects. 263 """ 264 result = [] 265 for path, depsentry in depsentry_dict.items(): 266 if path == dir_path: 267 result.append(depsentry) 268 else: 269 parts = path.split('/') 270 if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): 271 result.append(depsentry) 272 return result 273 274 275def BuildDepsentryDict(deps_dict): 276 """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" 277 result = {} 278 279 def AddDepsEntries(deps_subdict): 280 for path, dep in deps_subdict.items(): 281 if path in result: 282 continue 283 if not isinstance(dep, dict): 284 dep = {'url': dep} 285 if dep.get('dep_type') == 'cipd': 286 result[path] = CipdDepsEntry(path, dep['packages']) 287 else: 288 if '@' not in dep['url']: 289 continue 290 url, revision = dep['url'].split('@') 291 result[path] = DepsEntry(path, url, revision) 292 293 def AddVersionEntry(vars_subdict): 294 for key, value in vars_subdict.items(): 295 if key in result: 296 continue 297 if not key.endswith('_version'): 298 continue 299 key = re.sub('_version$', '', key) 300 result[key] = VersionEntry(value) 301 302 AddDepsEntries(deps_dict['deps']) 303 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 304 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 305 AddVersionEntry(deps_dict.get('vars', {})) 306 return result 307 308 309def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): 310 old_pkgs_names = {p['package'] for p in old_pkgs} 311 new_pkgs_names = {p['package'] for p in new_pkgs} 312 pkgs_equal = (old_pkgs_names == new_pkgs_names) 313 added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] 314 removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] 315 316 assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' 317 'and remove/add entries in DEPS so the old and new ' 318 'list match.\nMost likely, you should add \"%s\" and ' 319 'remove \"%s\"' % 320 (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) 321 322 for old_pkg in old_pkgs: 323 for new_pkg in new_pkgs: 324 old_version = old_pkg['version'] 325 new_version = new_pkg['version'] 326 if (old_pkg['package'] == new_pkg['package'] 327 and old_version != new_version): 328 logging.debug('Roll dependency %s to %s', path, new_version) 329 yield ChangedCipdPackage(path, old_pkg['package'], old_version, 330 new_version) 331 332 333def _FindChangedVars(name, old_version, new_version): 334 if old_version != new_version: 335 logging.debug('Roll dependency %s to %s', name, new_version) 336 yield ChangedVersionEntry(name, old_version, new_version) 337 338 339def _FindNewDeps(old, new): 340 """ Gather dependencies only in `new` and return corresponding paths. """ 341 old_entries = set(BuildDepsentryDict(old)) 342 new_entries = set(BuildDepsentryDict(new)) 343 return [ 344 path for path in new_entries - old_entries 345 if path not in DONT_AUTOROLL_THESE 346 ] 347 348 349def FindAddedDeps(libyuv_deps, new_cr_deps): 350 """ 351 Calculate new deps entries of interest. 352 353 Ideally, that would mean: only appearing in chromium DEPS 354 but transitively used in LibYUV. 355 356 Since it's hard to compute, we restrict ourselves to a well defined subset: 357 deps sitting in `ANDROID_DEPS_PATH`. 358 Otherwise, assumes that's a Chromium-only dependency. 359 360 Args: 361 libyuv_deps: dict of deps as defined in the LibYUV DEPS file. 362 new_cr_deps: dict of deps as defined in the chromium DEPS file. 363 364 Caveat: Doesn't detect a new package in existing dep. 365 366 Returns: 367 A tuple consisting of: 368 A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. 369 A list of paths for other added dependencies. 370 """ 371 all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps) 372 generated_android_deps = [ 373 path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) 374 ] 375 other_deps = [ 376 path for path in all_added_deps if path not in generated_android_deps 377 ] 378 return generated_android_deps, other_deps 379 380 381def FindRemovedDeps(libyuv_deps, new_cr_deps): 382 """ 383 Calculate obsolete deps entries. 384 385 Ideally, that would mean: no more appearing in chromium DEPS 386 and not used in LibYUV. 387 388 Since it's hard to compute: 389 1/ We restrict ourselves to a well defined subset: 390 deps sitting in `ANDROID_DEPS_PATH`. 391 2/ We rely on existing behavior of CalculateChangeDeps. 392 I.e. Assumes non-CIPD dependencies are LibYUV-only, don't remove them. 393 394 Args: 395 libyuv_deps: dict of deps as defined in the LibYUV DEPS file. 396 new_cr_deps: dict of deps as defined in the chromium DEPS file. 397 398 Caveat: Doesn't detect a deleted package in existing dep. 399 400 Returns: 401 A tuple consisting of: 402 A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. 403 A list of paths of unexpected disappearing dependencies. 404 """ 405 all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) 406 generated_android_deps = sorted( 407 [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) 408 # Webrtc-only dependencies are handled in CalculateChangedDeps. 409 other_deps = sorted([ 410 path for path in all_removed_deps 411 if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS 412 ]) 413 return generated_android_deps, other_deps 414 415 416def CalculateChangedDeps(libyuv_deps, new_cr_deps): 417 """ 418 Calculate changed deps entries based on entries defined in the LibYUV DEPS 419 file: 420 - If a shared dependency with the Chromium DEPS file: roll it to the same 421 revision as Chromium (i.e. entry in the new_cr_deps dict) 422 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 423 this means it may be ahead of the chromium_revision, but generally these 424 should be close). 425 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 426 unless it's configured to be skipped. 427 428 Returns: 429 A list of ChangedDep objects representing the changed deps. 430 """ 431 result = [] 432 libyuv_entries = BuildDepsentryDict(libyuv_deps) 433 new_cr_entries = BuildDepsentryDict(new_cr_deps) 434 for path, libyuv_deps_entry in libyuv_entries.items(): 435 if path in DONT_AUTOROLL_THESE: 436 continue 437 cr_deps_entry = new_cr_entries.get(path) 438 if cr_deps_entry: 439 assert type(cr_deps_entry) is type(libyuv_deps_entry) 440 441 if isinstance(cr_deps_entry, CipdDepsEntry): 442 result.extend( 443 _FindChangedCipdPackages(path, libyuv_deps_entry.packages, 444 cr_deps_entry.packages)) 445 continue 446 447 if isinstance(cr_deps_entry, VersionEntry): 448 result.extend( 449 _FindChangedVars(path, libyuv_deps_entry.version, 450 cr_deps_entry.version)) 451 continue 452 453 # Use the revision from Chromium's DEPS file. 454 new_rev = cr_deps_entry.revision 455 assert libyuv_deps_entry.url == cr_deps_entry.url, ( 456 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' % 457 (path, libyuv_deps_entry.url, cr_deps_entry.url)) 458 else: 459 if isinstance(libyuv_deps_entry, DepsEntry): 460 # Use the HEAD of the deps repo. 461 stdout, _ = _RunCommand( 462 ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']) 463 new_rev = stdout.strip().split('\t')[0] 464 else: 465 # The dependency has been removed from chromium. 466 # This is handled by FindRemovedDeps. 467 continue 468 469 # Check if an update is necessary. 470 if libyuv_deps_entry.revision != new_rev: 471 logging.debug('Roll dependency %s to %s', path, new_rev) 472 result.append( 473 ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision, 474 new_rev)) 475 return sorted(result) 476 477 478def CalculateChangedClang(new_cr_rev): 479 480 def GetClangRev(lines): 481 for line in lines: 482 match = CLANG_REVISION_RE.match(line) 483 if match: 484 return match.group(1) 485 raise RollError('Could not parse Clang revision!') 486 487 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: 488 current_lines = f.readlines() 489 current_rev = GetClangRev(current_lines) 490 491 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, 492 new_cr_rev).splitlines() 493 new_rev = GetClangRev(new_clang_update_py) 494 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) 495 496 497def GenerateCommitMessage( 498 rev_update, 499 current_commit_pos, 500 new_commit_pos, 501 changed_deps_list, 502 added_deps_paths=None, 503 removed_deps_paths=None, 504 clang_change=None, 505): 506 current_cr_rev = rev_update.current_chromium_rev[0:10] 507 new_cr_rev = rev_update.new_chromium_rev[0:10] 508 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 509 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 510 511 commit_msg = [ 512 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), 513 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), 514 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) 515 ] 516 517 def Section(adjective, deps): 518 noun = 'dependency' if len(deps) == 1 else 'dependencies' 519 commit_msg.append('%s %s' % (adjective, noun)) 520 521 if changed_deps_list: 522 Section('Changed', changed_deps_list) 523 524 for c in changed_deps_list: 525 if isinstance(c, ChangedCipdPackage): 526 commit_msg.append('* %s: %s..%s' % 527 (c.path, c.current_version, c.new_version)) 528 elif isinstance(c, ChangedVersionEntry): 529 commit_msg.append('* %s_vesion: %s..%s' % 530 (c.path, c.current_version, c.new_version)) 531 else: 532 commit_msg.append('* %s: %s/+log/%s..%s' % 533 (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) 534 535 if added_deps_paths: 536 Section('Added', added_deps_paths) 537 commit_msg.extend('* %s' % p for p in added_deps_paths) 538 539 if removed_deps_paths: 540 Section('Removed', removed_deps_paths) 541 commit_msg.extend('* %s' % p for p in removed_deps_paths) 542 543 if any([changed_deps_list, added_deps_paths, removed_deps_paths]): 544 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 545 commit_msg.append('DEPS diff: %s\n' % change_url) 546 else: 547 commit_msg.append('No dependencies changed.') 548 549 if clang_change and clang_change.current_rev != clang_change.new_rev: 550 commit_msg.append('Clang version changed %s:%s' % 551 (clang_change.current_rev, clang_change.new_rev)) 552 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 553 CLANG_UPDATE_SCRIPT_URL_PATH) 554 commit_msg.append('Details: %s\n' % change_url) 555 else: 556 commit_msg.append('No update to Clang.\n') 557 558 commit_msg.append('BUG=None') 559 return '\n'.join(commit_msg) 560 561 562def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): 563 """Update the DEPS file with the new revision.""" 564 565 with open(deps_filename, 'rb') as deps_file: 566 deps_content = deps_file.read().decode('utf-8') 567 568 # Update the chromium_revision variable. 569 deps_content = deps_content.replace(rev_update.current_chromium_rev, 570 rev_update.new_chromium_rev) 571 572 # Add and remove dependencies. For now: only generated android deps. 573 # Since gclient cannot add or remove deps, we on the fact that 574 # these android deps are located in one place we can copy/paste. 575 deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) 576 new_deps = deps_re.search(new_cr_content) 577 old_deps = deps_re.search(deps_content) 578 if not new_deps or not old_deps: 579 faulty = 'Chromium' if not new_deps else 'LibYUV' 580 raise RollError('Was expecting to find "%s" and "%s"\n' 581 'in %s DEPS' % 582 (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) 583 deps_content = deps_re.sub(new_deps.group(0), deps_content) 584 585 for dep in changed_deps: 586 if isinstance(dep, ChangedVersionEntry): 587 deps_content = deps_content.replace(dep.current_version, dep.new_version) 588 589 with open(deps_filename, 'wb') as deps_file: 590 deps_file.write(deps_content.encode('utf-8')) 591 592 # Update each individual DEPS entry. 593 for dep in changed_deps: 594 # ChangedVersionEntry types are already been processed. 595 if isinstance(dep, ChangedVersionEntry): 596 continue 597 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 598 if not os.path.isdir(local_dep_dir): 599 raise RollError( 600 'Cannot find local directory %s. Either run\n' 601 'gclient sync --deps=all\n' 602 'or make sure the .gclient file for your solution contains all ' 603 'platforms in the target_os list, i.e.\n' 604 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 605 'Then run "gclient sync" again.' % local_dep_dir) 606 if isinstance(dep, ChangedCipdPackage): 607 package = dep.package.format() # Eliminate double curly brackets 608 update = '%s:%s@%s' % (dep.path, package, dep.new_version) 609 else: 610 update = '%s@%s' % (dep.path, dep.new_rev) 611 _RunCommand(['gclient', 'setdep', '--revision', update], 612 working_dir=CHECKOUT_SRC_DIR) 613 614 615def _IsTreeClean(): 616 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 617 if len(stdout) == 0: 618 return True 619 620 logging.error('Dirty/unversioned files:\n%s', stdout) 621 return False 622 623 624def _EnsureUpdatedMainBranch(dry_run): 625 current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 626 'HEAD'])[0].splitlines()[0] 627 if current_branch != 'main': 628 logging.error('Please checkout the main branch and re-run this script.') 629 if not dry_run: 630 sys.exit(-1) 631 632 logging.info('Updating main branch...') 633 _RunCommand(['git', 'pull']) 634 635 636def _CreateRollBranch(dry_run): 637 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 638 if not dry_run: 639 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 640 641 642def _RemovePreviousRollBranch(dry_run): 643 active_branch, branches = _GetBranches() 644 if active_branch == ROLL_BRANCH_NAME: 645 active_branch = 'main' 646 if ROLL_BRANCH_NAME in branches: 647 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 648 if not dry_run: 649 _RunCommand(['git', 'checkout', active_branch]) 650 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 651 652 653def _LocalCommit(commit_msg, dry_run): 654 logging.info('Committing changes locally.') 655 if not dry_run: 656 _RunCommand(['git', 'add', '--update', '.']) 657 _RunCommand(['git', 'commit', '-m', commit_msg]) 658 659 660def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): 661 if skip_cq: 662 return 0 663 if (new_commit_pos - current_commit_pos) < cq_over: 664 return 1 665 return 2 666 667 668def _GetCcRecipients(changed_deps_list): 669 """Returns a list of emails to notify based on the changed deps list. 670 """ 671 cc_recipients = [] 672 for c in changed_deps_list: 673 pass 674 return cc_recipients 675 676 677def _UploadCL(commit_queue_mode, add_cc=None): 678 """Upload the committed changes as a changelist to Gerrit. 679 680 commit_queue_mode: 681 - 2: Submit to commit queue. 682 - 1: Run trybots but do not submit to CQ. 683 - 0: Skip CQ, upload only. 684 685 add_cc: A list of email addresses to add as CC recipients. 686 """ 687 cc_recipients = [] 688 if add_cc: 689 cc_recipients.extend(add_cc) 690 cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] 691 if commit_queue_mode >= 2: 692 logging.info('Sending the CL to the CQ...') 693 cmd.extend(['-o', 'label=Bot-Commit+1']) 694 cmd.extend(['-o', 'label=Commit-Queue+2']) 695 cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) 696 elif commit_queue_mode >= 1: 697 logging.info('Starting CQ dry run...') 698 cmd.extend(['-o', 'label=Commit-Queue+1']) 699 extra_env = { 700 'EDITOR': 'true', 701 'SKIP_GCE_AUTH_FOR_GIT': '1', 702 } 703 stdout, stderr = _RunCommand(cmd, extra_env=extra_env) 704 logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', 705 stdout, stderr) 706 707 708def GetRollRevisionRanges(opts, libyuv_deps): 709 current_cr_rev = libyuv_deps['vars']['chromium_revision'] 710 new_cr_rev = opts.revision 711 if not new_cr_rev: 712 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 713 head_rev = stdout.strip().split('\t')[0] 714 logging.info('No revision specified. Using HEAD: %s', head_rev) 715 new_cr_rev = head_rev 716 717 return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) 718 719 720def main(): 721 p = argparse.ArgumentParser() 722 p.add_argument('--clean', 723 action='store_true', 724 default=False, 725 help='Removes any previous local roll branch.') 726 p.add_argument('-r', 727 '--revision', 728 help=('Chromium Git revision to roll to. Defaults to the ' 729 'Chromium HEAD revision if omitted.')) 730 p.add_argument('--dry-run', 731 action='store_true', 732 default=False, 733 help=('Calculate changes and modify DEPS, but don\'t create ' 734 'any local branch, commit, upload CL or send any ' 735 'tryjobs.')) 736 p.add_argument('-i', 737 '--ignore-unclean-workdir', 738 action='store_true', 739 default=False, 740 help=('Ignore if the current branch is not main or if there ' 741 'are uncommitted changes (default: %(default)s).')) 742 grp = p.add_mutually_exclusive_group() 743 grp.add_argument('--skip-cq', 744 action='store_true', 745 default=False, 746 help='Skip sending the CL to the CQ (default: %(default)s)') 747 grp.add_argument('--cq-over', 748 type=int, 749 default=1, 750 help=('Commit queue dry run if the revision difference ' 751 'is below this number (default: %(default)s)')) 752 p.add_argument('-v', 753 '--verbose', 754 action='store_true', 755 default=False, 756 help='Be extra verbose in printing of log messages.') 757 opts = p.parse_args() 758 759 if opts.verbose: 760 logging.basicConfig(level=logging.DEBUG) 761 else: 762 logging.basicConfig(level=logging.INFO) 763 764 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 765 logging.error('Please clean your local checkout first.') 766 return 1 767 768 if opts.clean: 769 _RemovePreviousRollBranch(opts.dry_run) 770 771 if not opts.ignore_unclean_workdir: 772 _EnsureUpdatedMainBranch(opts.dry_run) 773 774 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 775 libyuv_deps = ParseLocalDepsFile(deps_filename) 776 777 rev_update = GetRollRevisionRanges(opts, libyuv_deps) 778 779 current_commit_pos = ParseCommitPosition( 780 ReadRemoteCrCommit(rev_update.current_chromium_rev)) 781 new_commit_pos = ParseCommitPosition( 782 ReadRemoteCrCommit(rev_update.new_chromium_rev)) 783 784 new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) 785 new_cr_deps = ParseDepsDict(new_cr_content) 786 changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) 787 # Discard other deps, assumed to be chromium-only dependencies. 788 new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) 789 removed_generated_android_deps, other_deps = FindRemovedDeps( 790 libyuv_deps, new_cr_deps) 791 if other_deps: 792 raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n' 793 'Remove them or add them to either ' 794 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) 795 clang_change = CalculateChangedClang(rev_update.new_chromium_rev) 796 commit_msg = GenerateCommitMessage( 797 rev_update, 798 current_commit_pos, 799 new_commit_pos, 800 changed_deps, 801 added_deps_paths=new_generated_android_deps, 802 removed_deps_paths=removed_generated_android_deps, 803 clang_change=clang_change) 804 logging.debug('Commit message:\n%s', commit_msg) 805 806 _CreateRollBranch(opts.dry_run) 807 if not opts.dry_run: 808 UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) 809 if _IsTreeClean(): 810 logging.info("No DEPS changes detected, skipping CL creation.") 811 else: 812 _LocalCommit(commit_msg, opts.dry_run) 813 commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, 814 current_commit_pos, new_commit_pos) 815 logging.info('Uploading CL...') 816 if not opts.dry_run: 817 _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) 818 return 0 819 820 821if __name__ == '__main__': 822 sys.exit(main()) 823