1#!/usr/bin/env python3 2# Copyright 2014 Google Inc. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Parse a DEPS file and git checkout all of the dependencies. 31""" 32 33EXTRA_HELP = """ 34Environment Variables: 35 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of 36 ['git', 'git.exe', 'git.bat'] in your default path. 37 38 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 39 will use the file ../DEPS relative to this script's directory. 40 41 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 42 43Git Config: 44 To disable syncing of a single repository: 45 cd path/to/repository 46 git config sync-deps.disable true 47 48 To re-enable sync: 49 cd path/to/repository 50 git config --unset sync-deps.disable 51""" 52 53 54import argparse 55import os 56import re 57import subprocess 58import sys 59import threading 60from builtins import bytes 61 62def git_executable(): 63 """Find the git executable. 64 65 Returns: 66 A triple: 67 A string suitable for passing to subprocess functions, or None. 68 The major version number 69 The minor version number 70 """ 71 envgit = os.environ.get('GIT_EXECUTABLE') 72 searchlist = ['git', 'git.exe', 'git.bat'] 73 if envgit: 74 searchlist.insert(0, envgit) 75 with open(os.devnull, 'w') as devnull: 76 for git in searchlist: 77 major=None 78 minor=None 79 try: 80 version_info = subprocess.check_output([git, '--version']).decode('utf-8') 81 match = re.search(r"^git version (\d+)\.(\d+)",version_info) 82 print("Using {}".format(version_info)) 83 if match: 84 major = int(match.group(1)) 85 minor = int(match.group(2)) 86 else: 87 continue 88 except (OSError,): 89 continue 90 return (git,major,minor) 91 return (None,0,0) 92 93 94DEFAULT_DEPS_PATH = os.path.normpath( 95 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 96 97def get_deps_os_str(deps_file): 98 parsed_deps = parse_file_to_dict(deps_file) 99 parts = [] 100 if 'deps_os' in parsed_deps: 101 for deps_os in parsed_deps['deps_os']: 102 parts.append(' [{}]]'.format(deps_os)) 103 return "\n".join(parts) 104 105def looks_like_raw_commit(commit): 106 return re.match('^[a-f0-9]{40}$', commit) is not None 107 108def git_repository_sync_is_disabled(git, directory): 109 try: 110 disable = subprocess.check_output( 111 [git, 'config', 'sync-deps.disable'], cwd=directory) 112 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 113 except subprocess.CalledProcessError: 114 return False 115 116 117def is_git_toplevel(git, directory): 118 """Return true iff the directory is the top level of a Git repository. 119 120 Args: 121 git (string) the git executable 122 123 directory (string) the path into which the repository 124 is expected to be checked out. 125 """ 126 try: 127 toplevel = subprocess.check_output( 128 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 129 return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel) 130 except subprocess.CalledProcessError: 131 return False 132 133 134def status(directory, checkoutable): 135 def truncate(s, length): 136 return s if len(s) <= length else '...' + s[-(length - 3):] 137 dlen = 36 138 directory = truncate(directory, dlen) 139 checkoutable = truncate(checkoutable, 40) 140 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) 141 142 143def git_checkout_to_directory(git, repo, checkoutable, directory, verbose, treeless): 144 """Checkout (and clone if needed) a Git repository. 145 146 Args: 147 git (string) the git executable 148 149 repo (string) the location of the repository, suitable 150 for passing to `git clone`. 151 152 checkoutable (string) a tag, branch, or commit, suitable for 153 passing to `git checkout` 154 155 directory (string) the path into which the repository 156 should be checked out. 157 158 verbose (boolean): emit status info to stdout 159 160 treeless (boolean): when true, clone without any trees. 161 162 Raises an exception if any calls to git fail. 163 """ 164 if not os.path.isdir(directory): 165 # Use blobless or treeless checkouts for faster downloads. 166 # This defers some work to checkout time. 167 # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ 168 filter = ['--filter=tree:0'] if treeless else ['--filter=blob:none'] 169 # If the thing to check out looks like a tag (and not like a commit), 170 # then limit the checkout to that branch. 171 branch = [] if looks_like_raw_commit(checkoutable) else ['--branch={}'.format(checkoutable)] 172 subprocess.check_call( 173 [git, 'clone', '--quiet', '--single-branch'] + filter + branch + [repo, directory]) 174 175 if not is_git_toplevel(git, directory): 176 # if the directory exists, but isn't a git repo, you will modify 177 # the parent repostory, which isn't what you want. 178 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 179 return 180 181 # Check to see if this repo is disabled. Quick return. 182 if git_repository_sync_is_disabled(git, directory): 183 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 184 return 185 186 with open(os.devnull, 'w') as devnull: 187 # If this fails, we will fetch before trying again. Don't spam user 188 # with error information. 189 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable], 190 cwd=directory, stderr=devnull): 191 # if this succeeds, skip slow `git fetch`. 192 if verbose: 193 status(directory, checkoutable) # Success. 194 return 195 196 # If the repo has changed, always force use of the correct repo. 197 # If origin already points to repo, this is a quick no-op. 198 subprocess.check_call( 199 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 200 201 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 202 203 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory) 204 205 if verbose: 206 status(directory, checkoutable) # Success. 207 208 209def parse_file_to_dict(path): 210 dictionary = {} 211 contents = open(path).read() 212 # Need to convert Var() to vars[], so that the DEPS is actually Python. Var() 213 # comes from Autoroller using gclient which has a slightly different DEPS 214 # format. 215 contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents) 216 exec(contents, dictionary) 217 return dictionary 218 219 220def git_sync_deps(deps_file_path, command_line_os_requests, verbose, treeless): 221 """Grab dependencies, with optional platform support. 222 223 Args: 224 deps_file_path (string) Path to the DEPS file. 225 226 command_line_os_requests (list of strings) Can be empty list. 227 List of strings that should each be a key in the deps_os 228 dictionary in the DEPS file. 229 230 verbose (boolean): emit status info to stdout 231 232 treeless (boolean): when true, clone as treeless instead of blobless 233 234 Raises git Exceptions. 235 """ 236 (git,git_major,git_minor) = git_executable() 237 assert git 238 239 # --filter=tree:0 is available in git 2.20 and later 240 if (git_major,git_minor) < (2,20): 241 print("disabling --treeless: git is older than v2.20") 242 treeless = False 243 244 deps_file_directory = os.path.dirname(deps_file_path) 245 deps_file = parse_file_to_dict(deps_file_path) 246 dependencies = deps_file['deps'].copy() 247 os_specific_dependencies = deps_file.get('deps_os', dict()) 248 if 'all' in command_line_os_requests: 249 for value in list(os_specific_dependencies.values()): 250 dependencies.update(value) 251 else: 252 for os_name in command_line_os_requests: 253 # Add OS-specific dependencies 254 if os_name in os_specific_dependencies: 255 dependencies.update(os_specific_dependencies[os_name]) 256 for directory in dependencies: 257 for other_dir in dependencies: 258 if directory.startswith(other_dir + '/'): 259 raise Exception('%r is parent of %r' % (other_dir, directory)) 260 list_of_arg_lists = [] 261 for directory in sorted(dependencies): 262 if '@' in dependencies[directory]: 263 repo, checkoutable = dependencies[directory].split('@', 1) 264 else: 265 raise Exception("please specify commit or tag") 266 267 relative_directory = os.path.join(deps_file_directory, directory) 268 269 list_of_arg_lists.append( 270 (git, repo, checkoutable, relative_directory, verbose, treeless)) 271 272 multithread(git_checkout_to_directory, list_of_arg_lists) 273 274 for directory in deps_file.get('recursedeps', []): 275 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') 276 git_sync_deps(recursive_path, command_line_os_requests, verbose) 277 278 279def multithread(function, list_of_arg_lists): 280 # for args in list_of_arg_lists: 281 # function(*args) 282 # return 283 threads = [] 284 for args in list_of_arg_lists: 285 thread = threading.Thread(None, function, None, args) 286 thread.start() 287 threads.append(thread) 288 for thread in threads: 289 thread.join() 290 291 292def main(argv): 293 argparser = argparse.ArgumentParser( 294 prog = "git-sync-deps", 295 description = "Checkout git-based dependencies as specified by the DEPS file", 296 add_help=False # Because we want to print deps_os with -h option 297 ) 298 argparser.add_argument("--help", "-h", 299 action='store_true', 300 help="show this help message and exit") 301 argparser.add_argument("--deps", 302 default = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH), 303 help="location of the the DEPS file") 304 argparser.add_argument("--verbose", 305 default=not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)), 306 action='store_true', 307 help="be verbose: print status messages") 308 argparser.add_argument("--treeless", 309 default=False, 310 action='store_true', 311 help=""" 312 Clone repos without trees (--filter=tree:0). 313 This is the fastest option for a build machine, 314 when you only need a single commit. 315 Defers getting objects until checking out a commit. 316 317 The default is to clone with trees but without blobs. 318 319 Only takes effect if using git 2.20 or later. 320 321 See https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ 322 """) 323 argparser.add_argument("os_requests",nargs="*", 324 help="OS requests, as keys in the deps_os dictionariy in the DEPS file") 325 326 args = argparser.parse_args() 327 if args.help: 328 print(argparser.format_help()) 329 print(EXTRA_HELP) 330 print(get_deps_os_str(args.deps)) 331 return 0 332 333 git_sync_deps(args.deps, args.os_requests, args.verbose, args.treeless) 334 return 0 335 336 337if __name__ == '__main__': 338 exit(main(sys.argv[1:])) 339