1#!/usr/bin/python3 2# 3# Copyright 2021 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7# sync_restricted_traces_to_cipd.py: 8# Ensures the restricted traces are uploaded to CIPD. Versions are encoded in 9# restricted_traces.json. Requires access to the CIPD path to work. 10 11import argparse 12from concurrent import futures 13import getpass 14import fnmatch 15import logging 16import json 17import os 18import platform 19import signal 20import subprocess 21import sys 22 23CIPD_PREFIX = 'angle/traces' 24EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces' 25LOG_LEVEL = 'info' 26JSON_PATH = 'restricted_traces.json' 27SCRIPT_DIR = os.path.dirname(sys.argv[0]) 28MAX_THREADS = 8 29LONG_TIMEOUT = 100000 30 31 32def cipd(args, suppress_stdout=True): 33 logging.debug('running cipd with args: %s', ' '.join(args)) 34 exe = 'cipd.bat' if platform.system() == 'Windows' else 'cipd' 35 if suppress_stdout: 36 # Capture stdout, only log if --log=debug after the process terminates 37 process = subprocess.run([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 38 if process.stdout: 39 logging.debug('cipd stdout:\n%s' % process.stdout.decode()) 40 else: 41 # Stdout is piped to the caller's stdout, visible immediately 42 process = subprocess.run([exe] + args) 43 return process.returncode 44 45 46def cipd_name_and_version(trace, trace_version): 47 if 'x' in trace_version: 48 trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser() 49 trace_version = trace_version.strip('x') 50 else: 51 trace_prefix = CIPD_PREFIX 52 53 trace_name = '%s/%s' % (trace_prefix, trace) 54 55 return trace_name, trace_version 56 57 58def check_trace_exists(args, trace, trace_version): 59 cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version) 60 61 # Determine if this version exists 62 return cipd(['describe', cipd_trace_name, '-version', 'version:%s' % cipd_trace_version]) == 0 63 64 65def upload_trace(args, trace, trace_version): 66 trace_folder = os.path.join(SCRIPT_DIR, trace) 67 cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version) 68 cipd_args = ['create', '-name', cipd_trace_name] 69 cipd_args += ['-in', trace_folder] 70 cipd_args += ['-tag', 'version:%s' % cipd_trace_version] 71 cipd_args += ['-log-level', args.log.lower()] 72 cipd_args += ['-install-mode', 'copy'] 73 if cipd(cipd_args, suppress_stdout=False) != 0: 74 logging.error('%s version %s: cipd create failed', trace, trace_version) 75 sys.exit(1) 76 77 logging.info('Uploaded trace to cipd: %s version:%s', cipd_trace_name, cipd_trace_version) 78 79 80def check_trace_before_upload(trace): 81 for root, dirs, files in os.walk(os.path.join(SCRIPT_DIR, trace)): 82 if dirs: 83 logging.error('Sub-directories detected for trace %s: %s' % (trace, dirs)) 84 sys.exit(1) 85 trace_json = trace + '.json' 86 with open(os.path.join(root, trace_json)) as f: 87 jtrace = json.load(f) 88 additional_files = set([trace_json, trace + '.angledata.gz']) 89 extra_files = set(files) - set(jtrace['TraceFiles']) - additional_files 90 required_extensions = 'RequiredExtensions' in jtrace 91 if extra_files: 92 logging.error('Unexpected files, not listed in %s.json [TraceFiles]:\n%s', trace, 93 '\n'.join(extra_files)) 94 if not required_extensions: 95 logging.error( 96 '"RequiredExtensions" missing from %s.json. Please run retrace_restricted_traces.py with "get_min_reqs":\n' 97 ' ./src/tests/restricted_traces/retrace_restricted_traces.py get_min_reqs out/LinuxDebug --traces "%s"\n', 98 trace, trace) 99 if extra_files or not required_extensions: 100 sys.exit(1) 101 102 103def main(args): 104 logging.basicConfig(level=args.log.upper()) 105 106 with open(os.path.join(SCRIPT_DIR, JSON_PATH)) as f: 107 traces = json.loads(f.read()) 108 109 logging.info('Checking cipd for existing versions (this takes time without --filter)') 110 f_exists = {} 111 trace_versions = {} 112 with futures.ThreadPoolExecutor(max_workers=args.threads) as executor: 113 for trace_info in traces['traces']: 114 trace, trace_version = trace_info.split(' ') 115 trace_versions[trace] = trace_version 116 if args.filter and not fnmatch.fnmatch(trace, args.filter): 117 logging.debug('Skipping %s because it does not match the test filter.' % trace) 118 continue 119 assert trace not in f_exists 120 f_exists[trace] = executor.submit(check_trace_exists, args, trace, trace_version) 121 122 to_upload = [trace for trace, f in f_exists.items() if not f.result()] 123 if not to_upload: 124 logging.info('All traces are in sync with cipd') 125 return 0 126 127 logging.info('The following traces are out of sync with cipd:') 128 for trace in to_upload: 129 print(' ', trace, trace_versions[trace]) 130 check_trace_before_upload(trace) 131 132 if args.upload or input('Upload [y/N]?') == 'y': 133 for trace in to_upload: 134 upload_trace(args, trace, trace_versions[trace]) 135 else: 136 logging.error('Aborted') 137 return 1 138 139 return 0 140 141 142if __name__ == '__main__': 143 parser = argparse.ArgumentParser() 144 parser.add_argument( 145 '-p', '--prefix', help='CIPD Prefix. Default: %s' % CIPD_PREFIX, default=CIPD_PREFIX) 146 parser.add_argument( 147 '-l', '--log', help='Logging level. Default: %s' % LOG_LEVEL, default=LOG_LEVEL) 148 parser.add_argument( 149 '-f', '--filter', help='Only sync specified tests. Supports fnmatch expressions.') 150 parser.add_argument( 151 '-t', 152 '--threads', 153 help='Maxiumum parallel threads. Default: %s' % MAX_THREADS, 154 default=MAX_THREADS) 155 parser.add_argument('--upload', action='store_true', help='Upload without asking.') 156 args = parser.parse_args() 157 158 logging.basicConfig(level=args.log.upper()) 159 160 sys.exit(main(args)) 161