xref: /aosp_15_r20/external/angle/src/tests/restricted_traces/sync_restricted_traces_to_cipd.py (revision 8975f5c5ed3d1c378011245431ada316dfb6f244)
1#!/usr/bin/python3
2#
3# Copyright 2021 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# sync_restricted_traces_to_cipd.py:
8#   Ensures the restricted traces are uploaded to CIPD. Versions are encoded in
9#   restricted_traces.json. Requires access to the CIPD path to work.
10
11import argparse
12from concurrent import futures
13import getpass
14import fnmatch
15import logging
16import json
17import os
18import platform
19import signal
20import subprocess
21import sys
22
23CIPD_PREFIX = 'angle/traces'
24EXPERIMENTAL_CIPD_PREFIX = 'experimental/google.com/%s/angle/traces'
25LOG_LEVEL = 'info'
26JSON_PATH = 'restricted_traces.json'
27SCRIPT_DIR = os.path.dirname(sys.argv[0])
28MAX_THREADS = 8
29LONG_TIMEOUT = 100000
30
31
32def cipd(args, suppress_stdout=True):
33    logging.debug('running cipd with args: %s', ' '.join(args))
34    exe = 'cipd.bat' if platform.system() == 'Windows' else 'cipd'
35    if suppress_stdout:
36        # Capture stdout, only log if --log=debug after the process terminates
37        process = subprocess.run([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
38        if process.stdout:
39            logging.debug('cipd stdout:\n%s' % process.stdout.decode())
40    else:
41        # Stdout is piped to the caller's stdout, visible immediately
42        process = subprocess.run([exe] + args)
43    return process.returncode
44
45
46def cipd_name_and_version(trace, trace_version):
47    if 'x' in trace_version:
48        trace_prefix = EXPERIMENTAL_CIPD_PREFIX % getpass.getuser()
49        trace_version = trace_version.strip('x')
50    else:
51        trace_prefix = CIPD_PREFIX
52
53    trace_name = '%s/%s' % (trace_prefix, trace)
54
55    return trace_name, trace_version
56
57
58def check_trace_exists(args, trace, trace_version):
59    cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
60
61    # Determine if this version exists
62    return cipd(['describe', cipd_trace_name, '-version', 'version:%s' % cipd_trace_version]) == 0
63
64
65def upload_trace(args, trace, trace_version):
66    trace_folder = os.path.join(SCRIPT_DIR, trace)
67    cipd_trace_name, cipd_trace_version = cipd_name_and_version(trace, trace_version)
68    cipd_args = ['create', '-name', cipd_trace_name]
69    cipd_args += ['-in', trace_folder]
70    cipd_args += ['-tag', 'version:%s' % cipd_trace_version]
71    cipd_args += ['-log-level', args.log.lower()]
72    cipd_args += ['-install-mode', 'copy']
73    if cipd(cipd_args, suppress_stdout=False) != 0:
74        logging.error('%s version %s: cipd create failed', trace, trace_version)
75        sys.exit(1)
76
77    logging.info('Uploaded trace to cipd: %s version:%s', cipd_trace_name, cipd_trace_version)
78
79
80def check_trace_before_upload(trace):
81    for root, dirs, files in os.walk(os.path.join(SCRIPT_DIR, trace)):
82        if dirs:
83            logging.error('Sub-directories detected for trace %s: %s' % (trace, dirs))
84            sys.exit(1)
85        trace_json = trace + '.json'
86        with open(os.path.join(root, trace_json)) as f:
87            jtrace = json.load(f)
88        additional_files = set([trace_json, trace + '.angledata.gz'])
89        extra_files = set(files) - set(jtrace['TraceFiles']) - additional_files
90        required_extensions = 'RequiredExtensions' in jtrace
91        if extra_files:
92            logging.error('Unexpected files, not listed in %s.json [TraceFiles]:\n%s', trace,
93                          '\n'.join(extra_files))
94        if not required_extensions:
95            logging.error(
96                '"RequiredExtensions" missing from %s.json. Please run retrace_restricted_traces.py with "get_min_reqs":\n'
97                '  ./src/tests/restricted_traces/retrace_restricted_traces.py get_min_reqs out/LinuxDebug --traces "%s"\n',
98                trace, trace)
99        if extra_files or not required_extensions:
100            sys.exit(1)
101
102
103def main(args):
104    logging.basicConfig(level=args.log.upper())
105
106    with open(os.path.join(SCRIPT_DIR, JSON_PATH)) as f:
107        traces = json.loads(f.read())
108
109    logging.info('Checking cipd for existing versions (this takes time without --filter)')
110    f_exists = {}
111    trace_versions = {}
112    with futures.ThreadPoolExecutor(max_workers=args.threads) as executor:
113        for trace_info in traces['traces']:
114            trace, trace_version = trace_info.split(' ')
115            trace_versions[trace] = trace_version
116            if args.filter and not fnmatch.fnmatch(trace, args.filter):
117                logging.debug('Skipping %s because it does not match the test filter.' % trace)
118                continue
119            assert trace not in f_exists
120            f_exists[trace] = executor.submit(check_trace_exists, args, trace, trace_version)
121
122    to_upload = [trace for trace, f in f_exists.items() if not f.result()]
123    if not to_upload:
124        logging.info('All traces are in sync with cipd')
125        return 0
126
127    logging.info('The following traces are out of sync with cipd:')
128    for trace in to_upload:
129        print(' ', trace, trace_versions[trace])
130        check_trace_before_upload(trace)
131
132    if args.upload or input('Upload [y/N]?') == 'y':
133        for trace in to_upload:
134            upload_trace(args, trace, trace_versions[trace])
135    else:
136        logging.error('Aborted')
137        return 1
138
139    return 0
140
141
142if __name__ == '__main__':
143    parser = argparse.ArgumentParser()
144    parser.add_argument(
145        '-p', '--prefix', help='CIPD Prefix. Default: %s' % CIPD_PREFIX, default=CIPD_PREFIX)
146    parser.add_argument(
147        '-l', '--log', help='Logging level. Default: %s' % LOG_LEVEL, default=LOG_LEVEL)
148    parser.add_argument(
149        '-f', '--filter', help='Only sync specified tests. Supports fnmatch expressions.')
150    parser.add_argument(
151        '-t',
152        '--threads',
153        help='Maxiumum parallel threads. Default: %s' % MAX_THREADS,
154        default=MAX_THREADS)
155    parser.add_argument('--upload', action='store_true', help='Upload without asking.')
156    args = parser.parse_args()
157
158    logging.basicConfig(level=args.log.upper())
159
160    sys.exit(main(args))
161