1#! /usr/bin/env vpython3 2# 3# Copyright 2023 The ANGLE Project Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6# 7 8import argparse 9import contextlib 10import difflib 11import json 12import logging 13import os 14import pathlib 15import shutil 16import subprocess 17import sys 18import tempfile 19import time 20 21SCRIPT_DIR = str(pathlib.Path(__file__).resolve().parent) 22PY_UTILS = str(pathlib.Path(SCRIPT_DIR) / '..' / 'py_utils') 23if PY_UTILS not in sys.path: 24 os.stat(PY_UTILS) and sys.path.insert(0, PY_UTILS) 25import angle_test_util 26 27 28@contextlib.contextmanager 29def temporary_dir(prefix=''): 30 path = tempfile.mkdtemp(prefix=prefix) 31 try: 32 yield path 33 finally: 34 logging.info("Removing temporary directory: %s" % path) 35 shutil.rmtree(path) 36 37 38def file_content(path): 39 with open(path, 'rb') as f: 40 content = f.read() 41 42 if path.endswith('.json'): 43 info = json.loads(content) 44 info['TraceMetadata']['CaptureRevision'] = '<ignored>' 45 return json.dumps(info, indent=2).encode() 46 47 return content 48 49 50def diff_files(path, expected_path): 51 content = file_content(path) 52 expected_content = file_content(expected_path) 53 fn = os.path.basename(path) 54 55 if content == expected_content: 56 return False 57 58 if fn.endswith('.angledata'): 59 logging.error('Checks failed. Binary file contents mismatch: %s', fn) 60 return True 61 62 # Captured files are expected to have LF line endings. 63 # Note that git's EOL conversion for these files is disabled via .gitattributes 64 assert b'\r\n' not in content 65 assert b'\r\n' not in expected_content 66 67 diff = list( 68 difflib.unified_diff( 69 expected_content.decode().splitlines(), 70 content.decode().splitlines(), 71 fromfile=fn, 72 tofile=fn, 73 )) 74 75 logging.error('Checks failed. Found diff in %s:\n%s\n', fn, '\n'.join(diff)) 76 return True 77 78 79def run_test(test_name, overwrite_expected): 80 with temporary_dir() as temp_dir: 81 cmd = [angle_test_util.ExecutablePathInCurrentDir('angle_end2end_tests')] 82 if angle_test_util.IsAndroid(): 83 cmd.append('--angle-test-runner') 84 85 test_args = ['--gtest_filter=%s' % test_name, '--angle-per-test-capture-label'] 86 extra_env = { 87 'ANGLE_CAPTURE_ENABLED': '1', 88 'ANGLE_CAPTURE_FRAME_START': '2', 89 'ANGLE_CAPTURE_FRAME_END': '5', 90 'ANGLE_CAPTURE_OUT_DIR': temp_dir, 91 'ANGLE_CAPTURE_COMPRESSION': '0', 92 } 93 subprocess.check_call(cmd + test_args, env={**os.environ.copy(), **extra_env}) 94 logging.info('Capture finished, comparing files') 95 files = sorted(fn for fn in os.listdir(temp_dir)) 96 expected_dir = os.path.join(SCRIPT_DIR, 'expected') 97 expected_files = sorted(fn for fn in os.listdir(expected_dir) if not fn.startswith('.')) 98 99 if overwrite_expected: 100 for f in expected_files: 101 os.remove(os.path.join(expected_dir, f)) 102 shutil.copytree(temp_dir, expected_dir, dirs_exist_ok=True) 103 return True 104 105 if files != expected_files: 106 logging.error( 107 'Checks failed. Capture produced a different set of files: %s\nDiff:\n%s\n', files, 108 '\n'.join(difflib.unified_diff(expected_files, files))) 109 return False 110 111 has_diffs = False 112 for fn in files: 113 has_diffs |= diff_files(os.path.join(temp_dir, fn), os.path.join(expected_dir, fn)) 114 115 return not has_diffs 116 117 118def main(): 119 parser = argparse.ArgumentParser() 120 parser.add_argument('--isolated-script-test-output', type=str) 121 parser.add_argument('--log', help='Logging level.', default='info') 122 parser.add_argument( 123 '--overwrite-expected', help='Overwrite contents of expected/', action='store_true') 124 args, extra_flags = parser.parse_known_args() 125 126 logging.basicConfig(level=args.log.upper()) 127 128 angle_test_util.Initialize('angle_end2end_tests') 129 130 test_name = 'CapturedTest*/ES3_Vulkan' 131 had_error = False 132 try: 133 if not run_test(test_name, args.overwrite_expected): 134 had_error = True 135 logging.error( 136 'Found capture diffs. If diffs are expected, build angle_end2end_tests and run ' 137 '(cd out/<build>; ../../src/tests/capture_tests/capture_tests.py --overwrite-expected)' 138 ) 139 except Exception as e: 140 logging.exception(e) 141 had_error = True 142 143 if args.isolated_script_test_output: 144 results = { 145 'tests': { 146 'capture_test': {} 147 }, 148 'interrupted': False, 149 'seconds_since_epoch': time.time(), 150 'path_delimiter': '.', 151 'version': 3, 152 'num_failures_by_type': { 153 'FAIL': 0, 154 'PASS': 0, 155 'SKIP': 0, 156 }, 157 } 158 result = 'FAIL' if had_error else 'PASS' 159 results['tests']['capture_test'][test_name] = {'expected': 'PASS', 'actual': result} 160 results['num_failures_by_type'][result] += 1 161 162 with open(args.isolated_script_test_output, 'w') as f: 163 f.write(json.dumps(results, indent=2)) 164 165 return 1 if had_error else 0 166 167 168if __name__ == '__main__': 169 sys.exit(main()) 170