xref: /aosp_15_r20/external/angle/scripts/run_code_generation.py (revision 8975f5c5ed3d1c378011245431ada316dfb6f244) 
1#!/usr/bin/python3
2#
3# Copyright 2017 The ANGLE Project Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7# run_code_generation.py:
8#   Runs ANGLE format table and other script code generation scripts.
9
10import argparse
11from concurrent import futures
12import hashlib
13import json
14import os
15import subprocess
16import sys
17import platform
18
19script_dir = sys.path[0]
20root_dir = os.path.abspath(os.path.join(script_dir, '..'))
21
22hash_dir = os.path.join(script_dir, 'code_generation_hashes')
23
24
25def get_child_script_dirname(script):
26    # All script names are relative to ANGLE's root
27    return os.path.dirname(os.path.abspath(os.path.join(root_dir, script)))
28
29
30def get_executable_name(script):
31    with open(script, 'r') as f:
32        # Check shebang
33        binary = os.path.basename(f.readline().strip().replace(' ', '/'))
34        assert binary in ['python3', 'vpython3']
35        if platform.system() == 'Windows':
36            return binary + '.bat'
37        else:
38            return binary
39
40
41def paths_from_auto_script(script, param):
42    script_dir = get_child_script_dirname(script)
43    # python3 (not vpython3) to get inputs/outputs faster
44    exe = 'python3'
45    try:
46        res = subprocess.check_output([exe, os.path.basename(script), param],
47                                      cwd=script_dir).decode().strip()
48    except Exception:
49        print('Error with auto_script %s: %s, executable %s' % (param, script, exe))
50        raise
51    if res == '':
52        return []
53    return [
54        os.path.relpath(os.path.join(script_dir, path), root_dir).replace("\\", "/")
55        for path in res.split(',')
56    ]
57
58
59# auto_script is a standard way for scripts to return their inputs and outputs.
60def auto_script(script):
61    info = {
62        'inputs': paths_from_auto_script(script, 'inputs'),
63        'outputs': paths_from_auto_script(script, 'outputs')
64    }
65    return info
66
67
68generators = {
69    'ANGLE format':
70        'src/libANGLE/renderer/gen_angle_format_table.py',
71    'ANGLE load functions table':
72        'src/libANGLE/renderer/gen_load_functions_table.py',
73    'ANGLE shader preprocessor':
74        'src/compiler/preprocessor/generate_parser.py',
75    'ANGLE shader translator':
76        'src/compiler/translator/generate_parser.py',
77    'D3D11 blit shader selection':
78        'src/libANGLE/renderer/d3d/d3d11/gen_blit11helper.py',
79    'D3D11 format':
80        'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py',
81    'DXGI format':
82        'src/libANGLE/renderer/gen_dxgi_format_table.py',
83    'DXGI format support':
84        'src/libANGLE/renderer/gen_dxgi_support_tables.py',
85    'Emulated HLSL functions':
86        'src/compiler/translator/hlsl/gen_emulated_builtin_function_tables.py',
87    'Extension files':
88        'src/libANGLE/gen_extensions.py',
89    'GL copy conversion table':
90        'src/libANGLE/gen_copy_conversion_table.py',
91    'GL CTS (dEQP) build files':
92        'scripts/gen_vk_gl_cts_build.py',
93    'GL/EGL/WGL loader':
94        'scripts/generate_loader.py',
95    'GL/EGL entry points':
96        'scripts/generate_entry_points.py',
97    'GLenum value to string map':
98        'scripts/gen_gl_enum_utils.py',
99    'GL format map':
100        'src/libANGLE/gen_format_map.py',
101    'interpreter utils':
102        'scripts/gen_interpreter_utils.py',
103    'Metal format table':
104        'src/libANGLE/renderer/metal/gen_mtl_format_table.py',
105    'Metal default shaders':
106        'src/libANGLE/renderer/metal/shaders/gen_mtl_internal_shaders.py',
107    'OpenGL dispatch table':
108        'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py',
109    'overlay fonts':
110        'src/libANGLE/gen_overlay_fonts.py',
111    'overlay widgets':
112        'src/libANGLE/gen_overlay_widgets.py',
113    'packed enum':
114        'src/common/gen_packed_gl_enums.py',
115    'proc table':
116        'scripts/gen_proc_table.py',
117    'restricted traces':
118        'src/tests/restricted_traces/gen_restricted_traces.py',
119    'SPIR-V helpers':
120        'src/common/spirv/gen_spirv_builder_and_parser.py',
121    'Static builtins':
122        'src/compiler/translator/gen_builtin_symbols.py',
123    'uniform type':
124        'src/common/gen_uniform_type_table.py',
125    'Vulkan format':
126        'src/libANGLE/renderer/vulkan/gen_vk_format_table.py',
127    'Vulkan internal shader programs':
128        'src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py',
129    'Vulkan mandatory format support table':
130        'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py',
131    'WebGPU format':
132        'src/libANGLE/renderer/wgpu/gen_wgpu_format_table.py',
133}
134
135
136# Fast and supports --verify-only without hashes.
137hashless_generators = {
138    'ANGLE features': 'include/platform/gen_features.py',
139    'Test spec JSON': 'infra/specs/generate_test_spec_json.py',
140}
141
142
143def md5(fname):
144    hash_md5 = hashlib.md5()
145    with open(fname, 'rb') as f:
146        if sys.platform.startswith('win') or sys.platform == 'cygwin':
147            # Beware: Windows crlf + git behavior + unicode in some files
148            hash_md5.update(f.read().replace(b'\r\n', b'\n'))
149        else:
150            for chunk in iter(lambda: f.read(4096), b''):
151                hash_md5.update(chunk)
152    return hash_md5.hexdigest()
153
154
155def get_hash_file_name(name):
156    return name.replace(' ', '_').replace('/', '_') + '.json'
157
158
159def any_hash_dirty(name, filenames, new_hashes, old_hashes):
160    found_dirty_hash = False
161
162    for fname in filenames:
163        if not os.path.isfile(os.path.join(root_dir, fname)):
164            print('File not found: "%s". Code gen dirty for %s' % (fname, name))
165            found_dirty_hash = True
166        else:
167            new_hashes[fname] = md5(fname)
168            if (not fname in old_hashes) or (old_hashes[fname] != new_hashes[fname]):
169                print('Hash for "%s" dirty for %s generator.' % (fname, name))
170                found_dirty_hash = True
171    return found_dirty_hash
172
173
174def any_old_hash_missing(all_new_hashes, all_old_hashes):
175    result = False
176    for file, old_hashes in all_old_hashes.items():
177        if file not in all_new_hashes:
178            print('"%s" does not exist. Code gen dirty.' % file)
179            result = True
180        else:
181            for name, _ in old_hashes.items():
182                if name not in all_new_hashes[file]:
183                    print('Hash for %s is missing from "%s". Code gen is dirty.' % (name, file))
184                    result = True
185    return result
186
187
188def update_output_hashes(script, outputs, new_hashes):
189    for output in outputs:
190        if not os.path.isfile(output):
191            print('Output is missing from %s: %s' % (script, output))
192            sys.exit(1)
193        new_hashes[output] = md5(output)
194
195
196def load_hashes():
197    hashes = {}
198    for file in os.listdir(hash_dir):
199        hash_fname = os.path.join(hash_dir, file)
200        with open(hash_fname) as hash_file:
201            try:
202                hashes[file] = json.load(hash_file)
203            except ValueError:
204                raise Exception("Could not decode JSON from %s" % file)
205    return hashes
206
207
208def main():
209    all_old_hashes = load_hashes()
210    all_new_hashes = {}
211    any_dirty = False
212    format_workaround = False
213
214    parser = argparse.ArgumentParser(description='Generate ANGLE internal code.')
215    parser.add_argument(
216        '-v',
217        '--verify-no-dirty',
218        dest='verify_only',
219        action='store_true',
220        help='verify hashes are not dirty')
221    parser.add_argument(
222        '-g', '--generator', action='append', nargs='*', type=str, dest='specified_generators'),
223
224    args = parser.parse_args()
225
226    ranGenerators = generators
227    runningSingleGenerator = False
228    if (args.specified_generators):
229        ranGenerators = {k: v for k, v in generators.items() if k in args.specified_generators[0]}
230        runningSingleGenerator = True
231
232    if len(ranGenerators) == 0:
233        print("No valid generators specified.")
234        return 1
235
236    # Just get 'inputs' and 'outputs' from scripts but this runs the scripts so it's a bit slow
237    infos = {}
238    with futures.ThreadPoolExecutor(max_workers=8) as executor:
239        for _, script in sorted(ranGenerators.items()):
240            infos[script] = executor.submit(auto_script, script)
241
242    for name, script in sorted(ranGenerators.items()):
243        info = infos[script].result()
244        fname = get_hash_file_name(name)
245        filenames = info['inputs'] + info['outputs'] + [script]
246        new_hashes = {}
247        if fname not in all_old_hashes:
248            all_old_hashes[fname] = {}
249        if any_hash_dirty(name, filenames, new_hashes, all_old_hashes[fname]):
250            any_dirty = True
251            if "preprocessor" in name:
252                format_workaround = True
253
254            if not args.verify_only:
255                print('Running ' + name + ' code generator')
256
257                exe = get_executable_name(script)
258                subprocess.check_call([exe, os.path.basename(script)],
259                                      cwd=get_child_script_dirname(script))
260
261        # Update the hash dictionary.
262        all_new_hashes[fname] = new_hashes
263
264    if not runningSingleGenerator and any_old_hash_missing(all_new_hashes, all_old_hashes):
265        any_dirty = True
266
267    # Handle hashless_generators separately as these don't have hash maps.
268    hashless_generators_dirty = False
269    for name, script in sorted(hashless_generators.items()):
270        cmd = [get_executable_name(script), os.path.basename(script)]
271        rc = subprocess.call(cmd + ['--verify-only'], cwd=get_child_script_dirname(script))
272        if rc != 0:
273            print(name + ' generator dirty')
274            # Don't set any_dirty as we don't need git cl format in this case.
275            hashless_generators_dirty = True
276
277            if not args.verify_only:
278                print('Running ' + name + ' code generator')
279                subprocess.check_call(cmd, cwd=get_child_script_dirname(script))
280
281    if args.verify_only:
282        return int(any_dirty or hashless_generators_dirty)
283
284    if any_dirty:
285        args = ['git.bat'] if os.name == 'nt' else ['git']
286        args += ['cl', 'format']
287        print('Calling git cl format')
288        subprocess.check_call(args)
289        if format_workaround:
290            # Some formattings fail, and thus we can never submit such a cl because
291            # of vicious circle of needing clean formatting but formatting not generating
292            # clean formatting.
293            print('Calling git cl format again')
294            subprocess.check_call(args)
295
296
297        # Update the output hashes again since they can be formatted.
298        for name, script in sorted(ranGenerators.items()):
299            info = auto_script(script)
300            fname = get_hash_file_name(name)
301            update_output_hashes(name, info['outputs'], all_new_hashes[fname])
302
303        for fname, new_hashes in all_new_hashes.items():
304            hash_fname = os.path.join(hash_dir, fname)
305            with open(hash_fname, "w") as f:
306                json.dump(new_hashes, f, indent=2, sort_keys=True, separators=(',', ':\n    '))
307                f.write('\n')  # json.dump doesn't end with newline
308
309    return 0
310
311
312if __name__ == '__main__':
313    sys.exit(main())
314