1#! /usr/bin/env python3 2assert __name__ == '__main__' 3 4''' 5To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and 6python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come 7before python3 in your path. 8 9Upstream: https://chromium.googlesource.com/angle/angle 10 11Our repo: https://github.com/mozilla/angle 12It has branches like 'firefox-60' which is the branch we use for pulling into 13Gecko with this script. 14 15This script leaves a record of the merge-base and cherry-picks that we pull into 16Gecko. (gfx/angle/cherries.log) 17 18ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/ 19An easy choice is to grab Chrome's Beta's ANGLE branch. 20 21## Usage 22 23Prepare your env: 24 25~~~ 26export PATH="$PATH:/path/to/depot_tools" 27~~~ 28 29If this is a new repo, don't forget: 30 31~~~ 32# In the angle repo: 33./scripts/bootstrap.py 34gclient sync 35~~~ 36 37Update: (in the angle repo) 38 39~~~ 40# In the angle repo: 41/path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX 42git push moz # Push the firefox-XX branch to github.com/mozilla/angle 43~~~~ 44 45''' 46 47import json 48import os 49import pathlib 50import re 51import shutil 52import subprocess 53import sys 54from typing import * # mypy annotations 55 56SCRIPT_DIR = os.path.dirname(__file__) 57 58GN_ENV = dict(os.environ) 59# We need to set DEPOT_TOOLS_WIN_TOOLCHAIN to 0 for non-Googlers, but otherwise 60# leave it unset since vs_toolchain.py assumes that the user is a Googler with 61# the Visual Studio files in depot_tools if DEPOT_TOOLS_WIN_TOOLCHAIN is not 62# explicitly set to 0. 63vs_found = False 64vs_dir = os.path.join(SCRIPT_DIR, '..', 'third_party', 'depot_tools', 'win_toolchain', 'vs_files') 65if not os.path.isdir(vs_dir): 66 GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' 67 68if len(sys.argv) < 3: 69 sys.exit('Usage: export_targets.py OUT_DIR ROOTS...') 70 71(OUT_DIR, *ROOTS) = sys.argv[1:] 72for x in ROOTS: 73 assert x.startswith('//:') 74 75# ------------------------------------------------------------------------------ 76 77def run_checked(*args, **kwargs): 78 print(' ', args, file=sys.stderr) 79 sys.stderr.flush() 80 return subprocess.run(args, check=True, **kwargs) 81 82 83def sortedi(x): 84 return sorted(x, key=str.lower) 85 86 87def dag_traverse(root_keys: Sequence[str], pre_recurse_func: Callable[[str], list]): 88 visited_keys: Set[str] = set() 89 90 def recurse(key): 91 if key in visited_keys: 92 return 93 visited_keys.add(key) 94 95 t = pre_recurse_func(key) 96 try: 97 (next_keys, post_recurse_func) = t 98 except ValueError: 99 (next_keys,) = t 100 post_recurse_func = None 101 102 for x in next_keys: 103 recurse(x) 104 105 if post_recurse_func: 106 post_recurse_func(key) 107 return 108 109 for x in root_keys: 110 recurse(x) 111 return 112 113# ------------------------------------------------------------------------------ 114 115print('Importing graph', file=sys.stderr) 116 117try: 118 p = run_checked('gn', 'desc', '--format=json', str(OUT_DIR), '*', stdout=subprocess.PIPE, 119 env=GN_ENV, shell=(True if sys.platform == 'win32' else False)) 120except subprocess.CalledProcessError: 121 sys.stderr.buffer.write(b'"gn desc" failed. Is depot_tools in your PATH?\n') 122 exit(1) 123 124# - 125 126print('\nProcessing graph', file=sys.stderr) 127descs = json.loads(p.stdout.decode()) 128 129# Ready to traverse 130# ------------------------------------------------------------------------------ 131 132LIBRARY_TYPES = ('shared_library', 'static_library') 133 134def flattened_target(target_name: str, descs: dict, stop_at_lib: bool =True) -> dict: 135 flattened = dict(descs[target_name]) 136 137 EXPECTED_TYPES = LIBRARY_TYPES + ('source_set', 'group', 'action') 138 139 def pre(k): 140 dep = descs[k] 141 142 dep_type = dep['type'] 143 deps = dep['deps'] 144 if stop_at_lib and dep_type in LIBRARY_TYPES: 145 return ((),) 146 147 if dep_type == 'copy': 148 assert not deps, (target_name, dep['deps']) 149 else: 150 assert dep_type in EXPECTED_TYPES, (k, dep_type) 151 for (k,v) in dep.items(): 152 if type(v) in (list, tuple, set): 153 # This is a workaround for 154 # https://bugs.chromium.org/p/gn/issues/detail?id=196, where 155 # the value of "public" can be a string instead of a list. 156 existing = flattened.get(k, []) 157 if isinstance(existing, str): 158 existing = [existing] 159 # Use temporary sets then sort them to avoid a bottleneck here 160 if not isinstance(existing, set): 161 flattened[k] = set(existing) 162 flattened[k].update(v) 163 else: 164 #flattened.setdefault(k, v) 165 pass 166 return (deps,) 167 168 dag_traverse(descs[target_name]['deps'], pre) 169 170 for k, v in flattened.items(): 171 if isinstance(v, set): 172 flattened[k] = sortedi(v) 173 return flattened 174 175# ------------------------------------------------------------------------------ 176# Check that includes are valid. (gn's version of this check doesn't seem to work!) 177 178INCLUDE_REGEX = re.compile(b'^ *# *include +([<"])([^>"]+)[>"].*$', re.MULTILINE) 179assert INCLUDE_REGEX.findall(b' # include <foo> //comment\n#include "bar"') == [(b'<', b'foo'), (b'"', b'bar')] 180 181# Most of these are ignored because this script does not currently handle 182# #includes in #ifdefs properly, so they will erroneously be marked as being 183# included, but not part of the source list. 184IGNORED_INCLUDES = { 185 b'absl/container/flat_hash_map.h', 186 b'absl/container/flat_hash_set.h', 187 b'compiler/translator/glsl/TranslatorESSL.h', 188 b'compiler/translator/glsl/TranslatorGLSL.h', 189 b'compiler/translator/hlsl/TranslatorHLSL.h', 190 b'compiler/translator/msl/TranslatorMSL.h', 191 b'compiler/translator/null/TranslatorNULL.h', 192 b'compiler/translator/spirv/TranslatorSPIRV.h', 193 b'compiler/translator/wgsl/TranslatorWGSL.h', 194 b'contrib/optimizations/slide_hash_neon.h', 195 b'dirent_on_windows.h', 196 b'dlopen_fuchsia.h', 197 b'kernel/image.h', 198 b'libANGLE/renderer/d3d/d3d11/Device11.h', 199 b'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h', 200 b'libANGLE/renderer/d3d/DisplayD3D.h', 201 b'libANGLE/renderer/d3d/RenderTargetD3D.h', 202 b'libANGLE/renderer/gl/cgl/DisplayCGL.h', 203 b'libANGLE/renderer/gl/egl/android/DisplayAndroid.h', 204 b'libANGLE/renderer/gl/egl/DisplayEGL.h', 205 b'libANGLE/renderer/gl/egl/gbm/DisplayGbm.h', 206 b'libANGLE/renderer/gl/glx/DisplayGLX.h', 207 b'libANGLE/renderer/gl/glx/DisplayGLX_api.h', 208 b'libANGLE/renderer/gl/wgl/DisplayWGL.h', 209 b'libANGLE/renderer/metal/DisplayMtl_api.h', 210 b'libANGLE/renderer/null/DisplayNULL.h', 211 b'libANGLE/renderer/vulkan/android/AHBFunctions.h', 212 b'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h', 213 b'libANGLE/renderer/vulkan/DisplayVk_api.h', 214 b'libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h', 215 b'libANGLE/renderer/vulkan/ggp/DisplayVkGGP.h', 216 b'libANGLE/renderer/vulkan/mac/DisplayVkMac.h', 217 b'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h', 218 b'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h', 219 b'libANGLE/renderer/vulkan/wayland/DisplayVkWayland.h', 220 b'loader_cmake_config.h', 221 b'loader_linux.h', 222 b'loader_windows.h', 223 b'optick.h', 224 b'spirv-tools/libspirv.h', 225 b'third_party/volk/volk.h', 226 b'vk_loader_extensions.c', 227 b'vk_snippets.h', 228 b'vulkan_android.h', 229 b'vulkan_beta.h', 230 b'vulkan_directfb.h', 231 b'vulkan_fuchsia.h', 232 b'vulkan_ggp.h', 233 b'vulkan_ios.h', 234 b'vulkan_macos.h', 235 b'vulkan_metal.h', 236 b'vulkan_sci.h', 237 b'vulkan_vi.h', 238 b'vulkan_wayland.h', 239 b'vulkan_win32.h', 240 b'vulkan_xcb.h', 241 b'vulkan_xlib.h', 242 b'vulkan_xlib_xrandr.h', 243 # rapidjson adds these include stubs into their documentation 244 # comments. Since the script doesn't skip comments they are 245 # erroneously marked as valid includes 246 b'rapidjson/...', 247 # Validation layers support building with robin hood hashing, but we are not enabling that 248 # See http://anglebug.com/42264327 249 b'robin_hood.h', 250 # Validation layers optionally use mimalloc 251 b'mimalloc-new-delete.h', 252 # From the Vulkan-Loader 253 b'winres.h', 254 # From a comment in vulkan-validation-layers/src/layers/vk_mem_alloc.h 255 b'my_custom_assert.h', 256 b'my_custom_min.h', 257 # https://bugs.chromium.org/p/gn/issues/detail?id=311 258 b'spirv/unified1/spirv.hpp11', 259 # Behind #if defined(QAT_COMPRESSION_ENABLED) in third_party/zlib/deflate.c 260 b'contrib/qat/deflate_qat.h', 261 # Behind #if defined(TRACY_ENABLE) in third_party/vulkan-validation-layers/src/layers/vulkan/generated/chassis.cpp 262 b'profiling/profiling.h', 263} 264 265IGNORED_INCLUDE_PREFIXES = { 266 b'android', 267 b'Carbon', 268 b'CoreFoundation', 269 b'CoreServices', 270 b'IOSurface', 271 b'mach', 272 b'mach-o', 273 b'OpenGL', 274 b'pci', 275 b'sys', 276 b'wrl', 277 b'X11', 278} 279 280IGNORED_DIRECTORIES = { 281 '//buildtools/third_party/libc++', 282 '//third_party/libc++/src', 283 '//third_party/abseil-cpp', 284 '//third_party/SwiftShader', 285 '//third_party/dawn', 286} 287 288def has_all_includes(target_name: str, descs: dict) -> bool: 289 for ignored_directory in IGNORED_DIRECTORIES: 290 if target_name.startswith(ignored_directory): 291 return True 292 293 flat = flattened_target(target_name, descs, stop_at_lib=False) 294 acceptable_sources = flat.get('sources', []) + flat.get('outputs', []) 295 acceptable_sources = {x.rsplit('/', 1)[-1].encode() for x in acceptable_sources} 296 297 ret = True 298 desc = descs[target_name] 299 for cur_file in desc.get('sources', []): 300 assert cur_file.startswith('/'), cur_file 301 if not cur_file.startswith('//'): 302 continue 303 cur_file = pathlib.Path(cur_file[2:]) 304 text = cur_file.read_bytes() 305 for m in INCLUDE_REGEX.finditer(text): 306 if m.group(1) == b'<': 307 continue 308 include = m.group(2) 309 if include in IGNORED_INCLUDES: 310 continue 311 try: 312 (prefix, _) = include.split(b'/', 1) 313 if prefix in IGNORED_INCLUDE_PREFIXES: 314 continue 315 except ValueError: 316 pass 317 318 include_file = include.rsplit(b'/', 1)[-1] 319 if include_file not in acceptable_sources: 320 #print(' acceptable_sources:') 321 #for x in sorted(acceptable_sources): 322 # print(' ', x) 323 print('Warning in {}: {}: Included file must be listed in the GN target or its public dependency: {}'.format(target_name, cur_file, include), file=sys.stderr) 324 ret = False 325 #print('Looks valid:', m.group()) 326 continue 327 328 return ret 329 330# - 331# Gather real targets: 332 333def gather_libraries(roots: Sequence[str], descs: dict) -> Set[str]: 334 libraries = set() 335 def fn(target_name): 336 cur = descs[target_name] 337 print(' ' + cur['type'], target_name, file=sys.stderr) 338 assert has_all_includes(target_name, descs), target_name 339 340 if cur['type'] in ('shared_library', 'static_library'): 341 libraries.add(target_name) 342 return (cur['deps'], ) 343 344 dag_traverse(roots, fn) 345 return libraries 346 347# - 348 349libraries = gather_libraries(ROOTS, descs) 350print(f'\n{len(libraries)} libraries:', file=sys.stderr) 351for k in libraries: 352 print(f' {k}', file=sys.stderr) 353print('\nstdout begins:', file=sys.stderr) 354sys.stderr.flush() 355 356# ------------------------------------------------------------------------------ 357# Output 358 359out = {k: flattened_target(k, descs) for k in libraries} 360 361for (k,desc) in out.items(): 362 dep_libs: Set[str] = set() 363 for dep_name in set(desc['deps']): 364 dep = descs[dep_name] 365 if dep['type'] in LIBRARY_TYPES: 366 dep_libs.add(dep_name) 367 desc['dep_libs'] = sortedi(dep_libs) 368 369json.dump(out, sys.stdout, indent=' ') 370exit(0) 371