1# Copyright (C) 2022 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# A collection of utilities for extracting build rule information from GN 16# projects. 17 18import copy 19import json 20import logging as log 21import os 22import re 23import collections 24 25LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library', 26 'source_set') 27# This is a list of java files that should not be collected 28# as they don't exist right now downstream (eg: apihelpers, cronetEngineBuilderTest). 29# This is temporary solution until they are up-streamed. 30JAVA_FILES_TO_IGNORE = ( 31 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ByteArrayCronetCallback.java", 32 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ContentTypeParametersParser.java", 33 "//components/cronet/android/api/src/org/chromium/net/apihelpers/CronetRequestCompletionListener.java", 34 "//components/cronet/android/api/src/org/chromium/net/apihelpers/CronetResponse.java", 35 "//components/cronet/android/api/src/org/chromium/net/apihelpers/ImplicitFlowControlCallback.java", 36 "//components/cronet/android/api/src/org/chromium/net/apihelpers/InMemoryTransformCronetCallback.java", 37 "//components/cronet/android/api/src/org/chromium/net/apihelpers/JsonCronetCallback.java", 38 "//components/cronet/android/api/src/org/chromium/net/apihelpers/RedirectHandler.java", 39 "//components/cronet/android/api/src/org/chromium/net/apihelpers/RedirectHandlers.java", 40 "//components/cronet/android/api/src/org/chromium/net/apihelpers/StringCronetCallback.java", 41 "//components/cronet/android/api/src/org/chromium/net/apihelpers/UrlRequestCallbacks.java", 42 "//components/cronet/android/test/javatests/src/org/chromium/net/CronetEngineBuilderTest.java", 43 # Api helpers does not exist downstream, hence the tests shouldn't be collected. 44 "//components/cronet/android/test/javatests/src/org/chromium/net/apihelpers/ContentTypeParametersParserTest.java", 45 # androidx-multidex is disabled on unbundled branches. 46 "//base/test/android/java/src/org/chromium/base/multidex/ChromiumMultiDexInstaller.java", 47) 48RESPONSE_FILE = '{{response_file_name}}' 49TESTING_SUFFIX = "__testing" 50AIDL_INCLUDE_DIRS_REGEX = r'--includes=\[(.*)\]' 51AIDL_IMPORT_DIRS_REGEX = r'--imports=\[(.*)\]' 52PROTO_IMPORT_DIRS_REGEX = r'--import-dir=(.*)' 53 54 55def repo_root(): 56 """Returns an absolute path to the repository root.""" 57 return os.path.join(os.path.realpath(os.path.dirname(__file__)), 58 os.path.pardir) 59 60 61def _clean_string(str): 62 return str.replace('\\', '').replace('../../', '').replace('"', '').strip() 63 64 65def _clean_aidl_import(orig_str): 66 str = _clean_string(orig_str) 67 src_idx = str.find("src/") 68 if src_idx == -1: 69 raise ValueError(f"Unable to clean aidl import {orig_str}") 70 return str[:src_idx + len("src")] 71 72 73def _extract_includes_from_aidl_args(args): 74 ret = [] 75 for arg in args: 76 is_match = re.match(AIDL_INCLUDE_DIRS_REGEX, arg) 77 if is_match: 78 local_includes = is_match.group(1).split(",") 79 ret += [ 80 _clean_string(local_include) 81 for local_include in local_includes 82 ] 83 # Treat imports like include for aidl by removing the package suffix. 84 is_match = re.match(AIDL_IMPORT_DIRS_REGEX, arg) 85 if is_match: 86 local_imports = is_match.group(1).split(",") 87 # Skip "third_party/android_sdk/public/platforms/android-34/framework.aidl" because Soong 88 # already links against the AIDL framework implicitly. 89 ret += [ 90 _clean_aidl_import(local_import) 91 for local_import in local_imports 92 if "framework.aidl" not in local_import 93 ] 94 return ret 95 96 97def contains_aidl(sources): 98 return any([src.endswith(".aidl") for src in sources]) 99 100 101def _get_jni_registration_deps(gn_target_name, gn_desc): 102 # the dependencies are stored within another target with the same name 103 # and a __java_sources suffix, see 104 # https://source.chromium.org/chromium/chromium/src/+/main:third_party/jni_zero/jni_zero.gni;l=117;drc=78e8e27142ed3fddf04fbcd122507517a87cb9ad 105 # for the auto-generated target name. 106 jni_registration_java_target = f'{gn_target_name}__java_sources' 107 if jni_registration_java_target in gn_desc.keys(): 108 return gn_desc[jni_registration_java_target]["deps"] 109 return set() 110 111 112def label_to_path(label): 113 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 114 assert label.startswith('//') 115 return label[2:] or "./" 116 117 118def label_without_toolchain(label): 119 """Strips the toolchain from a GN label. 120 121 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 122 gcc_like_host) without the parenthesised toolchain part. 123 """ 124 return label.split('(')[0] 125 126 127def _is_java_source(src): 128 return os.path.splitext(src)[1] == '.java' and not src.startswith("//out/") 129 130 131class GnParser(object): 132 """A parser with some cleverness for GN json desc files 133 134 The main goals of this parser are: 135 1) Deal with the fact that other build systems don't have an equivalent 136 notion to GN's source_set. Conversely to Bazel's and Soong's filegroups, 137 GN source_sets expect that dependencies, cflags and other source_set 138 properties propagate up to the linker unit (static_library, executable or 139 shared_library). This parser simulates the same behavior: when a 140 source_set is encountered, some of its variables (cflags and such) are 141 copied up to the dependent targets. This is to allow gen_xxx to create 142 one filegroup for each source_set and then squash all the other flags 143 onto the linker unit. 144 2) Detect and special-case protobuf targets, figuring out the protoc-plugin 145 being used. 146 """ 147 148 class Target(object): 149 """Reperesents A GN target. 150 151 Maked properties are propagated up the dependency chain when a 152 source_set dependency is encountered. 153 """ 154 155 class Arch(): 156 """Architecture-dependent properties 157 """ 158 159 def __init__(self): 160 self.sources = set() 161 self.cflags = set() 162 self.defines = set() 163 self.include_dirs = set() 164 self.deps = set() 165 self.transitive_static_libs_deps = set() 166 self.ldflags = set() 167 168 # These are valid only for type == 'action' 169 self.inputs = set() 170 self.outputs = set() 171 self.args = [] 172 self.response_file_contents = '' 173 174 def __init__(self, name, type): 175 self.name = name # e.g. //src/ipc:ipc 176 177 VALID_TYPES = ('static_library', 'shared_library', 'executable', 178 'group', 'action', 'source_set', 'proto_library', 179 'copy', 'action_foreach') 180 assert (type in VALID_TYPES) 181 self.type = type 182 self.testonly = False 183 self.toolchain = None 184 185 # These are valid only for type == proto_library. 186 # This is typically: 'proto', 'protozero', 'ipc'. 187 self.proto_plugin = None 188 self.proto_paths = set() 189 self.proto_exports = set() 190 self.proto_in_dir = "" 191 192 # TODO(primiano): consider whether the public section should be part of 193 # bubbled-up sources. 194 self.public_headers = set() # 'public' 195 196 # These are valid only for type == 'action' 197 self.script = '' 198 199 # These variables are propagated up when encountering a dependency 200 # on a source_set target. 201 self.libs = set() 202 self.proto_deps = set() 203 self.rtti = False 204 205 # TODO: come up with a better way to only run this once. 206 # is_finalized tracks whether finalize() was called on this target. 207 self.is_finalized = False 208 # 'common' is a pseudo-architecture used to store common architecture dependent properties (to 209 # make handling of common vs architecture-specific arguments more consistent). 210 self.arch = {'common': self.Arch()} 211 212 # This is used to get the name/version of libcronet 213 self.output_name = None 214 # Local Includes used for AIDL 215 self.local_aidl_includes = set() 216 # Each java_target will contain the transitive java sources found 217 # in generate_jni type target. 218 self.transitive_jni_java_sources = set() 219 # Deps for JNI Registration. Those are not added to deps so that 220 # the generated module would not depend on those deps. 221 self.jni_registration_java_deps = set() 222 # Path to the java jar path. This is used if the java library is 223 # an import of a JAR like `android_java_prebuilt` targets in GN 224 self.jar_path = "" 225 226 # Properties to forward access to common arch. 227 # TODO: delete these after the transition has been completed. 228 @property 229 def sources(self): 230 return self.arch['common'].sources 231 232 @sources.setter 233 def sources(self, val): 234 self.arch['common'].sources = val 235 236 @property 237 def inputs(self): 238 return self.arch['common'].inputs 239 240 @inputs.setter 241 def inputs(self, val): 242 self.arch['common'].inputs = val 243 244 @property 245 def outputs(self): 246 return self.arch['common'].outputs 247 248 @outputs.setter 249 def outputs(self, val): 250 self.arch['common'].outputs = val 251 252 @property 253 def args(self): 254 return self.arch['common'].args 255 256 @args.setter 257 def args(self, val): 258 self.arch['common'].args = val 259 260 @property 261 def response_file_contents(self): 262 return self.arch['common'].response_file_contents 263 264 @response_file_contents.setter 265 def response_file_contents(self, val): 266 self.arch['common'].response_file_contents = val 267 268 @property 269 def cflags(self): 270 return self.arch['common'].cflags 271 272 @property 273 def defines(self): 274 return self.arch['common'].defines 275 276 @property 277 def deps(self): 278 return self.arch['common'].deps 279 280 @deps.setter 281 def deps(self, val): 282 self.arch['common'].deps = val 283 284 @property 285 def include_dirs(self): 286 return self.arch['common'].include_dirs 287 288 @property 289 def ldflags(self): 290 return self.arch['common'].ldflags 291 292 def host_supported(self): 293 return 'host' in self.arch 294 295 def device_supported(self): 296 return any( 297 [name.startswith('android') for name in self.arch.keys()]) 298 299 def is_linker_unit_type(self): 300 return self.type in LINKER_UNIT_TYPES 301 302 def __lt__(self, other): 303 if isinstance(other, self.__class__): 304 return self.name < other.name 305 raise TypeError( 306 '\'<\' not supported between instances of \'%s\' and \'%s\'' % 307 (type(self).__name__, type(other).__name__)) 308 309 def __repr__(self): 310 return json.dumps( 311 { 312 k: (list(sorted(v)) if isinstance(v, set) else v) 313 for (k, v) in self.__dict__.items() 314 }, 315 indent=4, 316 sort_keys=True) 317 318 def update(self, other, arch): 319 for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags', 320 'proto_deps', 'libs', 'proto_paths'): 321 getattr(self, key).update(getattr(other, key, [])) 322 323 for key_in_arch in ('cflags', 'defines', 'include_dirs', 'deps', 324 'ldflags'): 325 getattr(self.arch[arch], key_in_arch).update( 326 getattr(other.arch[arch], key_in_arch, [])) 327 328 def get_archs(self): 329 """ Returns a dict of archs without the common arch """ 330 return { 331 arch: val 332 for arch, val in self.arch.items() if arch != 'common' 333 } 334 335 def _finalize_set_attribute(self, key): 336 # Target contains the intersection of arch-dependent properties 337 getattr(self, key).update( 338 set.intersection( 339 * 340 [getattr(arch, key) 341 for arch in self.get_archs().values()])) 342 343 # Deduplicate arch-dependent properties 344 for arch in self.get_archs().values(): 345 getattr(arch, key).difference_update(getattr(self, key)) 346 347 def _finalize_non_set_attribute(self, key): 348 # Only when all the arch has the same non empty value, move the value to the target common 349 val = getattr(list(self.get_archs().values())[0], key) 350 if val and all([ 351 val == getattr(arch, key) 352 for arch in self.get_archs().values() 353 ]): 354 setattr(self, key, copy.deepcopy(val)) 355 356 def _finalize_attribute(self, key): 357 val = getattr(self, key) 358 if isinstance(val, set): 359 self._finalize_set_attribute(key) 360 elif isinstance(val, (list, str)): 361 self._finalize_non_set_attribute(key) 362 else: 363 raise TypeError(f'Unsupported type: {type(val)}') 364 365 def finalize(self): 366 """Move common properties out of arch-dependent subobjects to Target object. 367 368 TODO: find a better name for this function. 369 """ 370 if self.is_finalized: 371 return 372 self.is_finalized = True 373 374 if len(self.arch) == 1: 375 return 376 377 for key in ('sources', 'cflags', 'defines', 'include_dirs', 'deps', 378 'inputs', 'outputs', 'args', 'response_file_contents', 379 'ldflags'): 380 self._finalize_attribute(key) 381 382 def get_target_name(self): 383 return self.name[self.name.find(":") + 1:] 384 385 def __init__(self, builtin_deps): 386 self.builtin_deps = builtin_deps 387 self.all_targets = {} 388 self.jni_java_sources = set() 389 390 def _get_response_file_contents(self, action_desc): 391 # response_file_contents are formatted as: 392 # ['--flags', '--flag=true && false'] and need to be formatted as: 393 # '--flags --flag=\"true && false\"' 394 flags = action_desc.get('response_file_contents', []) 395 formatted_flags = [] 396 for flag in flags: 397 if '=' in flag: 398 key, val = flag.split('=') 399 formatted_flags.append('%s=\\"%s\\"' % (key, val)) 400 else: 401 formatted_flags.append(flag) 402 403 return ' '.join(formatted_flags) 404 405 def _is_java_group(self, type_, target_name): 406 # Per https://chromium.googlesource.com/chromium/src/build/+/HEAD/android/docs/java_toolchain.md 407 # java target names must end in "_java". 408 # TODO: There are some other possible variations we might need to support. 409 return type_ == 'group' and target_name.endswith('_java') 410 411 def _get_arch(self, toolchain): 412 if toolchain == '//build/toolchain/android:android_clang_x86': 413 return 'android_x86' 414 elif toolchain == '//build/toolchain/android:android_clang_x64': 415 return 'android_x86_64' 416 elif toolchain == '//build/toolchain/android:android_clang_arm': 417 return 'android_arm' 418 elif toolchain == '//build/toolchain/android:android_clang_arm64': 419 return 'android_arm64' 420 else: 421 return 'host' 422 423 def get_target(self, gn_target_name): 424 """Returns a Target object from the fully qualified GN target name. 425 426 get_target() requires that parse_gn_desc() has already been called. 427 """ 428 # Run this every time as parse_gn_desc can be called at any time. 429 for target in self.all_targets.values(): 430 target.finalize() 431 432 return self.all_targets[label_without_toolchain(gn_target_name)] 433 434 def parse_gn_desc(self, 435 gn_desc, 436 gn_target_name, 437 java_group_name=None, 438 is_test_target=False): 439 """Parses a gn desc tree and resolves all target dependencies. 440 441 It bubbles up variables from source_set dependencies as described in the 442 class-level comments. 443 """ 444 # Use name without toolchain for targets to support targets built for 445 # multiple archs. 446 target_name = label_without_toolchain(gn_target_name) 447 desc = gn_desc[gn_target_name] 448 type_ = desc['type'] 449 arch = self._get_arch(desc['toolchain']) 450 metadata = desc.get("metadata", {}) 451 452 if is_test_target: 453 target_name += TESTING_SUFFIX 454 455 target = self.all_targets.get(target_name) 456 if target is None: 457 target = GnParser.Target(target_name, type_) 458 self.all_targets[target_name] = target 459 460 if arch not in target.arch: 461 target.arch[arch] = GnParser.Target.Arch() 462 else: 463 return target # Target already processed. 464 465 if 'target_type' in metadata.keys( 466 ) and metadata["target_type"][0] == 'java_library': 467 target.type = 'java_library' 468 469 if target.name in self.builtin_deps: 470 # return early, no need to parse any further as the module is a builtin. 471 return target 472 473 target.testonly = desc.get('testonly', False) 474 475 deps = desc.get("deps", {}) 476 if desc.get("script", 477 "") == "//tools/protoc_wrapper/protoc_wrapper.py": 478 target.type = 'proto_library' 479 target.proto_plugin = "proto" 480 target.proto_paths.update(self.get_proto_paths(desc)) 481 target.proto_exports.update(self.get_proto_exports(desc)) 482 target.proto_in_dir = self.get_proto_in_dir(desc) 483 target.arch[arch].sources.update(desc.get('sources', [])) 484 target.arch[arch].inputs.update(desc.get('inputs', [])) 485 elif target.type == 'source_set': 486 target.arch[arch].sources.update( 487 source for source in desc.get('sources', []) 488 if not source.startswith("//out")) 489 elif target.is_linker_unit_type(): 490 target.arch[arch].sources.update( 491 source for source in desc.get('sources', []) 492 if not source.startswith("//out")) 493 elif target.type == 'java_library': 494 sources = set() 495 for java_source in metadata.get("source_files", []): 496 if not java_source.startswith( 497 "//out") and java_source not in JAVA_FILES_TO_IGNORE: 498 sources.add(java_source) 499 target.sources.update(sources) 500 # Metadata attributes must be list, for jar_path, it is always a list 501 # of size one, the first element is an empty string if `jar_path` is not 502 # defined otherwise it is a path. 503 if metadata.get("jar_path", [""])[0]: 504 target.jar_path = label_to_path(metadata["jar_path"][0]) 505 deps = metadata.get("all_deps", {}) 506 log.info('Found Java Target %s', target.name) 507 elif target.script == "//build/android/gyp/aidl.py": 508 target.type = "java_library" 509 target.sources.update(desc.get('sources', {})) 510 target.local_aidl_includes = _extract_includes_from_aidl_args( 511 desc.get('args', '')) 512 elif target.type in ['action', 'action_foreach']: 513 target.arch[arch].inputs.update(desc.get('inputs', [])) 514 target.arch[arch].sources.update(desc.get('sources', [])) 515 outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']] 516 target.arch[arch].outputs.update(outs) 517 # While the arguments might differ, an action should always use the same script for every 518 # architecture. (gen_android_bp's get_action_sanitizer actually relies on this fact. 519 target.script = desc['script'] 520 target.arch[arch].args = desc['args'] 521 target.arch[ 522 arch].response_file_contents = self._get_response_file_contents( 523 desc) 524 # _get_jni_registration_deps will return the dependencies of a target if 525 # the target is of type `generate_jni_registration` otherwise it will 526 # return an empty set. 527 target.jni_registration_java_deps.update( 528 _get_jni_registration_deps(gn_target_name, gn_desc)) 529 # JNI java sources are embedded as metadata inside `jni_headers` targets. 530 # See https://source.chromium.org/chromium/chromium/src/+/main:third_party/jni_zero/jni_zero.gni;l=421;drc=78e8e27142ed3fddf04fbcd122507517a87cb9ad 531 # for more details 532 target.transitive_jni_java_sources.update( 533 metadata.get("jni_source_files_abs", set())) 534 self.jni_java_sources.update( 535 metadata.get("jni_source_files_abs", set())) 536 elif target.type == 'copy': 537 # TODO: copy rules are not currently implemented. 538 pass 539 elif target.type == 'group': 540 # Groups are bubbled upward without creating an equivalent GN target. 541 pass 542 else: 543 raise Exception( 544 f"Encountered GN target with unknown type\nCulprit target: {gn_target_name}\ntype: {type_}" 545 ) 546 547 # Default for 'public' is //* - all headers in 'sources' are public. 548 # TODO(primiano): if a 'public' section is specified (even if empty), then 549 # the rest of 'sources' is considered inaccessible by gn. Consider 550 # emulating that, so that generated build files don't end up with overly 551 # accessible headers. 552 public_headers = [x for x in desc.get('public', []) if x != '*'] 553 target.public_headers.update(public_headers) 554 555 target.arch[arch].cflags.update( 556 desc.get('cflags', []) + desc.get('cflags_cc', [])) 557 target.libs.update(desc.get('libs', [])) 558 target.arch[arch].ldflags.update(desc.get('ldflags', [])) 559 target.arch[arch].defines.update(desc.get('defines', [])) 560 target.arch[arch].include_dirs.update(desc.get('include_dirs', [])) 561 target.output_name = desc.get('output_name', None) 562 if "-frtti" in target.arch[arch].cflags: 563 target.rtti = True 564 565 for gn_dep_name in set(target.jni_registration_java_deps): 566 dep = self.parse_gn_desc(gn_desc, gn_dep_name, java_group_name, 567 is_test_target) 568 target.transitive_jni_java_sources.update( 569 dep.transitive_jni_java_sources) 570 571 # Recurse in dependencies. 572 for gn_dep_name in set(deps): 573 dep = self.parse_gn_desc(gn_desc, gn_dep_name, java_group_name, 574 is_test_target) 575 576 if dep.type == 'proto_library': 577 target.proto_deps.add(dep.name) 578 elif dep.type == 'group': 579 target.update(dep, 580 arch) # Bubble up groups's cflags/ldflags etc. 581 elif dep.type in ['action', 'action_foreach', 'copy']: 582 target.arch[arch].deps.add(dep.name) 583 target.transitive_jni_java_sources.update( 584 dep.transitive_jni_java_sources) 585 elif dep.is_linker_unit_type(): 586 target.arch[arch].deps.add(dep.name) 587 elif dep.type == 'java_library': 588 target.deps.add(dep.name) 589 target.transitive_jni_java_sources.update( 590 dep.transitive_jni_java_sources) 591 592 if dep.type in ['static_library', 'source_set']: 593 # Bubble up static_libs and source_set. Necessary, since soong does not propagate 594 # static_libs up the build tree. 595 # Source sets are later translated to static_libraries, so it makes sense 596 # to reuse transitive_static_libs_deps. 597 target.arch[arch].transitive_static_libs_deps.add(dep.name) 598 599 if arch in dep.arch: 600 target.arch[arch].transitive_static_libs_deps.update( 601 dep.arch[arch].transitive_static_libs_deps) 602 target.arch[arch].deps.update( 603 target.arch[arch].transitive_static_libs_deps) 604 return target 605 606 def get_proto_exports(self, proto_desc): 607 # exports in metadata will be available for source_set targets. 608 metadata = proto_desc.get('metadata', {}) 609 return metadata.get('exports', []) 610 611 def get_proto_paths(self, proto_desc): 612 args = proto_desc.get('args') 613 proto_paths = set() 614 for arg in args: 615 is_match = re.match(PROTO_IMPORT_DIRS_REGEX, arg) 616 if is_match: 617 proto_paths.add(re.sub('^\.\./\.\./', '', is_match.group(1))) 618 return proto_paths 619 620 def get_proto_in_dir(self, proto_desc): 621 args = proto_desc.get('args') 622 return re.sub('^\.\./\.\./', '', 623 args[args.index('--proto-in-dir') + 1]) 624