1#!/usr/bin/env python3 2# 3# Copyright (C) 2021 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Builds SDK snapshots. 17 18If the environment variable TARGET_BUILD_APPS is nonempty then only the SDKs for 19the APEXes in it are built, otherwise all configured SDKs are built. 20""" 21import argparse 22import dataclasses 23import datetime 24import enum 25import functools 26import io 27import json 28import os 29from pathlib import Path 30import re 31import shutil 32import subprocess 33import sys 34import tempfile 35import typing 36from collections import defaultdict 37from typing import Callable, List 38import zipfile 39 40COPYRIGHT_BOILERPLATE = """ 41// 42// Copyright (C) 2020 The Android Open Source Project 43// 44// Licensed under the Apache License, Version 2.0 (the "License"); 45// you may not use this file except in compliance with the License. 46// You may obtain a copy of the License at 47// 48// http://www.apache.org/licenses/LICENSE-2.0 49// 50// Unless required by applicable law or agreed to in writing, software 51// distributed under the License is distributed on an "AS IS" BASIS, 52// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 53// See the License for the specific language governing permissions and 54// limitations under the License. 55// 56""".lstrip() 57 58 59@dataclasses.dataclass(frozen=True) 60class ConfigVar: 61 """Represents a Soong configuration variable""" 62 # The config variable namespace, e.g. ANDROID. 63 namespace: str 64 65 # The name of the variable within the namespace. 66 name: str 67 68 69@dataclasses.dataclass(frozen=True) 70class FileTransformation: 71 """Performs a transformation on a file within an SDK snapshot zip file.""" 72 73 # The path of the file within the SDK snapshot zip file. 74 path: str 75 76 def apply(self, producer, path, build_release): 77 """Apply the transformation to the path; changing it in place.""" 78 with open(path, "r+", encoding="utf8") as file: 79 self._apply_transformation(producer, file, build_release) 80 81 def _apply_transformation(self, producer, file, build_release): 82 """Apply the transformation to the file. 83 84 The file has been opened in read/write mode so the implementation of 85 this must read the contents and then reset the file to the beginning 86 and write the altered contents. 87 """ 88 raise NotImplementedError 89 90 91@dataclasses.dataclass(frozen=True) 92class SoongConfigVarTransformation(FileTransformation): 93 94 # The configuration variable that will control the prefer setting. 95 configVar: ConfigVar 96 97 # The line containing the prefer property. 98 PREFER_LINE = " prefer: false," 99 100 def _apply_transformation(self, producer, file, build_release): 101 raise NotImplementedError 102 103 104@dataclasses.dataclass(frozen=True) 105class SoongConfigBoilerplateInserter(SoongConfigVarTransformation): 106 """Transforms an Android.bp file to add soong config boilerplate. 107 108 The boilerplate allows the prefer setting of the modules to be controlled 109 through a Soong configuration variable. 110 """ 111 112 # The configuration variable that will control the prefer setting. 113 configVar: ConfigVar 114 115 # The prefix to use for the soong config module types. 116 configModuleTypePrefix: str 117 118 def config_module_type(self, module_type): 119 return self.configModuleTypePrefix + module_type 120 121 def _apply_transformation(self, producer, file, build_release): 122 # TODO(b/174997203): Remove this when we have a proper way to control 123 # prefer flags in Mainline modules. 124 125 header_lines = [] 126 for line in file: 127 line = line.rstrip("\n") 128 if not line.startswith("//"): 129 break 130 header_lines.append(line) 131 132 config_module_types = set() 133 134 content_lines = [] 135 for line in file: 136 line = line.rstrip("\n") 137 138 # Check to see whether the line is the start of a new module type, 139 # e.g. <module-type> { 140 module_header = re.match("([a-z0-9_]+) +{$", line) 141 if not module_header: 142 # It is not so just add the line to the output and skip to the 143 # next line. 144 content_lines.append(line) 145 continue 146 147 module_type = module_header.group(1) 148 module_content = [] 149 150 # Iterate over the Soong module contents 151 for module_line in file: 152 module_line = module_line.rstrip("\n") 153 154 # When the end of the module has been reached then exit. 155 if module_line == "}": 156 break 157 158 # Check to see if the module is an unversioned module, i.e. 159 # without @<version>. If it is then it needs to have the soong 160 # config boilerplate added to control the setting of the prefer 161 # property. Versioned modules do not need that because they are 162 # never preferred. 163 # At the moment this differentiation between versioned and 164 # unversioned relies on the fact that the unversioned modules 165 # set "prefer: false", while the versioned modules do not. That 166 # is a little bit fragile so may require some additional checks. 167 if module_line != self.PREFER_LINE: 168 # The line does not indicate that the module needs the 169 # soong config boilerplate so add the line and skip to the 170 # next one. 171 module_content.append(module_line) 172 continue 173 174 # Add the soong config boilerplate instead of the line: 175 # prefer: false, 176 namespace = self.configVar.namespace 177 name = self.configVar.name 178 module_content.append(f"""\ 179 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 180 prefer: true, 181 soong_config_variables: {{ 182 {name}: {{ 183 prefer: false, 184 }}, 185 }},""") 186 187 # Add the module type to the list of module types that need to 188 # have corresponding config module types. 189 config_module_types.add(module_type) 190 191 # Change the module type to the corresponding soong config 192 # module type by adding the prefix. 193 module_type = self.config_module_type(module_type) 194 195 # Generate the module, possibly with the new module type and 196 # containing the soong config variables entry. 197 content_lines.append(module_type + " {") 198 content_lines.extend(module_content) 199 content_lines.append("}") 200 201 # Add the soong_config_module_type module definitions to the header 202 # lines so that they appear before any uses. 203 header_lines.append("") 204 for module_type in sorted(config_module_types): 205 # Create the corresponding soong config module type name by adding 206 # the prefix. 207 config_module_type = self.configModuleTypePrefix + module_type 208 header_lines.append(f""" 209// Soong config variable module type added by {producer.script}. 210soong_config_module_type {{ 211 name: "{config_module_type}", 212 module_type: "{module_type}", 213 config_namespace: "{self.configVar.namespace}", 214 bool_variables: ["{self.configVar.name}"], 215 properties: ["prefer"], 216}} 217""".lstrip()) 218 219 # Overwrite the file with the updated contents. 220 file.seek(0) 221 file.truncate() 222 file.write("\n".join(header_lines + content_lines) + "\n") 223 224 225@dataclasses.dataclass(frozen=True) 226class UseSourceConfigVarTransformation(SoongConfigVarTransformation): 227 228 def _apply_transformation(self, producer, file, build_release): 229 lines = [] 230 for line in file: 231 line = line.rstrip("\n") 232 if line != self.PREFER_LINE: 233 lines.append(line) 234 continue 235 236 # Replace "prefer: false" with "use_source_config_var {...}". 237 namespace = self.configVar.namespace 238 name = self.configVar.name 239 lines.append(f"""\ 240 // Do not prefer prebuilt if the Soong config variable "{name}" in namespace "{namespace}" is true. 241 use_source_config_var: {{ 242 config_namespace: "{namespace}", 243 var_name: "{name}", 244 }},""") 245 246 # Overwrite the file with the updated contents. 247 file.seek(0) 248 file.truncate() 249 file.write("\n".join(lines) + "\n") 250 251# Removes any lines containing prefer 252@dataclasses.dataclass(frozen=True) 253class UseNoPreferPropertyTransformation(SoongConfigVarTransformation): 254 255 def _apply_transformation(self, producer, file, build_release): 256 lines = [] 257 for line in file: 258 line = line.rstrip("\n") 259 if line != self.PREFER_LINE: 260 lines.append(line) 261 continue 262 263 # Overwrite the file with the updated contents. 264 file.seek(0) 265 file.truncate() 266 file.write("\n".join(lines) + "\n") 267 268@dataclasses.dataclass() 269class SubprocessRunner: 270 """Runs subprocesses""" 271 272 # Destination for stdout from subprocesses. 273 # 274 # This (and the following stderr) are needed to allow the tests to be run 275 # in Intellij. This ensures that the tests are run with stdout/stderr 276 # objects that work when passed to subprocess.run(stdout/stderr). Without it 277 # the tests are run with a FlushingStringIO object that has no fileno 278 # attribute - https://youtrack.jetbrains.com/issue/PY-27883. 279 stdout: io.TextIOBase = sys.stdout 280 281 # Destination for stderr from subprocesses. 282 stderr: io.TextIOBase = sys.stderr 283 284 def run(self, *args, **kwargs): 285 return subprocess.run( 286 *args, check=True, stdout=self.stdout, stderr=self.stderr, **kwargs) 287 288 289def sdk_snapshot_zip_file(snapshots_dir, sdk_name): 290 """Get the path to the sdk snapshot zip file.""" 291 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.zip") 292 293 294def sdk_snapshot_info_file(snapshots_dir, sdk_name): 295 """Get the path to the sdk snapshot info file.""" 296 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}.info") 297 298 299def sdk_snapshot_api_diff_file(snapshots_dir, sdk_name): 300 """Get the path to the sdk snapshot api diff file.""" 301 return os.path.join(snapshots_dir, f"{sdk_name}-{SDK_VERSION}-api-diff.txt") 302 303 304def sdk_snapshot_gantry_metadata_json_file(snapshots_dir, sdk_name): 305 """Get the path to the sdk snapshot gantry metadata json file.""" 306 return os.path.join(snapshots_dir, 307 f"{sdk_name}-{SDK_VERSION}-gantry-metadata.json") 308 309 310# The default time to use in zip entries. Ideally, this should be the same as is 311# used by soong_zip and ziptime but there is no strict need for that to be the 312# case. What matters is this is a fixed time so that the contents of zip files 313# created by this script do not depend on when it is run, only the inputs. 314default_zip_time = datetime.datetime(2008, 1, 1, 0, 0, 0, 0, 315 datetime.timezone.utc) 316 317 318# set the timestamps of the paths to the default_zip_time. 319def set_default_timestamp(base_dir, paths): 320 for path in paths: 321 timestamp = default_zip_time.timestamp() 322 p = os.path.join(base_dir, path) 323 os.utime(p, (timestamp, timestamp)) 324 325 326# Find the git project path of the module_sdk for given module. 327def module_sdk_project_for_module(module, root_dir): 328 module = module.rsplit(".", 1)[1] 329 # git_master-art and aosp-master-art branches does not contain project for 330 # art, hence adding special case for art. 331 if module == "art": 332 return "prebuilts/module_sdk/art" 333 if module == "btservices": 334 return "prebuilts/module_sdk/Bluetooth" 335 if module == "media": 336 return "prebuilts/module_sdk/Media" 337 if module == "nfcservices": 338 return "prebuilts/module_sdk/Nfc" 339 if module == "rkpd": 340 return "prebuilts/module_sdk/RemoteKeyProvisioning" 341 if module == "tethering": 342 return "prebuilts/module_sdk/Connectivity" 343 344 target_dir = "" 345 for dir in os.listdir(os.path.join(root_dir, "prebuilts/module_sdk/")): 346 if module.lower() in dir.lower(): 347 if target_dir: 348 print( 349 'Multiple target dirs matched "%s": %s' 350 % (module, (target_dir, dir)) 351 ) 352 sys.exit(1) 353 target_dir = dir 354 if not target_dir: 355 print("Could not find a target dir for %s" % module) 356 sys.exit(1) 357 358 return "prebuilts/module_sdk/%s" % target_dir 359 360 361@dataclasses.dataclass() 362class SnapshotBuilder: 363 """Builds sdk snapshots""" 364 365 # The path to this tool. 366 tool_path: str 367 368 # Used to run subprocesses for building snapshots. 369 subprocess_runner: SubprocessRunner 370 371 # The OUT_DIR environment variable. 372 out_dir: str 373 374 # The out/soong/mainline-sdks directory. 375 mainline_sdks_dir: str = "" 376 377 # True if apex-allowed-deps-check is to be skipped. 378 skip_allowed_deps_check: bool = False 379 380 def __post_init__(self): 381 self.mainline_sdks_dir = os.path.join(self.out_dir, 382 "soong/mainline-sdks") 383 384 def get_sdk_path(self, sdk_name): 385 """Get the path to the sdk snapshot zip file produced by soong""" 386 return os.path.join(self.mainline_sdks_dir, 387 f"{sdk_name}-{SDK_VERSION}.zip") 388 389 def build_target_paths(self, build_release, target_paths): 390 # Extra environment variables to pass to the build process. 391 extraEnv = { 392 # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but 393 # we currently break without it. 394 "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", 395 # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside 396 # sdk zip files as expected by prebuilt drop. 397 "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", 398 } 399 extraEnv.update(build_release.soong_env) 400 401 # Unless explicitly specified in the calling environment set 402 # TARGET_BUILD_VARIANT=user. 403 # This MUST be identical to the TARGET_BUILD_VARIANT used to build 404 # the corresponding APEXes otherwise it could result in different 405 # hidden API flags, see http://b/202398851#comment29 for more info. 406 target_build_variant = os.environ.get("TARGET_BUILD_VARIANT", "user") 407 cmd = [ 408 "build/soong/soong_ui.bash", 409 "--make-mode", 410 "--soong-only", 411 f"TARGET_BUILD_VARIANT={target_build_variant}", 412 "TARGET_PRODUCT=mainline_sdk", 413 "MODULE_BUILD_FROM_SOURCE=true", 414 ] + target_paths 415 if not self.skip_allowed_deps_check: 416 cmd += ["apex-allowed-deps-check"] 417 print_command(extraEnv, cmd) 418 env = os.environ.copy() 419 env.update(extraEnv) 420 self.subprocess_runner.run(cmd, env=env) 421 422 def build_snapshots(self, build_release, modules): 423 # Compute the paths to all the Soong generated sdk snapshot files 424 # required by this script. 425 paths = [ 426 sdk_snapshot_zip_file(self.mainline_sdks_dir, sdk) 427 for module in modules 428 for sdk in module.sdks 429 ] 430 431 if paths: 432 self.build_target_paths(build_release, paths) 433 return self.mainline_sdks_dir 434 435 def build_snapshots_for_build_r(self, build_release, modules): 436 # Build the snapshots as standard. 437 snapshot_dir = self.build_snapshots(build_release, modules) 438 439 # Each module will extract needed files from the original snapshot zip 440 # file and then use that to create a replacement zip file. 441 r_snapshot_dir = os.path.join(snapshot_dir, "for-R-build") 442 shutil.rmtree(r_snapshot_dir, ignore_errors=True) 443 444 build_number_file = os.path.join(self.out_dir, "soong/build_number.txt") 445 446 for module in modules: 447 apex = module.apex 448 dest_dir = os.path.join(r_snapshot_dir, apex) 449 os.makedirs(dest_dir, exist_ok=True) 450 451 # Write the bp file in the sdk_library sub-directory rather than the 452 # root of the zip file as it will be unpacked in a directory that 453 # already contains an Android.bp file that defines the corresponding 454 # apex_set. 455 bp_file = os.path.join(dest_dir, "sdk_library/Android.bp") 456 os.makedirs(os.path.dirname(bp_file), exist_ok=True) 457 458 # The first sdk in the list is the name to use. 459 sdk_name = module.sdks[0] 460 461 with open(bp_file, "w", encoding="utf8") as bp: 462 bp.write("// DO NOT EDIT. Auto-generated by the following:\n") 463 bp.write(f"// {self.tool_path}\n") 464 bp.write(COPYRIGHT_BOILERPLATE) 465 aosp_apex = google_to_aosp_name(apex) 466 467 for library in module.for_r_build.sdk_libraries: 468 module_name = library.name 469 shared_library = str(library.shared_library).lower() 470 sdk_file = sdk_snapshot_zip_file(snapshot_dir, sdk_name) 471 extract_matching_files_from_zip( 472 sdk_file, dest_dir, 473 sdk_library_files_pattern( 474 scope_pattern=r"(public|system|module-lib)", 475 name_pattern=fr"({module_name}(-removed|-stubs)?)")) 476 477 available_apexes = [f'"{aosp_apex}"'] 478 if aosp_apex != "com.android.tethering": 479 available_apexes.append(f'"test_{aosp_apex}"') 480 apex_available = ",\n ".join(available_apexes) 481 482 bp.write(f""" 483java_sdk_library_import {{ 484 name: "{module_name}", 485 owner: "google", 486 prefer: true, 487 shared_library: {shared_library}, 488 apex_available: [ 489 {apex_available}, 490 ], 491 public: {{ 492 jars: ["public/{module_name}-stubs.jar"], 493 current_api: "public/{module_name}.txt", 494 removed_api: "public/{module_name}-removed.txt", 495 sdk_version: "module_current", 496 }}, 497 system: {{ 498 jars: ["system/{module_name}-stubs.jar"], 499 current_api: "system/{module_name}.txt", 500 removed_api: "system/{module_name}-removed.txt", 501 sdk_version: "module_current", 502 }}, 503 module_lib: {{ 504 jars: ["module-lib/{module_name}-stubs.jar"], 505 current_api: "module-lib/{module_name}.txt", 506 removed_api: "module-lib/{module_name}-removed.txt", 507 sdk_version: "module_current", 508 }}, 509}} 510""") 511 512 # Copy the build_number.txt file into the snapshot. 513 snapshot_build_number_file = os.path.join( 514 dest_dir, "snapshot-creation-build-number.txt") 515 shutil.copy(build_number_file, snapshot_build_number_file) 516 517 # Make sure that all the paths being added to the zip file have a 518 # fixed timestamp so that the contents of the zip file do not depend 519 # on when this script is run, only the inputs. 520 for root, dirs, files in os.walk(dest_dir): 521 set_default_timestamp(root, dirs) 522 set_default_timestamp(root, files) 523 524 # Now zip up the files into a snapshot zip file. 525 base_file = os.path.join(r_snapshot_dir, sdk_name + "-current") 526 shutil.make_archive(base_file, "zip", dest_dir) 527 528 return r_snapshot_dir 529 530 @staticmethod 531 def does_sdk_library_support_latest_api(sdk_library): 532 if sdk_library == "conscrypt.module.platform.api" or \ 533 sdk_library == "conscrypt.module.intra.core.api": 534 return False 535 return True 536 537 def latest_api_file_targets(self, sdk_info_file): 538 # Read the sdk info file and fetch the latest scope targets. 539 with open(sdk_info_file, "r", encoding="utf8") as sdk_info_file_object: 540 sdk_info_file_json = json.loads(sdk_info_file_object.read()) 541 542 target_paths = [] 543 target_dict = {} 544 for jsonItem in sdk_info_file_json: 545 if not jsonItem["@type"] == "java_sdk_library": 546 continue 547 548 sdk_library = jsonItem["@name"] 549 if not self.does_sdk_library_support_latest_api(sdk_library): 550 continue 551 552 target_dict[sdk_library] = {} 553 for scope in jsonItem["scopes"]: 554 scope_json = jsonItem["scopes"][scope] 555 target_dict[sdk_library][scope] = {} 556 target_list = [ 557 "current_api", "latest_api", "removed_api", 558 "latest_removed_api" 559 ] 560 for target in target_list: 561 target_dict[sdk_library][scope][target] = scope_json[target] 562 target_paths.append(scope_json["latest_api"]) 563 target_paths.append(scope_json["latest_removed_api"]) 564 target_paths.append(scope_json["latest_api"] 565 .replace(".latest", ".latest.extension_version")) 566 target_paths.append(scope_json["latest_removed_api"] 567 .replace(".latest", ".latest.extension_version")) 568 569 return target_paths, target_dict 570 571 def build_sdk_scope_targets(self, build_release, modules): 572 # Build the latest scope targets for each module sdk 573 # Compute the paths to all the latest scope targets for each module sdk. 574 target_paths = [] 575 target_dict = {} 576 for module in modules: 577 for sdk in module.sdks: 578 sdk_type = sdk_type_from_name(sdk) 579 if not sdk_type.providesApis: 580 continue 581 582 sdk_info_file = sdk_snapshot_info_file(self.mainline_sdks_dir, 583 sdk) 584 paths, dict_item = self.latest_api_file_targets(sdk_info_file) 585 target_paths.extend(paths) 586 target_dict[sdk_info_file] = dict_item 587 if target_paths: 588 self.build_target_paths(build_release, target_paths) 589 return target_dict 590 591 def appendDiffToFile(self, file_object, sdk_zip_file, current_api, 592 latest_api, snapshots_dir): 593 """Extract current api and find its diff with the latest api.""" 594 with zipfile.ZipFile(sdk_zip_file, "r") as zipObj: 595 extracted_current_api = zipObj.extract( 596 member=current_api, path=snapshots_dir) 597 # The diff tool has an exit code of 0, 1 or 2 depending on whether 598 # it find no differences, some differences or an error (like missing 599 # file). As 0 or 1 are both valid results this cannot use check=True 600 # so disable the pylint check. 601 # pylint: disable=subprocess-run-check 602 diff = subprocess.run([ 603 "diff", "-u0", latest_api, extracted_current_api, "--label", 604 latest_api, "--label", extracted_current_api 605 ], 606 capture_output=True).stdout.decode("utf-8") 607 file_object.write(diff) 608 609 def create_snapshot_gantry_metadata_and_api_diff(self, sdk, target_dict, 610 snapshots_dir, 611 module_extension_version): 612 """Creates gantry metadata and api diff files for each module sdk. 613 614 For each module sdk, the scope targets are obtained for each java sdk 615 library and the api diff files are generated by performing a diff 616 operation between the current api file vs the latest api file. 617 """ 618 sdk_info_file = sdk_snapshot_info_file(snapshots_dir, sdk) 619 sdk_zip_file = sdk_snapshot_zip_file(snapshots_dir, sdk) 620 sdk_api_diff_file = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 621 622 gantry_metadata_dict = {} 623 with open( 624 sdk_api_diff_file, "w", 625 encoding="utf8") as sdk_api_diff_file_object: 626 last_finalized_version_set = set() 627 for sdk_library in target_dict[sdk_info_file]: 628 for scope in target_dict[sdk_info_file][sdk_library]: 629 scope_json = target_dict[sdk_info_file][sdk_library][scope] 630 current_api = scope_json["current_api"] 631 latest_api = scope_json["latest_api"] 632 self.appendDiffToFile(sdk_api_diff_file_object, 633 sdk_zip_file, current_api, latest_api, 634 snapshots_dir) 635 636 removed_api = scope_json["removed_api"] 637 latest_removed_api = scope_json["latest_removed_api"] 638 self.appendDiffToFile(sdk_api_diff_file_object, 639 sdk_zip_file, removed_api, 640 latest_removed_api, snapshots_dir) 641 642 def read_extension_version(target): 643 extension_target = target.replace( 644 ".latest", ".latest.extension_version") 645 with open( 646 extension_target, "r", encoding="utf8") as file: 647 version = int(file.read()) 648 # version equal to -1 means "not an extension version". 649 if version != -1: 650 last_finalized_version_set.add(version) 651 652 read_extension_version(scope_json["latest_api"]) 653 read_extension_version(scope_json["latest_removed_api"]) 654 655 if len(last_finalized_version_set) == 0: 656 # Either there is no java sdk library or all java sdk libraries 657 # have not been finalized in sdk extensions yet and hence have 658 # last finalized version set as -1. 659 gantry_metadata_dict["last_finalized_version"] = -1 660 elif len(last_finalized_version_set) == 1: 661 # All java sdk library extension version match. 662 gantry_metadata_dict["last_finalized_version"] =\ 663 last_finalized_version_set.pop() 664 else: 665 # Fail the build 666 raise ValueError( 667 "Not all sdk libraries finalized with the same version.\n") 668 669 gantry_metadata_dict["api_diff_file"] = sdk_api_diff_file.rsplit( 670 "/", 1)[-1] 671 gantry_metadata_dict["api_diff_file_size"] = os.path.getsize( 672 sdk_api_diff_file) 673 gantry_metadata_dict[ 674 "module_extension_version"] = module_extension_version 675 sdk_metadata_json_file = sdk_snapshot_gantry_metadata_json_file( 676 snapshots_dir, sdk) 677 678 gantry_metadata_json_object = json.dumps(gantry_metadata_dict, indent=4) 679 with open(sdk_metadata_json_file, 680 "w") as gantry_metadata_json_file_object: 681 gantry_metadata_json_file_object.write(gantry_metadata_json_object) 682 683 if os.path.getsize(sdk_metadata_json_file) > 1048576: # 1 MB 684 raise ValueError("Metadata file size should not exceed 1 MB.\n") 685 686 def get_module_extension_version(self): 687 return int( 688 subprocess.run([ 689 "build/soong/soong_ui.bash", "--dumpvar-mode", 690 "PLATFORM_SDK_EXTENSION_VERSION" 691 ], 692 capture_output=True).stdout.decode("utf-8").strip()) 693 694 def build_snapshot_gantry_metadata_and_api_diff(self, modules, target_dict, 695 snapshots_dir): 696 """For each module sdk, create the metadata and api diff file.""" 697 module_extension_version = self.get_module_extension_version() 698 for module in modules: 699 for sdk in module.sdks: 700 sdk_type = sdk_type_from_name(sdk) 701 if not sdk_type.providesApis: 702 continue 703 self.create_snapshot_gantry_metadata_and_api_diff( 704 sdk, target_dict, snapshots_dir, module_extension_version) 705 706 707# The sdk version to build 708# 709# This is legacy from the time when this could generate versioned sdk snapshots. 710SDK_VERSION = "current" 711 712# The initially empty list of build releases. Every BuildRelease that is created 713# automatically appends itself to this list. 714ALL_BUILD_RELEASES = [] 715 716 717class PreferHandling(enum.Enum): 718 """Enumeration of the various ways of handling prefer properties""" 719 720 # No special prefer property handling is required. 721 NONE = enum.auto() 722 723 # Apply the SoongConfigBoilerplateInserter transformation. 724 SOONG_CONFIG = enum.auto() 725 726 # Use the use_source_config_var property added in T. 727 USE_SOURCE_CONFIG_VAR_PROPERTY = enum.auto() 728 729 # No prefer in Android.bp file 730 # Starting with V, prebuilts will be enabled using apex_contributions flags. 731 USE_NO_PREFER_PROPERTY = enum.auto() 732 733 734@dataclasses.dataclass(frozen=True) 735@functools.total_ordering 736class BuildRelease: 737 """Represents a build release""" 738 739 # The name of the build release, e.g. Q, R, S, T, etc. 740 name: str 741 742 # The function to call to create the snapshot in the dist, that covers 743 # building and copying the snapshot into the dist. 744 creator: Callable[ 745 ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] 746 747 # The sub-directory of dist/mainline-sdks into which the build release 748 # specific snapshots will be copied. 749 # 750 # Defaults to for-<name>-build. 751 sub_dir: str = None 752 753 # Additional environment variables to pass to Soong when building the 754 # snapshots for this build release. 755 # 756 # Defaults to { 757 # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>, 758 # } 759 soong_env: typing.Dict[str, str] = None 760 761 # The position of this instance within the BUILD_RELEASES list. 762 ordinal: int = dataclasses.field(default=-1, init=False) 763 764 # Whether this build release supports the Soong config boilerplate that is 765 # used to control the prefer setting of modules via a Soong config variable. 766 preferHandling: PreferHandling = \ 767 PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY 768 769 # Whether the generated snapshots should include flagged APIs. Defaults to 770 # false because flagged APIs are not suitable for use outside Android. 771 include_flagged_apis: bool = False 772 773 # Whether the build release should generate Gantry metadata and API diff. 774 generate_gantry_metadata_and_api_diff: bool = False 775 776 def __post_init__(self): 777 # The following use object.__setattr__ as this object is frozen and 778 # attempting to set the fields directly would cause an exception to be 779 # thrown. 780 object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) 781 # Add this to the end of the list of all build releases. 782 ALL_BUILD_RELEASES.append(self) 783 # If no sub_dir was specified then set the default. 784 if self.sub_dir is None: 785 object.__setattr__(self, "sub_dir", f"for-{self.name}-build") 786 # If no soong_env was specified then set the default. 787 if self.soong_env is None: 788 object.__setattr__( 789 self, 790 "soong_env", 791 { 792 # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a 793 # snapshot suitable for a specific target build release. 794 "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, 795 }) 796 797 def __eq__(self, other): 798 return self.ordinal == other.ordinal 799 800 def __le__(self, other): 801 return self.ordinal <= other.ordinal 802 803 804def create_no_dist_snapshot(_: BuildRelease, __: "SdkDistProducer", 805 modules: List["MainlineModule"]): 806 """A place holder dist snapshot creation function that does nothing.""" 807 print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") 808 809 810def create_dist_snapshot_for_r(build_release: BuildRelease, 811 producer: "SdkDistProducer", 812 modules: List["MainlineModule"]): 813 """Generate a snapshot suitable for use in an R build.""" 814 producer.product_dist_for_build_r(build_release, modules) 815 816 817def create_sdk_snapshots_in_soong(build_release: BuildRelease, 818 producer: "SdkDistProducer", 819 modules: List["MainlineModule"]): 820 """Builds sdks and populates the dist for unbundled modules.""" 821 producer.produce_unbundled_dist_for_build_release(build_release, modules) 822 823 824def create_latest_sdk_snapshots(build_release: BuildRelease, 825 producer: "SdkDistProducer", 826 modules: List["MainlineModule"]): 827 """Builds and populates the latest release, including bundled modules.""" 828 producer.produce_unbundled_dist_for_build_release(build_release, modules) 829 producer.produce_bundled_dist_for_build_release(build_release, modules) 830 831 832Q = BuildRelease( 833 name="Q", 834 # At the moment we do not generate a snapshot for Q. 835 creator=create_no_dist_snapshot, 836 # This does not support or need any special prefer property handling. 837 preferHandling=PreferHandling.NONE, 838) 839R = BuildRelease( 840 name="R", 841 # Generate a simple snapshot for R. 842 creator=create_dist_snapshot_for_r, 843 # By default a BuildRelease creates an environment to pass to Soong that 844 # creates a release specific snapshot. However, Soong does not yet (and is 845 # unlikely to) support building an sdk snapshot for R so create an empty 846 # environment to pass to Soong instead. 847 soong_env={}, 848 # This does not support or need any special prefer property handling. 849 preferHandling=PreferHandling.NONE, 850) 851S = BuildRelease( 852 name="S", 853 # Generate a snapshot for this build release using Soong. 854 creator=create_sdk_snapshots_in_soong, 855 # This requires the SoongConfigBoilerplateInserter transformation to be 856 # applied. 857 preferHandling=PreferHandling.SOONG_CONFIG, 858) 859Tiramisu = BuildRelease( 860 name="Tiramisu", 861 # Generate a snapshot for this build release using Soong. 862 creator=create_sdk_snapshots_in_soong, 863 # This build release supports the use_source_config_var property. 864 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 865) 866UpsideDownCake = BuildRelease( 867 name="UpsideDownCake", 868 # Generate a snapshot for this build release using Soong. 869 creator=create_sdk_snapshots_in_soong, 870 # This build release supports the use_source_config_var property. 871 preferHandling=PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY, 872) 873VanillaIceCream = BuildRelease( 874 name="VanillaIceCream", 875 # Generate a snapshot for this build release using Soong. 876 creator=create_sdk_snapshots_in_soong, 877 # There are no build release specific environment variables to pass to 878 # Soong. 879 soong_env={}, 880 # Starting with V, setting `prefer|use_source_config_var` on soong modules 881 # in prebuilts/module_sdk is not necessary. 882 # prebuilts will be enabled using apex_contributions release build flags. 883 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 884) 885Baklava = BuildRelease( 886 name="Baklava", 887 # Generate a snapshot for this build release using Soong. 888 creator=create_sdk_snapshots_in_soong, 889 # There are no build release specific environment variables to pass to 890 # Soong. 891 soong_env={}, 892 # Starting with V, setting `prefer|use_source_config_var` on soong modules 893 # in prebuilts/module_sdk is not necessary. 894 # prebuilts will be enabled using apex_contributions release build flags. 895 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 896) 897 898# Insert additional BuildRelease definitions for following releases here, 899# before LATEST. 900 901# A build release for the latest build excluding flagged apis. 902NEXT = BuildRelease( 903 name="next", 904 creator=create_latest_sdk_snapshots, 905 # There are no build release specific environment variables to pass to 906 # Soong. 907 soong_env={}, 908 generate_gantry_metadata_and_api_diff=True, 909 # Starting with V, setting `prefer|use_source_config_var` on soong modules 910 # in prebuilts/module_sdk is not necessary. 911 # prebuilts will be enabled using apex_contributions release build flags. 912 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 913) 914 915# The build release for the latest build supported by this build, i.e. the 916# current build. This must be the last BuildRelease defined in this script. 917LATEST = BuildRelease( 918 name="latest", 919 creator=create_latest_sdk_snapshots, 920 # There are no build release specific environment variables to pass to 921 # Soong. 922 soong_env={}, 923 # Latest must include flagged APIs because it may be dropped into the main 924 # Android branches. 925 include_flagged_apis=True, 926 generate_gantry_metadata_and_api_diff=True, 927 # Starting with V, setting `prefer|use_source_config_var` on soong modules 928 # in prebuilts/module_sdk is not necessary. 929 # prebuilts will be enabled using apex_contributions release build flags. 930 preferHandling=PreferHandling.USE_NO_PREFER_PROPERTY, 931) 932 933 934@dataclasses.dataclass(frozen=True) 935class SdkLibrary: 936 """Information about a java_sdk_library.""" 937 938 # The name of java_sdk_library module. 939 name: str 940 941 # True if the sdk_library module is a shared library. 942 shared_library: bool = False 943 944 945@dataclasses.dataclass(frozen=True) 946class ForRBuild: 947 """Data structure needed for generating a snapshot for an R build.""" 948 949 # The java_sdk_library modules to export to the r snapshot. 950 sdk_libraries: typing.List[SdkLibrary] = dataclasses.field( 951 default_factory=list) 952 953 954@dataclasses.dataclass(frozen=True) 955class MainlineModule: 956 """Represents an unbundled mainline module. 957 958 This is a module that is distributed as a prebuilt and intended to be 959 updated with Mainline trains. 960 """ 961 # The name of the apex. 962 apex: str 963 964 # The names of the sdk and module_exports. 965 sdks: list[str] 966 967 # The first build release in which the SDK snapshot for this module is 968 # needed. 969 # 970 # Note: This is not necessarily the same build release in which the SDK 971 # source was first included. So, a module that was added in build T 972 # could potentially be used in an S release and so its SDK will need 973 # to be made available for S builds. 974 first_release: BuildRelease 975 976 # The configuration variable, defaults to ANDROID:module_build_from_source 977 configVar: ConfigVar = ConfigVar( 978 namespace="ANDROID", 979 name="module_build_from_source", 980 ) 981 982 for_r_build: typing.Optional[ForRBuild] = None 983 984 # The last release on which this module was optional. 985 # 986 # Some modules are optional when they are first released, usually because 987 # some vendors of Android devices have their own customizations of the 988 # module that they would like to preserve and which cannot yet be achieved 989 # through the existing APIs. Once those issues have been resolved then they 990 # will become mandatory. 991 # 992 # This field records the last build release in which they are optional. It 993 # defaults to None which indicates that the module was never optional. 994 # 995 # TODO(b/238203992): remove the following warning once all modules can be 996 # treated as optional at build time. 997 # 998 # DO NOT use this attr for anything other than controlling whether the 999 # generated snapshot uses its own Soong config variable or the common one. 1000 # That is because this is being temporarily used to force Permission to have 1001 # its own Soong config variable even though Permission is not actually 1002 # optional at runtime on a GMS capable device. 1003 # 1004 # b/238203992 will make all modules have their own Soong config variable by 1005 # default at which point this will no longer be needed on Permission and so 1006 # it can be used to indicate that a module is optional at runtime. 1007 last_optional_release: typing.Optional[BuildRelease] = None 1008 1009 # The short name for the module. 1010 # 1011 # Defaults to the last part of the apex name. 1012 short_name: str = "" 1013 1014 # Additional transformations 1015 additional_transformations: list[FileTransformation] = None 1016 1017 # The module key of SdkModule Enum defined in 1018 # packages/modules/common/proto/sdk.proto. 1019 module_proto_key: str = "" 1020 1021 def __post_init__(self): 1022 # If short_name is not set then set it to the last component of the apex 1023 # name. 1024 if not self.short_name: 1025 short_name = self.apex.rsplit(".", 1)[-1] 1026 object.__setattr__(self, "short_name", short_name) 1027 1028 def is_bundled(self): 1029 """Returns true for bundled modules. See BundledMainlineModule.""" 1030 return False 1031 1032 def transformations(self, build_release, sdk_type): 1033 """Returns the transformations to apply to this module's snapshot(s).""" 1034 transformations = [] 1035 1036 config_var = self.configVar 1037 1038 # If the module is optional then it needs its own Soong config 1039 # variable to allow it to be managed separately from other modules. 1040 if self.last_optional_release: 1041 config_var = ConfigVar( 1042 namespace=f"{self.short_name}_module", 1043 name="source_build", 1044 ) 1045 1046 prefer_handling = build_release.preferHandling 1047 if prefer_handling == PreferHandling.SOONG_CONFIG: 1048 sdk_type_prefix = sdk_type.configModuleTypePrefix 1049 config_module_type_prefix = \ 1050 f"{self.short_name}{sdk_type_prefix}_prebuilt_" 1051 inserter = SoongConfigBoilerplateInserter( 1052 "Android.bp", 1053 configVar=config_var, 1054 configModuleTypePrefix=config_module_type_prefix) 1055 transformations.append(inserter) 1056 elif prefer_handling == PreferHandling.USE_SOURCE_CONFIG_VAR_PROPERTY: 1057 transformation = UseSourceConfigVarTransformation( 1058 "Android.bp", configVar=config_var) 1059 transformations.append(transformation) 1060 elif prefer_handling == PreferHandling.USE_NO_PREFER_PROPERTY: 1061 transformation = UseNoPreferPropertyTransformation( 1062 "Android.bp", configVar=config_var 1063 ) 1064 transformations.append(transformation) 1065 1066 if self.additional_transformations and build_release > R: 1067 transformations.extend(self.additional_transformations) 1068 1069 return transformations 1070 1071 def is_required_for(self, target_build_release): 1072 """True if this module is required for the target build release.""" 1073 return self.first_release <= target_build_release 1074 1075 1076@dataclasses.dataclass(frozen=True) 1077class BundledMainlineModule(MainlineModule): 1078 """Represents a bundled Mainline module or a platform SDK for module use. 1079 1080 A bundled module is always preloaded into the platform images. 1081 """ 1082 1083 # Defaults to the latest build, i.e. the build on which this script is run 1084 # as bundled modules are, by definition, only needed in this build. 1085 first_release: BuildRelease = LATEST 1086 1087 def is_bundled(self): 1088 return True 1089 1090 def transformations(self, build_release, sdk_type): 1091 # Bundled modules are only used on thin branches where the corresponding 1092 # sources are absent, so skip transformations and keep the default 1093 # `prefer: false`. 1094 return [] 1095 1096 1097# List of mainline modules. 1098MAINLINE_MODULES = [ 1099 MainlineModule( 1100 apex="com.android.adservices", 1101 sdks=["adservices-module-sdk"], 1102 first_release=Tiramisu, 1103 last_optional_release=LATEST, 1104 module_proto_key="AD_SERVICES", 1105 ), 1106 MainlineModule( 1107 apex="com.android.appsearch", 1108 sdks=["appsearch-sdk"], 1109 first_release=Tiramisu, 1110 last_optional_release=LATEST, 1111 module_proto_key="APPSEARCH", 1112 ), 1113 MainlineModule( 1114 apex="com.android.art", 1115 sdks=[ 1116 "art-module-sdk", 1117 "art-module-test-exports", 1118 "art-module-host-exports", 1119 ], 1120 first_release=S, 1121 # Override the config... fields. 1122 configVar=ConfigVar( 1123 namespace="art_module", 1124 name="source_build", 1125 ), 1126 module_proto_key="ART", 1127 ), 1128 MainlineModule( 1129 apex="com.android.btservices", 1130 sdks=["btservices-module-sdk"], 1131 first_release=UpsideDownCake, 1132 # Bluetooth has always been and is still optional. 1133 last_optional_release=LATEST, 1134 module_proto_key="", 1135 ), 1136 MainlineModule( 1137 apex="com.android.configinfrastructure", 1138 sdks=["configinfrastructure-sdk"], 1139 first_release=UpsideDownCake, 1140 last_optional_release=LATEST, 1141 module_proto_key="CONFIG_INFRASTRUCTURE", 1142 ), 1143 MainlineModule( 1144 apex="com.android.conscrypt", 1145 sdks=[ 1146 "conscrypt-module-sdk", 1147 "conscrypt-module-test-exports", 1148 "conscrypt-module-host-exports", 1149 ], 1150 first_release=Q, 1151 # No conscrypt java_sdk_library modules are exported to the R snapshot. 1152 # Conscrypt was updatable in R but the generate_ml_bundle.sh does not 1153 # appear to generate a snapshot for it. 1154 for_r_build=None, 1155 last_optional_release=LATEST, 1156 module_proto_key="CONSCRYPT", 1157 ), 1158 MainlineModule( 1159 apex="com.android.devicelock", 1160 sdks=["devicelock-module-sdk"], 1161 first_release=UpsideDownCake, 1162 # Treat DeviceLock as optional at build time 1163 # TODO(b/238203992): remove once all modules are optional at build time. 1164 last_optional_release=LATEST, 1165 module_proto_key="", 1166 ), 1167 MainlineModule( 1168 apex="com.android.healthfitness", 1169 sdks=["healthfitness-module-sdk"], 1170 first_release=UpsideDownCake, 1171 last_optional_release=LATEST, 1172 module_proto_key="HEALTH_FITNESS", 1173 ), 1174 MainlineModule( 1175 apex="com.android.ipsec", 1176 sdks=["ipsec-module-sdk"], 1177 first_release=R, 1178 for_r_build=ForRBuild(sdk_libraries=[ 1179 SdkLibrary( 1180 name="android.net.ipsec.ike", 1181 shared_library=True, 1182 ), 1183 ]), 1184 last_optional_release=LATEST, 1185 module_proto_key="IPSEC", 1186 ), 1187 MainlineModule( 1188 apex="com.android.media", 1189 sdks=["media-module-sdk"], 1190 first_release=R, 1191 for_r_build=ForRBuild(sdk_libraries=[ 1192 SdkLibrary(name="framework-media"), 1193 ]), 1194 last_optional_release=LATEST, 1195 module_proto_key="MEDIA", 1196 ), 1197 MainlineModule( 1198 apex="com.android.mediaprovider", 1199 sdks=["mediaprovider-module-sdk"], 1200 first_release=R, 1201 for_r_build=ForRBuild(sdk_libraries=[ 1202 SdkLibrary(name="framework-mediaprovider"), 1203 ]), 1204 # MP is a mandatory mainline module but in some cases (b/294190883) this 1205 # needs to be optional for Android Go on T. GTS tests might be needed to 1206 # to check the specific condition mentioned in the bug. 1207 last_optional_release=LATEST, 1208 module_proto_key="MEDIA_PROVIDER", 1209 ), 1210 MainlineModule( 1211 apex="com.android.nfcservices", 1212 sdks=["nfcservices-module-sdk"], 1213 first_release=Baklava, 1214 # NFC is optional. 1215 last_optional_release=LATEST, 1216 module_proto_key="", 1217 ), 1218 MainlineModule( 1219 apex="com.android.ondevicepersonalization", 1220 sdks=["ondevicepersonalization-module-sdk"], 1221 first_release=Tiramisu, 1222 last_optional_release=LATEST, 1223 module_proto_key="ON_DEVICE_PERSONALIZATION", 1224 ), 1225 MainlineModule( 1226 apex="com.android.permission", 1227 sdks=["permission-module-sdk"], 1228 first_release=R, 1229 for_r_build=ForRBuild(sdk_libraries=[ 1230 SdkLibrary(name="framework-permission"), 1231 # framework-permission-s is not needed on R as it contains classes 1232 # that are provided in R by non-updatable parts of the 1233 # bootclasspath. 1234 ]), 1235 # Although Permission is not, and has never been, optional for GMS 1236 # capable devices it does need to be treated as optional at build time 1237 # when building non-GMS devices. 1238 # TODO(b/238203992): remove once all modules are optional at build time. 1239 last_optional_release=LATEST, 1240 module_proto_key="PERMISSIONS", 1241 ), 1242 MainlineModule( 1243 apex="com.android.rkpd", 1244 sdks=["rkpd-sdk"], 1245 first_release=UpsideDownCake, 1246 # Rkpd has always been and is still optional. 1247 last_optional_release=LATEST, 1248 module_proto_key="", 1249 ), 1250 MainlineModule( 1251 apex="com.android.scheduling", 1252 sdks=["scheduling-sdk"], 1253 first_release=S, 1254 last_optional_release=LATEST, 1255 module_proto_key="SCHEDULING", 1256 ), 1257 MainlineModule( 1258 apex="com.android.sdkext", 1259 sdks=["sdkextensions-sdk"], 1260 first_release=R, 1261 for_r_build=ForRBuild(sdk_libraries=[ 1262 SdkLibrary(name="framework-sdkextensions"), 1263 ]), 1264 last_optional_release=LATEST, 1265 module_proto_key="SDK_EXTENSIONS", 1266 ), 1267 MainlineModule( 1268 apex="com.android.os.statsd", 1269 sdks=["statsd-module-sdk"], 1270 first_release=R, 1271 for_r_build=ForRBuild(sdk_libraries=[ 1272 SdkLibrary(name="framework-statsd"), 1273 ]), 1274 last_optional_release=LATEST, 1275 module_proto_key="STATSD", 1276 ), 1277 MainlineModule( 1278 apex="com.android.tethering", 1279 sdks=["tethering-module-sdk"], 1280 first_release=R, 1281 for_r_build=ForRBuild(sdk_libraries=[ 1282 SdkLibrary(name="framework-tethering"), 1283 ]), 1284 last_optional_release=LATEST, 1285 module_proto_key="TETHERING", 1286 ), 1287 MainlineModule( 1288 apex="com.android.uwb", 1289 sdks=["uwb-module-sdk"], 1290 first_release=Tiramisu, 1291 # Uwb has always been and is still optional. 1292 last_optional_release=LATEST, 1293 module_proto_key="", 1294 ), 1295 MainlineModule( 1296 apex="com.android.wifi", 1297 sdks=["wifi-module-sdk"], 1298 first_release=R, 1299 for_r_build=ForRBuild(sdk_libraries=[ 1300 SdkLibrary(name="framework-wifi"), 1301 ]), 1302 # Wifi has always been and is still optional. 1303 last_optional_release=LATEST, 1304 module_proto_key="", 1305 ), 1306] 1307 1308# List of Mainline modules that currently are never built unbundled. They must 1309# not specify first_release, and they don't have com.google.android 1310# counterparts. 1311BUNDLED_MAINLINE_MODULES = [ 1312 BundledMainlineModule( 1313 apex="com.android.i18n", 1314 sdks=[ 1315 "i18n-module-sdk", 1316 "i18n-module-test-exports", 1317 "i18n-module-host-exports", 1318 ], 1319 ), 1320 BundledMainlineModule( 1321 apex="com.android.runtime", 1322 sdks=[ 1323 "runtime-module-host-exports", 1324 "runtime-module-sdk", 1325 ], 1326 ), 1327 BundledMainlineModule( 1328 apex="com.android.tzdata", 1329 sdks=["tzdata-module-test-exports"], 1330 ), 1331] 1332 1333# List of platform SDKs for Mainline module use. 1334PLATFORM_SDKS_FOR_MAINLINE = [ 1335 BundledMainlineModule( 1336 apex="platform-mainline", 1337 sdks=[ 1338 "platform-mainline-sdk", 1339 "platform-mainline-test-exports", 1340 ], 1341 ), 1342] 1343 1344 1345@dataclasses.dataclass 1346class SdkDistProducer: 1347 """Produces the DIST_DIR/mainline-sdks and DIST_DIR/stubs directories. 1348 1349 Builds SDK snapshots for mainline modules and then copies them into the 1350 DIST_DIR/mainline-sdks directory. Also extracts the sdk_library txt, jar and 1351 srcjar files from each SDK snapshot and copies them into the DIST_DIR/stubs 1352 directory. 1353 """ 1354 1355 # Used to run subprocesses for this. 1356 subprocess_runner: SubprocessRunner 1357 1358 # Builds sdk snapshots 1359 snapshot_builder: SnapshotBuilder 1360 1361 # The DIST_DIR environment variable. 1362 dist_dir: str = "uninitialized-dist" 1363 1364 # The path to this script. It may be inserted into files that are 1365 # transformed to document where the changes came from. 1366 script: str = sys.argv[0] 1367 1368 # The path to the mainline-sdks dist directory for unbundled modules. 1369 # 1370 # Initialized in __post_init__(). 1371 mainline_sdks_dir: str = dataclasses.field(init=False) 1372 1373 # The path to the mainline-sdks dist directory for bundled modules and 1374 # platform SDKs. 1375 # 1376 # Initialized in __post_init__(). 1377 bundled_mainline_sdks_dir: str = dataclasses.field(init=False) 1378 1379 def __post_init__(self): 1380 self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") 1381 self.bundled_mainline_sdks_dir = os.path.join(self.dist_dir, 1382 "bundled-mainline-sdks") 1383 1384 def prepare(self): 1385 pass 1386 1387 def produce_dist(self, modules, build_releases): 1388 # Prepare the dist directory for the sdks. 1389 self.prepare() 1390 1391 # Group build releases so that those with the same Soong environment are 1392 # run consecutively to avoid having to regenerate ninja files. 1393 grouped_by_env = defaultdict(list) 1394 for build_release in build_releases: 1395 grouped_by_env[str(build_release.soong_env)].append(build_release) 1396 ordered = [br for _, group in grouped_by_env.items() for br in group] 1397 1398 for build_release in ordered: 1399 # Only build modules that are required for this build release. 1400 filtered_modules = [ 1401 m for m in modules if m.is_required_for(build_release) 1402 ] 1403 if filtered_modules: 1404 print(f"Building SDK snapshots for {build_release.name}" 1405 f" build release") 1406 build_release.creator(build_release, self, filtered_modules) 1407 1408 def product_dist_for_build_r(self, build_release, modules): 1409 # Although we only need a subset of the files that a java_sdk_library 1410 # adds to an sdk snapshot generating the whole snapshot is the simplest 1411 # way to ensure that all the necessary files are produced. 1412 1413 # Filter out any modules that do not provide sdk for R. 1414 modules = [m for m in modules if m.for_r_build] 1415 1416 snapshot_dir = self.snapshot_builder.build_snapshots_for_build_r( 1417 build_release, modules) 1418 self.populate_unbundled_dist(build_release, modules, snapshot_dir) 1419 1420 def produce_unbundled_dist_for_build_release(self, build_release, modules): 1421 modules = [m for m in modules if not m.is_bundled()] 1422 snapshots_dir = self.snapshot_builder.build_snapshots( 1423 build_release, modules) 1424 if build_release.generate_gantry_metadata_and_api_diff: 1425 target_dict = self.snapshot_builder.build_sdk_scope_targets( 1426 build_release, modules) 1427 self.snapshot_builder.build_snapshot_gantry_metadata_and_api_diff( 1428 modules, target_dict, snapshots_dir) 1429 self.populate_unbundled_dist(build_release, modules, snapshots_dir) 1430 return snapshots_dir 1431 1432 def produce_bundled_dist_for_build_release(self, build_release, modules): 1433 modules = [m for m in modules if m.is_bundled()] 1434 if modules: 1435 snapshots_dir = self.snapshot_builder.build_snapshots( 1436 build_release, modules) 1437 self.populate_bundled_dist(build_release, modules, snapshots_dir) 1438 1439 def dist_sdk_snapshot_gantry_metadata_and_api_diff(self, sdk_dist_dir, sdk, 1440 module, snapshots_dir): 1441 """Copy the sdk snapshot api diff file to a dist directory.""" 1442 sdk_type = sdk_type_from_name(sdk) 1443 if not sdk_type.providesApis: 1444 return 1445 1446 sdk_dist_module_subdir = os.path.join(sdk_dist_dir, module.apex) 1447 sdk_dist_subdir = os.path.join(sdk_dist_module_subdir, "sdk") 1448 os.makedirs(sdk_dist_subdir, exist_ok=True) 1449 sdk_api_diff_path = sdk_snapshot_api_diff_file(snapshots_dir, sdk) 1450 shutil.copy(sdk_api_diff_path, sdk_dist_subdir) 1451 1452 sdk_gantry_metadata_json_path = sdk_snapshot_gantry_metadata_json_file( 1453 snapshots_dir, sdk) 1454 sdk_dist_gantry_metadata_json_path = os.path.join( 1455 sdk_dist_module_subdir, "gantry-metadata.json") 1456 shutil.copy(sdk_gantry_metadata_json_path, 1457 sdk_dist_gantry_metadata_json_path) 1458 1459 def dist_generate_sdk_supported_modules_file(self, modules): 1460 sdk_modules_file = os.path.join(self.dist_dir, "sdk-modules.txt") 1461 os.makedirs(os.path.dirname(sdk_modules_file), exist_ok=True) 1462 with open(sdk_modules_file, "w", encoding="utf8") as file: 1463 for module in modules: 1464 if module in MAINLINE_MODULES: 1465 file.write(aosp_to_google_name(module.apex) + "\n") 1466 1467 def generate_mainline_modules_info_file(self, modules, root_dir): 1468 mainline_modules_info_file = os.path.join( 1469 self.dist_dir, "mainline-modules-info.json" 1470 ) 1471 os.makedirs(os.path.dirname(mainline_modules_info_file), exist_ok=True) 1472 mainline_modules_info_dict = {} 1473 for module in modules: 1474 if module not in MAINLINE_MODULES: 1475 continue 1476 module_name = aosp_to_google_name(module.apex) 1477 mainline_modules_info_dict[module_name] = dict() 1478 mainline_modules_info_dict[module_name]["module_sdk_project"] = ( 1479 module_sdk_project_for_module(module_name, root_dir) 1480 ) 1481 mainline_modules_info_dict[module_name][ 1482 "module_proto_key" 1483 ] = module.module_proto_key 1484 # The first sdk in the list is the name to use. 1485 mainline_modules_info_dict[module_name]["sdk_name"] = module.sdks[0] 1486 1487 with open(mainline_modules_info_file, "w", encoding="utf8") as file: 1488 json.dump(mainline_modules_info_dict, file, indent=4) 1489 1490 def populate_unbundled_dist(self, build_release, modules, snapshots_dir): 1491 build_release_dist_dir = os.path.join(self.mainline_sdks_dir, 1492 build_release.sub_dir) 1493 for module in modules: 1494 for sdk in module.sdks: 1495 sdk_dist_dir = os.path.join(build_release_dist_dir, SDK_VERSION) 1496 if build_release.generate_gantry_metadata_and_api_diff: 1497 self.dist_sdk_snapshot_gantry_metadata_and_api_diff( 1498 sdk_dist_dir, sdk, module, snapshots_dir) 1499 self.populate_dist_snapshot(build_release, module, sdk, 1500 sdk_dist_dir, snapshots_dir) 1501 1502 def populate_bundled_dist(self, build_release, modules, snapshots_dir): 1503 sdk_dist_dir = self.bundled_mainline_sdks_dir 1504 for module in modules: 1505 for sdk in module.sdks: 1506 self.populate_dist_snapshot(build_release, module, sdk, 1507 sdk_dist_dir, snapshots_dir) 1508 1509 def populate_dist_snapshot(self, build_release, module, sdk, sdk_dist_dir, 1510 snapshots_dir): 1511 sdk_type = sdk_type_from_name(sdk) 1512 subdir = sdk_type.name 1513 1514 sdk_dist_subdir = os.path.join(sdk_dist_dir, module.apex, subdir) 1515 sdk_path = sdk_snapshot_zip_file(snapshots_dir, sdk) 1516 sdk_type = sdk_type_from_name(sdk) 1517 transformations = module.transformations(build_release, sdk_type) 1518 self.dist_sdk_snapshot_zip( 1519 build_release, sdk_path, sdk_dist_subdir, transformations) 1520 1521 def dist_sdk_snapshot_zip( 1522 self, build_release, src_sdk_zip, sdk_dist_dir, transformations): 1523 """Copy the sdk snapshot zip file to a dist directory. 1524 1525 If no transformations are provided then this simply copies the show sdk 1526 snapshot zip file to the dist dir. However, if transformations are 1527 provided then the files to be transformed are extracted from the 1528 snapshot zip file, they are transformed to files in a separate directory 1529 and then a new zip file is created in the dist directory with the 1530 original files replaced by the newly transformed files. build_release is 1531 provided for transformations if it is needed. 1532 """ 1533 os.makedirs(sdk_dist_dir, exist_ok=True) 1534 dest_sdk_zip = os.path.join(sdk_dist_dir, os.path.basename(src_sdk_zip)) 1535 print(f"Copying sdk snapshot {src_sdk_zip} to {dest_sdk_zip}") 1536 1537 # If no transformations are provided then just copy the zip file 1538 # directly. 1539 if len(transformations) == 0: 1540 shutil.copy(src_sdk_zip, sdk_dist_dir) 1541 return 1542 1543 with tempfile.TemporaryDirectory() as tmp_dir: 1544 # Create a single pattern that will match any of the paths provided 1545 # in the transformations. 1546 pattern = "|".join( 1547 [f"({re.escape(t.path)})" for t in transformations]) 1548 1549 # Extract the matching files from the zip into the temporary 1550 # directory. 1551 extract_matching_files_from_zip(src_sdk_zip, tmp_dir, pattern) 1552 1553 # Apply the transformations to the extracted files in situ. 1554 apply_transformations(self, tmp_dir, transformations, build_release) 1555 1556 # Replace the original entries in the zip with the transformed 1557 # files. 1558 paths = [transformation.path for transformation in transformations] 1559 copy_zip_and_replace(self, src_sdk_zip, dest_sdk_zip, tmp_dir, 1560 paths) 1561 1562 1563def print_command(env, cmd): 1564 print(" ".join([f"{name}={value}" for name, value in env.items()] + cmd)) 1565 1566 1567def sdk_library_files_pattern(*, scope_pattern=r"[^/]+", name_pattern=r"[^/]+"): 1568 """Return a pattern to match sdk_library related files in an sdk snapshot""" 1569 return rf"sdk_library/{scope_pattern}/{name_pattern}\.(txt|jar|srcjar)" 1570 1571 1572def extract_matching_files_from_zip(zip_path, dest_dir, pattern): 1573 """Extracts files from a zip file into a destination directory. 1574 1575 The extracted files are those that match the specified regular expression 1576 pattern. 1577 """ 1578 os.makedirs(dest_dir, exist_ok=True) 1579 with zipfile.ZipFile(zip_path) as zip_file: 1580 for filename in zip_file.namelist(): 1581 if re.match(pattern, filename): 1582 print(f" extracting {filename}") 1583 zip_file.extract(filename, dest_dir) 1584 1585 1586def copy_zip_and_replace(producer, src_zip_path, dest_zip_path, src_dir, paths): 1587 """Copies a zip replacing some of its contents in the process. 1588 1589 The files to replace are specified by the paths parameter and are relative 1590 to the src_dir. 1591 """ 1592 # Get the absolute paths of the source and dest zip files so that they are 1593 # not affected by a change of directory. 1594 abs_src_zip_path = os.path.abspath(src_zip_path) 1595 abs_dest_zip_path = os.path.abspath(dest_zip_path) 1596 1597 # Make sure that all the paths being added to the zip file have a fixed 1598 # timestamp so that the contents of the zip file do not depend on when this 1599 # script is run, only the inputs. 1600 set_default_timestamp(src_dir, paths) 1601 1602 producer.subprocess_runner.run( 1603 ["zip", "-q", abs_src_zip_path, "--out", abs_dest_zip_path] + paths, 1604 # Change into the source directory before running zip. 1605 cwd=src_dir) 1606 1607 1608def apply_transformations(producer, tmp_dir, transformations, build_release): 1609 for transformation in transformations: 1610 path = os.path.join(tmp_dir, transformation.path) 1611 1612 # Record the timestamp of the file. 1613 modified = os.path.getmtime(path) 1614 1615 # Transform the file. 1616 transformation.apply(producer, path, build_release) 1617 1618 # Reset the timestamp of the file to the original timestamp before the 1619 # transformation was applied. 1620 os.utime(path, (modified, modified)) 1621 1622 1623def create_producer(tool_path, skip_allowed_deps_check): 1624 # Variables initialized from environment variables that are set by the 1625 # calling mainline_modules_sdks.sh. 1626 out_dir = os.environ["OUT_DIR"] 1627 dist_dir = os.environ["DIST_DIR"] 1628 1629 top_dir = os.environ["ANDROID_BUILD_TOP"] 1630 tool_path = os.path.relpath(tool_path, top_dir) 1631 tool_path = tool_path.replace(".py", ".sh") 1632 1633 subprocess_runner = SubprocessRunner() 1634 snapshot_builder = SnapshotBuilder( 1635 tool_path=tool_path, 1636 subprocess_runner=subprocess_runner, 1637 out_dir=out_dir, 1638 skip_allowed_deps_check=skip_allowed_deps_check, 1639 ) 1640 return SdkDistProducer( 1641 subprocess_runner=subprocess_runner, 1642 snapshot_builder=snapshot_builder, 1643 dist_dir=dist_dir, 1644 ) 1645 1646 1647def aosp_to_google(module): 1648 """Transform an AOSP module into a Google module""" 1649 new_apex = aosp_to_google_name(module.apex) 1650 # Create a copy of the AOSP module with the internal specific APEX name. 1651 return dataclasses.replace(module, apex=new_apex) 1652 1653 1654def aosp_to_google_name(name): 1655 """Transform an AOSP module name into a Google module name""" 1656 return name.replace("com.android.", "com.google.android.") 1657 1658 1659def google_to_aosp_name(name): 1660 """Transform a Google module name into an AOSP module name""" 1661 return name.replace("com.google.android.", "com.android.") 1662 1663 1664@dataclasses.dataclass(frozen=True) 1665class SdkType: 1666 name: str 1667 1668 configModuleTypePrefix: str 1669 1670 providesApis: bool = False 1671 1672 1673Sdk = SdkType( 1674 name="sdk", 1675 configModuleTypePrefix="", 1676 providesApis=True, 1677) 1678HostExports = SdkType( 1679 name="host-exports", 1680 configModuleTypePrefix="_host_exports", 1681) 1682TestExports = SdkType( 1683 name="test-exports", 1684 configModuleTypePrefix="_test_exports", 1685) 1686 1687 1688def sdk_type_from_name(name): 1689 if name.endswith("-sdk"): 1690 return Sdk 1691 if name.endswith("-host-exports"): 1692 return HostExports 1693 if name.endswith("-test-exports"): 1694 return TestExports 1695 1696 raise Exception(f"{name} is not a valid sdk name, expected it to end" 1697 f" with -(sdk|host-exports|test-exports)") 1698 1699 1700def filter_modules(modules, target_build_apps): 1701 if target_build_apps: 1702 target_build_apps = target_build_apps.split() 1703 return [m for m in modules if m.apex in target_build_apps] 1704 return modules 1705 1706 1707def main(args): 1708 """Program entry point.""" 1709 if not os.path.exists("build/make/core/Makefile"): 1710 sys.exit("This script must be run from the top of the tree.") 1711 1712 args_parser = argparse.ArgumentParser( 1713 description="Build snapshot zips for consumption by Gantry.") 1714 args_parser.add_argument( 1715 "--tool-path", 1716 help="The path to this tool.", 1717 default="unspecified", 1718 ) 1719 args_parser.add_argument( 1720 "--build-release", 1721 action="append", 1722 choices=[br.name for br in ALL_BUILD_RELEASES], 1723 help="A target build for which snapshots are required. " 1724 "If it is \"latest\" then Mainline module SDKs from platform and " 1725 "bundled modules are included.", 1726 ) 1727 args_parser.add_argument( 1728 "--build-platform-sdks-for-mainline", 1729 action="store_true", 1730 help="Also build the platform SDKs for Mainline modules. " 1731 "Defaults to true when TARGET_BUILD_APPS is not set. " 1732 "Applicable only if the \"latest\" build release is built.", 1733 ) 1734 args_parser.add_argument( 1735 "--skip-allowed-deps-check", 1736 action="store_true", 1737 help="Skip apex-allowed-deps-check.", 1738 ) 1739 args = args_parser.parse_args(args) 1740 1741 build_releases = ALL_BUILD_RELEASES 1742 if args.build_release: 1743 selected_build_releases = {b.lower() for b in args.build_release} 1744 build_releases = [ 1745 b for b in build_releases 1746 if b.name.lower() in selected_build_releases 1747 ] 1748 1749 target_build_apps = os.environ.get("TARGET_BUILD_APPS") 1750 modules = filter_modules(MAINLINE_MODULES + BUNDLED_MAINLINE_MODULES, 1751 target_build_apps) 1752 1753 # Also build the platform Mainline SDKs either if no specific modules are 1754 # requested or if --build-platform-sdks-for-mainline is given. 1755 if not target_build_apps or args.build_platform_sdks_for_mainline: 1756 modules += PLATFORM_SDKS_FOR_MAINLINE 1757 1758 producer = create_producer(args.tool_path, args.skip_allowed_deps_check) 1759 producer.dist_generate_sdk_supported_modules_file(modules) 1760 producer.generate_mainline_modules_info_file( 1761 modules, os.environ["ANDROID_BUILD_TOP"] 1762 ) 1763 producer.produce_dist(modules, build_releases) 1764 1765 1766if __name__ == "__main__": 1767 main(sys.argv[1:]) 1768