1#!/usr/bin/python3 2 3"""Updates prebuilt libraries used by Android builds. 4 5For details on how to use this script, visit go/update-prebuilts. 6""" 7import os 8import sys 9import zipfile 10import re 11import argparse 12import subprocess 13import shlex 14import glob 15import shutil 16 17# Modules not in Android repo. Ok to ignore if they are not really used. 18try: 19 import six 20except ImportError: 21 six = None 22 23from urllib import request 24from shutil import which 25from distutils.version import LooseVersion 26from pathlib import Path 27from io import StringIO 28from typing import Iterable, Optional 29import xml.etree.ElementTree as ET 30from maven import MavenLibraryInfo, GMavenArtifact, maven_path_for_artifact 31from buildserver import fetch_and_extract, extract_artifact, \ 32 parse_build_id, fetch_artifact as buildserver_fetch_artifact, fetch_artifacts as buildserver_fetch_artifacts 33from utils import print_e, append, cp, mv, rm 34 35 36current_path = 'current' 37framework_sdk_target = 'sdk' 38androidx_dir = os.path.join(current_path, 'androidx') 39androidx_owners = os.path.join(androidx_dir, 'OWNERS') 40java_plugins_bp_path = os.path.join(androidx_dir, 'JavaPlugins.bp') 41test_mapping_file = os.path.join(androidx_dir, 'TEST_MAPPING') 42drop_config_toml = os.path.join(androidx_dir, 'drop_config.toml') 43compose_test_mapping_file = os.path.join(androidx_dir, 'm2repository/androidx/compose/TEST_MAPPING') 44gmaven_dir = os.path.join(current_path, 'gmaven') 45extras_dir = os.path.join(current_path, 'extras') 46buildtools_dir = 'tools' 47jetifier_dir = os.path.join(buildtools_dir, 'jetifier', 'jetifier-standalone') 48repo_root_dir = Path(sys.argv[0]).resolve().parents[3] 49extension_sdk_finalization_cmd = 'prebuilts/build-tools/path/linux-x86/python3 %s -r "{readme}" {local_mode} -b {bug} -f {extension_version} {build_id}' % ( 50 "packages/modules/common/tools/finalize_sdk.py" 51) 52temp_dir = os.path.join(os.getcwd(), 'support_tmp') 53os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[0])))) 54git_dir = os.getcwd() 55 56# Suffixes used by KMP artifacts. If an artifact in maven_to_make ends with one 57# of these, it will replace the anchor artifact. 58kmp_suffixes = ['android','jvm'] 59 60# Leave map blank to automatically populate name and path: 61# - Name format is MAVEN.replaceAll(':','_') 62# - Path format is MAVEN.replaceAll(':','/').replaceAll('.','/') 63maven_to_make = { 64 # AndroidX 65 'androidx.benchmark:benchmark-macro': {}, 66 'androidx.benchmark:benchmark-macro-junit4': {}, 67 'androidx.benchmark:benchmark-common': {}, 68 'androidx.benchmark:benchmark-junit4': {}, 69 'androidx.tracing:tracing': {}, 70 'androidx.tracing:tracing-perfetto': {}, 71 'androidx.tracing:tracing-perfetto-binary': {}, 72 'androidx.tracing:tracing-perfetto-handshake': {}, 73 'androidx.tracing:tracing-perfetto-common': {}, 74 'androidx.tracing:tracing-ktx': {}, 75 'androidx.slice:slice-builders': {}, 76 'androidx.slice:slice-core': {}, 77 'androidx.slice:slice-view': {}, 78 'androidx.remotecallback:remotecallback': {}, 79 'androidx.remotecallback:remotecallback-processor': { 80 'host': True 81 }, 82 'androidx.versionedparcelable:versionedparcelable': {}, 83 'androidx.vectordrawable:vectordrawable-animated': {}, 84 'androidx.activity:activity': {}, 85 'androidx.activity:activity-ktx': {}, 86 'androidx.annotation:annotation-jvm': { 87 'host_and_device': True 88 }, 89 'androidx.annotation:annotation-experimental': {}, 90 'androidx.asynclayoutinflater:asynclayoutinflater': {}, 91 'androidx.camera:camera-viewfinder':{}, 92 'androidx.camera:camera-camera2' :{}, 93 'androidx.camera:camera-core': {}, 94 'androidx.camera:camera-lifecycle': {}, 95 'androidx.camera:camera-extensions': {}, 96 'androidx.collection:collection-ktx': {}, 97 'androidx.collection:collection-jvm': {}, 98 'androidx.concurrent:concurrent-futures': {}, 99 'androidx.concurrent:concurrent-futures-ktx': {}, 100 'androidx.concurrent:concurrent-listenablefuture-callback': {}, 101 'androidx.concurrent:concurrent-listenablefuture': {}, 102 'androidx.core:core': {}, 103 'androidx.core:core-animation': {}, 104 'androidx.core:core-animation-testing': {}, 105 'androidx.core:core-ktx': {}, 106 'androidx.core.uwb:uwb': {}, 107 'androidx.core.uwb:uwb-rxjava3': {}, 108 'androidx.contentpaging:contentpaging': {}, 109 'androidx.coordinatorlayout:coordinatorlayout': {}, 110 'androidx.datastore:datastore-android': {}, 111 'androidx.datastore:datastore-core-okio-jvm': {}, 112 'androidx.datastore:datastore-core-android': {}, 113 'androidx.datastore:datastore-preferences-android': {}, 114 'androidx.datastore:datastore-preferences-core-jvm': {}, 115 'androidx.datastore:datastore-preferences-rxjava2': {}, 116 'androidx.datastore:datastore-rxjava2': {}, 117 'androidx.legacy:legacy-support-core-ui': {}, 118 'androidx.legacy:legacy-support-core-utils': {}, 119 'androidx.cursoradapter:cursoradapter': {}, 120 'androidx.browser:browser': {}, 121 'androidx.customview:customview': {}, 122 'androidx.customview:customview-poolingcontainer': {}, 123 'androidx.credentials:credentials': {}, 124 'androidx.documentfile:documentfile': {}, 125 'androidx.drawerlayout:drawerlayout': {}, 126 'androidx.dynamicanimation:dynamicanimation': {}, 127 'androidx.emoji:emoji': {}, 128 'androidx.emoji:emoji-appcompat': {}, 129 'androidx.emoji:emoji-bundled': {}, 130 'androidx.emoji2:emoji2': {}, 131 'androidx.emoji2:emoji2-views-helper': {}, 132 'androidx.exifinterface:exifinterface': {}, 133 'androidx.fragment:fragment': {}, 134 'androidx.fragment:fragment-ktx': {}, 135 'androidx.fragment:fragment-testing': {}, 136 'androidx.fragment:fragment-testing-manifest': {}, 137 'androidx.heifwriter:heifwriter': {}, 138 'androidx.health:health-services-client': {}, 139 'androidx.interpolator:interpolator': {}, 140 'androidx.loader:loader': {}, 141 'androidx.media:media': {}, 142 'androidx.media2:media2-player': {}, 143 'androidx.media2:media2-session': {}, 144 'androidx.media2:media2-common': {}, 145 'androidx.media2:media2-exoplayer': {}, 146 'androidx.media2:media2-widget': {}, 147 'androidx.navigation:navigation-common': {}, 148 'androidx.navigation:navigation-common-ktx': {}, 149 'androidx.navigation:navigation-fragment': {}, 150 'androidx.navigation:navigation-fragment-ktx': {}, 151 'androidx.navigation:navigation-runtime': {}, 152 'androidx.navigation:navigation-runtime-ktx': {}, 153 'androidx.navigation:navigation-ui': {}, 154 'androidx.navigation:navigation-ui-ktx': {}, 155 'androidx.percentlayout:percentlayout': {}, 156 'androidx.print:print': {}, 157 'androidx.privacysandbox.ads:ads-adservices': {}, 158 'androidx.privacysandbox.ads:ads-adservices-java': {}, 159 'androidx.privacysandbox.ui:ui-client': {}, 160 'androidx.privacysandbox.ui:ui-provider': {}, 161 'androidx.privacysandbox.ui:ui-core': {}, 162 'androidx.privacysandbox.sdkruntime:sdkruntime-client': {}, 163 'androidx.privacysandbox.sdkruntime:sdkruntime-core': {}, 164 'androidx.privacysandbox.tools:tools': { 165 'host': True 166 }, 167 'androidx.privacysandbox.tools:tools-apicompiler': { 168 'host': True 169 }, 170 'androidx.privacysandbox.tools:tools-apigenerator': { 171 'host': True 172 }, 173 'androidx.privacysandbox.tools:tools-apipackager': { 174 'host': True 175 }, 176 'androidx.privacysandbox.tools:tools-core': { 177 'host': True 178 }, 179 'androidx.privacysandbox.ui:ui-tests': {}, 180 'androidx.recommendation:recommendation': {}, 181 'androidx.recyclerview:recyclerview-selection': {}, 182 'androidx.savedstate:savedstate': {}, 183 'androidx.savedstate:savedstate-ktx': {}, 184 'androidx.slidingpanelayout:slidingpanelayout': {}, 185 'androidx.swiperefreshlayout:swiperefreshlayout': {}, 186 'androidx.textclassifier:textclassifier': {}, 187 'androidx.transition:transition': {}, 188 'androidx.transition:transition-ktx': {}, 189 'androidx.tvprovider:tvprovider': {}, 190 'androidx.legacy:legacy-support-v13': {}, 191 'androidx.legacy:legacy-preference-v14': {}, 192 'androidx.leanback:leanback': {}, 193 'androidx.leanback:leanback-grid': {}, 194 'androidx.leanback:leanback-preference': {}, 195 'androidx.legacy:legacy-support-v4': {}, 196 'androidx.appcompat:appcompat': {}, 197 'androidx.appcompat:appcompat-resources': {}, 198 'androidx.cardview:cardview': {}, 199 'androidx.gridlayout:gridlayout': {}, 200 'androidx.mediarouter:mediarouter': {}, 201 'androidx.palette:palette': {}, 202 'androidx.preference:preference': {}, 203 'androidx.recyclerview:recyclerview': {}, 204 'androidx.vectordrawable:vectordrawable': {}, 205 'androidx.viewpager:viewpager': {}, 206 'androidx.viewpager2:viewpager2': {}, 207 'androidx.wear:wear': {}, 208 'androidx.wear:wear-ongoing': {}, 209 'androidx.javascriptengine:javascriptengine': {}, 210 'androidx.webkit:webkit': {}, 211 'androidx.biometric:biometric': {}, 212 'androidx.autofill:autofill': {}, 213 'androidx.appsearch:appsearch': {}, 214 'androidx.appsearch:appsearch-builtin-types': {}, 215 'androidx.appsearch:appsearch-compiler': { 216 'name': 'androidx.appsearch_appsearch-compiler', 217 'host': True 218 }, 219 'androidx.appsearch:appsearch-local-storage': { 220 'name': 'androidx.appsearch_appsearch_local_storage' 221 }, 222 'androidx.appsearch:appsearch-platform-storage': {}, 223 'androidx.car.app:app': {}, 224 'androidx.car.app:app-automotive': {}, 225 'androidx.car.app:app-testing': {}, 226 'androidx.startup:startup-runtime': {}, 227 'androidx.window:window': { 228 'optional-uses-libs': { 229 'androidx.window.extensions', 230 'androidx.window.sidecar' 231 } 232 }, 233 'androidx.window.extensions:extensions': {}, 234 'androidx.window.extensions.core:core': {}, 235 'androidx.window:window-core': {}, 236 'androidx.window:window-java':{}, 237 'androidx.resourceinspection:resourceinspection-annotation': {}, 238 'androidx.profileinstaller:profileinstaller': {}, 239 'androidx.test.uiautomator:uiautomator': {}, 240 241 # AndroidX for Compose 242 'androidx.compose.compiler:compiler-hosted': { 243 'host': True 244 }, 245 'androidx.compose.animation:animation-android': {}, 246 'androidx.compose.animation:animation-core-android': {}, 247 'androidx.compose.animation:animation-graphics-android': {}, 248 'androidx.compose.foundation:foundation-android': {}, 249 'androidx.compose.foundation:foundation-layout-android': {}, 250 'androidx.compose.foundation:foundation-text-android': {}, 251 'androidx.compose.material:material-android': {}, 252 'androidx.compose.material:material-icons-core-android': {}, 253 'androidx.compose.material:material-icons-extended-android': {}, 254 'androidx.compose.material:material-ripple-android': {}, 255 'androidx.compose.material3:material3-android': {}, 256 'androidx.compose.material3:material3-window-size-class-android': {}, 257 'androidx.compose.runtime:runtime-android': {}, 258 'androidx.compose.runtime:runtime-livedata': {}, 259 'androidx.compose.runtime:runtime-saveable-android': {}, 260 'androidx.compose.runtime:runtime-tracing': {}, 261 'androidx.compose.ui:ui-util-android': {}, 262 'androidx.compose.ui:ui-android': {}, 263 'androidx.compose.ui:ui-geometry-android': {}, 264 'androidx.compose.ui:ui-graphics-android': {}, 265 'androidx.compose.ui:ui-test-manifest': {}, 266 'androidx.compose.ui:ui-test-android': {}, 267 'androidx.compose.ui:ui-test-junit4-android': {}, 268 'androidx.compose.ui:ui-text-android': {}, 269 'androidx.compose.ui:ui-tooling-android': {}, 270 'androidx.compose.ui:ui-tooling-data-android': {}, 271 'androidx.compose.ui:ui-tooling-preview-android': {}, 272 'androidx.compose.ui:ui-unit-android': {}, 273 'androidx.activity:activity-compose': {}, 274 'androidx.navigation:navigation-compose': { }, 275 'androidx.lifecycle:lifecycle-viewmodel-compose': { }, 276 277 # Compose for wear 278 'androidx.wear.compose:compose-material-core': {}, 279 'androidx.wear.compose:compose-foundation': {}, 280 'androidx.wear.compose:compose-material': {}, 281 'androidx.wear.compose:compose-navigation': {}, 282 283 # AndroidX for Multidex 284 'androidx.multidex:multidex': {}, 285 'androidx.multidex:multidex-instrumentation': {}, 286 287 # AndroidX for Constraint Layout 288 'androidx.constraintlayout:constraintlayout': { 289 'name': 'androidx-constraintlayout_constraintlayout' 290 }, 291 'androidx.constraintlayout:constraintlayout-solver': { 292 'name': 'androidx-constraintlayout_constraintlayout-solver' 293 }, 294 'androidx.constraintlayout:constraintlayout-core': {}, 295 'androidx.constraintlayout:constraintlayout-compose-android': {}, 296 # AndroidX for Architecture Components 297 'androidx.arch.core:core-common': {}, 298 'androidx.arch.core:core-runtime': {}, 299 'androidx.arch.core:core-testing': {}, 300 'androidx.lifecycle:lifecycle-common': {}, 301 'androidx.lifecycle:lifecycle-common-java8': {}, 302 'androidx.lifecycle:lifecycle-extensions': {}, 303 'androidx.lifecycle:lifecycle-livedata': {}, 304 'androidx.lifecycle:lifecycle-livedata-ktx': {}, 305 'androidx.lifecycle:lifecycle-livedata-core': {}, 306 'androidx.lifecycle:lifecycle-livedata-core-ktx': {}, 307 'androidx.lifecycle:lifecycle-process': {}, 308 'androidx.lifecycle:lifecycle-runtime': {}, 309 'androidx.lifecycle:lifecycle-runtime-ktx': {}, 310 'androidx.lifecycle:lifecycle-runtime-compose': {}, 311 'androidx.lifecycle:lifecycle-runtime-testing': {}, 312 'androidx.lifecycle:lifecycle-service': {}, 313 'androidx.lifecycle:lifecycle-viewmodel': {}, 314 'androidx.lifecycle:lifecycle-viewmodel-ktx': {}, 315 'androidx.lifecycle:lifecycle-viewmodel-savedstate': {}, 316 'androidx.paging:paging-common-jvm': {}, 317 'androidx.paging:paging-common-ktx': {}, 318 'androidx.paging:paging-guava': {}, 319 'androidx.paging:paging-runtime': {}, 320 'androidx.sqlite:sqlite': {}, 321 'androidx.sqlite:sqlite-framework': {}, 322 'androidx.room:room-common-jvm': { 323 'host_and_device': True 324 }, 325 'androidx.room:room-compiler': { 326 'host': True, 327 'extra-static-libs': { 328 'guava' 329 } 330 }, 331 'androidx.room:room-guava': {}, 332 'androidx.room:room-migration': { 333 'host_and_device': True 334 }, 335 'androidx.room:room-ktx': {}, 336 'androidx.room:room-paging': {}, 337 'androidx.room:room-paging-guava': {}, 338 'androidx.room:room-runtime': {}, 339 'androidx.room:room-testing': {}, 340 'androidx.room:room-compiler-processing': { 341 'host': True 342 }, 343 'androidx.work:work-runtime': {}, 344 'androidx.work:work-runtime-ktx': {}, 345 'androidx.work:work-testing': {}, 346 347 # Third-party dependencies 348 'com.google.android:flexbox': { 349 'name': 'flexbox', 350 'path': 'flexbox' 351 }, 352 353 # Androidx Material Design Components 354 'com.google.android.material:material': {}, 355} 356 357# Mapping of POM dependencies to Soong build targets 358deps_rewrite = { 359 'auto-common': 'auto_common', 360 'auto-value-annotations': 'auto_value_annotations', 361 'com.google.auto.value:auto-value': 'libauto_value_plugin', 362 'com.google.protobuf:protobuf-java': 'libprotobuf-java-full', 363 'com.google.protobuf:protobuf-javalite': 'libprotobuf-java-lite', 364 'org.ow2.asm:asm': 'ow2-asm', 365 'org.ow2.asm:asm-commons': 'ow2-asm-commons', 366 'monitor': 'androidx.test.monitor', 367 'rules': 'androidx.test.rules', 368 'runner': 'androidx.test.runner', 369 'androidx.test:core': 'androidx.test.core', 370 'com.squareup:javapoet': 'javapoet', 371 'com.squareup.okio:okio-jvm': 'okio-lib', 372 'com.google.guava:listenablefuture': 'guava-listenablefuture-prebuilt-jar', 373 'sqlite-jdbc': 'xerial-sqlite-jdbc', 374 'com.intellij:annotations': 'jetbrains-annotations', 375 'javax.annotation:javax.annotation-api': 'javax-annotation-api-prebuilt-host-jar', 376 'org.robolectric:robolectric': 'Robolectric_all-target', 377 'org.jetbrains.kotlin:kotlin-stdlib-common': 'kotlin-stdlib', 378 'org.jetbrains.kotlinx:kotlinx-coroutines-core': 'kotlinx_coroutines', 379 'org.jetbrains.kotlinx:kotlinx-coroutines-test-jvm': 'kotlinx_coroutines_test', 380 'org.jetbrains.kotlinx:kotlinx-coroutines-guava': 'kotlinx_coroutines_guava', 381 'org.jetbrains.kotlinx:kotlinx-coroutines-android': 'kotlinx_coroutines_android', 382 'org.jetbrains.kotlinx:kotlinx-coroutines-test':'kotlinx_coroutines_test', 383 'org.jetbrains.kotlinx:kotlinx-coroutines-rx2': 'kotlinx_coroutines_rx2', 384 'org.jetbrains.kotlinx:kotlinx-metadata-jvm': 'kotlinx_metadata_jvm', 385 'androidx.test.espresso:espresso-core':'androidx.test.espresso.core', 386 'androidx.test.espresso:espresso-idling-resource':'androidx.test.espresso.idling-resource', 387 'androidx.datastore:datastore-core-jvm': 'androidx.datastore_datastore-core', 388} 389 390# List of artifacts that will be updated from GMaven 391# Use pattern: `group:library:version:extension` 392# e.g.: 393# androidx.appcompat:appcompat:1.2.0:aar 394# Use `latest` to always fetch the latest version. 395# e.g.: 396# androidx.appcompat:appcompat:latest:aar 397# Also make sure you add `group:library`:{} to maven_to_make as well. 398gmaven_artifacts = {} 399 400# Always remove these files. 401denylist_files = [ 402 'annotations.zip', 403 'public.txt', 404 'R.txt', 405 'AndroidManifest.xml', 406 os.path.join('libs', 'noto-emoji-compat-java.jar') 407] 408 409# Explicitly allow-listed initializers 410enabled_initializers = set([ 411 'androidx.lifecycle.ProcessLifecycleInitializer', 412 'androidx.work.WorkManagerInitializer', 413 # TODO(282947321): update after http://aosp/2600447 lands 414 'androidx.compose.runtime.tracing.TracingInitializer', 415]) 416 417android_manifest_namepaces = { 418 'android': 'http://schemas.android.com/apk/res/android', 419 'tools': 'http://schemas.android.com/tools' 420} 421 422startup_initializer_pattern = re.compile(r'(\s+)android:value="androidx.startup".*') 423 424artifact_pattern = re.compile(r'^(.+?)-(\d+\.\d+\.\d+(?:-\w+\d+)?(?:-[\d.]+)*)\.(jar|aar)$') 425 426 427def name_for_artifact(group_artifact): 428 """Returns the build system target name for a given library's Maven coordinate. 429 430 Args: 431 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core 432 Returns: 433 The build system target name for the artifact, ex. androidx.core_core. 434 """ 435 for kmp_suffix in kmp_suffixes: 436 if group_artifact.endswith("-" + kmp_suffix): 437 loc = group_artifact.rfind("-" + kmp_suffix) 438 group_artifact = group_artifact[0:loc] 439 if group_artifact in maven_to_make: 440 raise ValueError(f'Do not specify KMP anchor artifact in ' 441 f'maven_to_make: {group_artifact}') 442 deps_rewrite[group_artifact] = group_artifact.replace(':', '_') 443 break 444 return group_artifact.replace(':', '_') 445 446 447def path_for_artifact(group_artifact): 448 """Returns the file system path for a given library's Maven coordinate. 449 450 Args: 451 group_artifact: an unversioned Maven artifact coordinate, ex. androidx.core:core 452 Returns: 453 The file system path for the artifact, ex. androidx/core/core. 454 """ 455 return group_artifact.replace('.', '/').replace(':', '/') 456 457 458def populate_maven_to_make(mapping): 459 """Modifies the input mapping by expanding Maven coordinate keys into build target names and 460 paths. 461 462 Args: 463 mapping: a map where the keys are Maven coordinates 464 """ 465 for key in mapping: 466 if 'name' not in mapping[key]: 467 mapping[key]['name'] = name_for_artifact(key) 468 if 'path' not in maven_to_make[key]: 469 mapping[key]['path'] = path_for_artifact(key) 470 471 472def detect_artifacts(maven_repo_dirs): 473 """Parses Maven libraries from the specified directories. 474 475 Args: 476 maven_repo_dirs: a list of maven repository roots 477 Returns: 478 A map of Maven coordinate keys to MavenLibraryInfo objects parsed from POM files. 479 """ 480 maven_lib_info = {} 481 482 # Find the latest revision for each artifact, remove others 483 for repo_dir in maven_repo_dirs: 484 for root, dirs, files in os.walk(repo_dir): 485 for file in files: 486 if file[-4:] == '.pom': 487 # Read the POM (hack hack hack). 488 group_id = '' 489 artifact_id = '' 490 version = '' 491 file = os.path.join(root, file) 492 with open(file) as pom_file: 493 for line in pom_file: 494 if line[:11] == ' <groupId>': 495 group_id = line[11:-11] 496 elif line[:14] == ' <artifactId>': 497 artifact_id = line[14:-14] 498 elif line[:11] == ' <version>': 499 version = line[11:-11] 500 if group_id == '' or artifact_id == '' or version == '': 501 print_e('Failed to find Maven artifact data in ' + file) 502 continue 503 504 # Locate the artifact. 505 artifact_file = file[:-4] 506 if os.path.exists(artifact_file + '.jar'): 507 artifact_file = artifact_file + '.jar' 508 elif os.path.exists(artifact_file + '.aar'): 509 artifact_file = artifact_file + '.aar' 510 else: 511 # This error only occurs for a handful of gradle.plugin artifacts that only 512 # ship POM files, so we probably don't need to log unless we're debugging. 513 # print_e('Failed to find artifact for ' + artifact_file) 514 continue 515 516 # Make relative to root. 517 artifact_file = artifact_file[len(root) + 1:] 518 519 # Find the mapping. 520 group_artifact = group_id + ':' + artifact_id 521 if group_artifact in maven_to_make: 522 key = group_artifact 523 elif artifact_id in maven_to_make: 524 key = artifact_id 525 else: 526 # No mapping entry, skip this library. 527 continue 528 529 # Store the latest version. 530 version = LooseVersion(version) 531 if key not in maven_lib_info \ 532 or version > maven_lib_info[key].version: 533 maven_lib_info[key] = MavenLibraryInfo(key, group_id, artifact_id, version, 534 root, repo_dir, artifact_file) 535 536 return maven_lib_info 537 538 539def find_invalid_spec(artifact_list): 540 """Verifies whether all the artifacts in the list correspond to an entry in maven_to_make. 541 542 Args: 543 artifact_list: list of group IDs or artifact coordinates 544 Returns: 545 The first invalid artifact specification in the list, or None if all specs are valid. 546 """ 547 if artifact_list is None: 548 return None 549 for prefix in artifact_list: 550 has_prefix = False 551 for artifact_id in maven_to_make: 552 if artifact_id.startswith(prefix): 553 has_prefix = True 554 break 555 if not has_prefix: 556 return prefix 557 return None 558 559 560def transform_maven_repos(maven_repo_dirs, transformed_dir, extract_res=True, 561 write_pom2bp_cmd=True, include_static_deps=True, include=None, 562 exclude=None, prepend=None): 563 """Transforms a standard Maven repository to be compatible with the Android build system. 564 565 When using the include argument by itself, all other libraries will be excluded. When using the 566 exclude argument by itself, all other libraries will be included. When using both arguments, the 567 inclusion list will be applied followed by the exclusion list. 568 569 Args: 570 maven_repo_dirs: path to local Maven repository 571 transformed_dir: relative path for output, ex. androidx 572 extract_res: whether to extract Android resources like AndroidManifest.xml from AARs 573 write_pom2bp_cmd: whether pom2bp should write its own invocation arguments to output 574 include_static_deps: whether to pass --static-deps to pom2bp 575 include: list of Maven groupIds or unversioned artifact coordinates to include for 576 updates, ex. androidx.core or androidx.core:core 577 exclude: list of Maven groupIds or unversioned artifact coordinates to exclude from 578 updates, ex. androidx.core or androidx.core:core 579 prepend: Path to a file containing text to be inserted at the beginning of the generated 580 build file 581 Returns: 582 True if successful, false otherwise. 583 """ 584 # If neither include nor exclude is set, fall back to legacy behavior of including everything. 585 include_all = exclude is None and include is None 586 587 if exclude is None: 588 exclude = [] 589 if include is None: 590 include = [] 591 592 cwd = os.getcwd() 593 local_repo = os.path.join(cwd, transformed_dir) 594 working_dir = temp_dir 595 596 # Handle inclusions by stashing the remote artifacts for the inclusions, replacing the entire 597 # remote repo with the local repo, then restoring the stashed artifacts. 598 for remote_repo in maven_repo_dirs: 599 remote_repo = os.path.join(cwd, remote_repo) 600 paths_to_copy = [] 601 602 # If we're including everything, move the entire repo to temp. 603 if include_all: 604 cp(remote_repo, working_dir) 605 else: 606 # Move included artifacts from repo to temp. 607 for group_artifact in include: 608 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact)) 609 remote_path = os.path.join(remote_repo, artifact_path) 610 working_path = os.path.join(working_dir, artifact_path) 611 if os.path.exists(remote_path): 612 print(f'Included {group_artifact} in update') 613 paths_to_copy.append([remote_path, working_path]) 614 for [remote_path, working_path] in paths_to_copy: 615 mv(remote_path, working_path) 616 617 # Replace all remaining artifacts in remote repo with local repo. 618 cp(local_repo, remote_repo) 619 620 # If we're including everything, restore the entire repo. 621 if include_all: 622 cp(working_dir, remote_repo) 623 else: 624 # Restore included artifacts to remote repo. 625 for [remote_path, working_path] in paths_to_copy: 626 mv(working_path, remote_path) 627 628 # Handle exclusions by replacing the remote artifacts for the exclusions with local artifacts. 629 # This must happen before we parse the artifacts. 630 for remote_repo in maven_repo_dirs: 631 for group_artifact in exclude: 632 artifact_path = os.path.join('m2repository', path_for_artifact(group_artifact)) 633 remote_path = os.path.join(remote_repo, artifact_path) 634 if os.path.exists(remote_path): 635 rm(remote_path) 636 local_path = os.path.join(local_repo, artifact_path) 637 if os.path.exists(local_path): 638 print(f'Excluded {group_artifact} from update, used local artifact') 639 mv(local_path, remote_path) 640 else: 641 print(f'Excluded {group_artifact} from update, no local artifact present') 642 643 # Parse artifacts. 644 maven_lib_info = detect_artifacts(maven_repo_dirs) 645 646 if not maven_lib_info: 647 print_e('Failed to detect artifacts') 648 return False 649 650 # Move libraries into the working directory, performing any necessary transformations. 651 for info in maven_lib_info.values(): 652 transform_maven_lib(working_dir, info, extract_res) 653 654 # Generate a single Android.bp that specifies to use all of the above artifacts. 655 makefile = os.path.join(working_dir, 'Android.bp') 656 with open(makefile, 'w') as f: 657 args = ['pom2bp'] 658 args.extend(['-sdk-version', '31']) 659 args.extend(['-default-min-sdk-version', '24']) 660 if not write_pom2bp_cmd: 661 args.extend(['-write-cmd=false']) 662 if include_static_deps: 663 args.append('-static-deps') 664 if prepend: 665 args.append(f'-prepend={prepend}') 666 rewrite_names = sorted(maven_to_make.keys()) 667 args.extend([f'-rewrite=^{name}$={maven_to_make[name]["name"]}' for name in rewrite_names]) 668 args.extend([f'-rewrite=^{key}$={value}' for key, value in deps_rewrite.items()]) 669 args.extend(["-extra-static-libs=" + maven_to_make[name]['name'] + "=" + ",".join( 670 sorted(maven_to_make[name]['extra-static-libs'])) for name in maven_to_make if 671 'extra-static-libs' in maven_to_make[name]]) 672 args.extend(["-optional-uses-libs=" + maven_to_make[name]['name'] + "=" + ",".join( 673 sorted(maven_to_make[name]['optional-uses-libs'])) for name in maven_to_make if 674 'optional-uses-libs' in maven_to_make[name]]) 675 args.extend([f'-host={name}' for name in maven_to_make 676 if maven_to_make[name].get('host')]) 677 args.extend([f'-host-and-device={name}' for name in maven_to_make 678 if maven_to_make[name].get('host_and_device')]) 679 args.extend(['.']) 680 subprocess.check_call(args, stdout=f, cwd=working_dir) 681 682 # Replace the old directory. 683 local_repo = os.path.join(cwd, transformed_dir) 684 mv(working_dir, local_repo) 685 return True 686 687 688def transform_maven_lib(working_dir, artifact_info, extract_res): 689 """Transforms the specified artifact for use in the Android build system. 690 691 Moves relevant files for the artifact represented by artifact_info of type MavenLibraryInfo into 692 the appropriate path inside working_dir, unpacking files needed by the build system from AARs. 693 694 Args: 695 working_dir: The directory into which the artifact should be moved 696 artifact_info: A MavenLibraryInfo representing the library artifact 697 extract_res: True to extract resources from AARs, false otherwise. 698 """ 699 # Move library into working dir 700 new_dir = os.path.normpath( 701 os.path.join(working_dir, os.path.relpath(artifact_info.dir, artifact_info.repo_dir))) 702 mv(artifact_info.dir, new_dir) 703 704 maven_lib_type = os.path.splitext(artifact_info.file)[1][1:] 705 706 group_artifact = artifact_info.key 707 make_lib_name = maven_to_make[group_artifact]['name'] 708 make_dir_name = maven_to_make[group_artifact]['path'] 709 710 artifact_file = os.path.join(new_dir, artifact_info.file) 711 712 if maven_lib_type == 'aar': 713 if extract_res: 714 target_dir = os.path.join(working_dir, make_dir_name) 715 if not os.path.exists(target_dir): 716 os.makedirs(target_dir) 717 718 process_aar(artifact_file, target_dir) 719 720 with zipfile.ZipFile(artifact_file) as zip_file: 721 manifests_dir = os.path.join(working_dir, 'manifests') 722 lib_path = Path(os.path.join(manifests_dir, make_lib_name)) 723 manifest_path = lib_path / 'AndroidManifest.xml' 724 zip_file.extract('AndroidManifest.xml', lib_path.as_posix()) 725 contents = check_startup_initializers(manifest_path) 726 if contents: 727 manifest_path.write_text(contents) 728 729 730def process_aar(artifact_file, target_dir): 731 """Extracts and cleans up the contents of an AAR file to the specified directory. 732 733 Removes classes.jar, empty directories, and denylisted files. 734 735 Args: 736 artifact_file: path to the AAR to extract 737 target_dir: directory into which the contents should be extracted 738 """ 739 # Extract AAR file to target_dir. 740 with zipfile.ZipFile(artifact_file) as zip_file: 741 zip_file.extractall(target_dir) 742 743 # Remove classes.jar 744 classes_jar = os.path.join(target_dir, 'classes.jar') 745 if os.path.exists(classes_jar): 746 os.remove(classes_jar) 747 748 # Remove empty dirs. 749 for root, dirs, files in os.walk(target_dir, topdown=False): 750 for dir_name in dirs: 751 dir_path = os.path.join(root, dir_name) 752 if not os.listdir(dir_path): 753 os.rmdir(dir_path) 754 755 # Remove top-level cruft. 756 for file in denylist_files: 757 file_path = os.path.join(target_dir, file) 758 if os.path.exists(file_path): 759 os.remove(file_path) 760 761 762def fetch_gmaven_artifact(artifact): 763 """Fetch a GMaven artifact. 764 765 Downloads a GMaven artifact 766 (https://developer.android.com/studio/build/dependencies#gmaven-access) 767 768 Args: 769 artifact: an instance of GMavenArtifact. 770 """ 771 pom_path = maven_path_for_artifact( 772 'gmaven', artifact.group, artifact.library, artifact.version, 'pom') 773 artifact_path = maven_path_for_artifact( 774 'gmaven', artifact.group, artifact.library, artifact.version, artifact.ext) 775 776 download_file_to_disk(artifact.get_pom_file_url(), pom_path) 777 download_file_to_disk(artifact.get_artifact_url(), artifact_path) 778 779 return os.path.dirname(artifact_path) 780 781 782def download_file_to_disk(url, filepath): 783 """Download the file at URL to the location dictated by the path. 784 785 Args: 786 url: Remote URL to download file from. 787 filepath: Filesystem path to write the file to. 788 """ 789 print(f'Downloading URL: {url}') 790 file_data = request.urlopen(url) 791 792 try: 793 os.makedirs(os.path.dirname(filepath)) 794 except os.error: 795 # This is a common situation - os.makedirs fails if dir already exists. 796 pass 797 try: 798 with open(filepath, 'wb') as f: 799 f.write(six.ensure_binary(file_data.read())) 800 except Exception as e: 801 print_e(e.__class__, 'occurred while reading', filepath) 802 os.remove(os.path.dirname(filepath)) 803 raise 804 805 806def check_startup_initializers(manifest_path: Path) -> Optional[str]: 807 try: 808 for prefix in android_manifest_namepaces: 809 ET.register_namespace(prefix, android_manifest_namepaces[prefix]) 810 811 # Use ElementTree to check if we need updates. 812 # That way we avoid false positives. 813 contents = manifest_path.read_text() 814 root = ET.fromstring(contents) 815 needs_changes = _check_node(root) 816 if needs_changes: 817 # Ideally we would use ElementTree here. 818 # Instead, we are using regular expressions here so we can 819 # preserve comments and whitespaces. 820 lines = contents.splitlines() 821 output = StringIO() 822 for line in lines: 823 matcher = startup_initializer_pattern.match(line) 824 if matcher: 825 prefix = matcher.group(1) 826 # Adding an explicit tools:node="remove" so this is still traceable 827 # when looking at the source. 828 output.write(f'{prefix}android:value="androidx.startup"\n') 829 output.write(f'{prefix}tools:node="remove" />') 830 else: 831 output.write(line) 832 output.write('\n') 833 834 output.write('\n') 835 return output.getvalue() 836 except BaseException as exception: 837 print( 838 f'Unable to parse manifest file with path {manifest_path}.\n\n Details ({exception})' 839 ) 840 841def _attribute_name(namespace: str, attribute: str) -> str: 842 if not namespace in android_manifest_namepaces: 843 raise ValueError(f'Unexpected namespace {namespace}') 844 845 return f'{{{android_manifest_namepaces[namespace]}}}{attribute}' 846 847 848def _check_node(node: ET.Element) -> bool: 849 for child in node: 850 # Find the initialization provider 851 is_provider = child.tag == 'provider' 852 provider_name = child.attrib.get(_attribute_name('android', 'name')) 853 is_initialization_provider = provider_name == 'androidx.startup.InitializationProvider' 854 855 if is_provider and is_initialization_provider: 856 metadata_nodes = child.findall('meta-data', namespaces=android_manifest_namepaces) 857 return _needs_disable_initialization(metadata_nodes) 858 859 if len(child) > 0: 860 return _check_node(child) 861 862 return False 863 864 865def _needs_disable_initialization(metadata_nodes: Iterable[ET.Element]) -> bool: 866 needs_update = False 867 for node in metadata_nodes: 868 name = node.attrib.get(_attribute_name('android', 'name')) 869 value = node.attrib.get(_attribute_name('android', 'value')) 870 if value == 'androidx.startup': 871 if name not in enabled_initializers: 872 needs_update = True 873 874 return needs_update 875 876 877def update_gmaven(gmaven_artifacts_list): 878 artifacts = [GMavenArtifact(artifact) for artifact in gmaven_artifacts_list] 879 for artifact in artifacts: 880 if artifact.version == 'latest': 881 artifact.version = artifact.get_latest_version() 882 883 if not transform_maven_repos(['gmaven'], gmaven_dir, extract_res=False): 884 return [] 885 return [artifact.key for artifact in artifacts] 886 887 888def update_androidx(target, build_id, local_file, include, exclude, beyond_corp): 889 """Fetches and extracts Jetpack library prebuilts. 890 891 Args: 892 target: Android build server target name, must be specified if local_file is empty 893 build_id: Optional Android build server ID, must be specified if local_file is empty 894 local_file: Optional local top-of-tree ZIP, must be specified if build_id is empty 895 include: List of Maven groupIds or unversioned artifact coordinates to include for 896 updates, ex. android.core or androidx.core:core 897 exclude: List of Maven groupIds or unversioned artifact coordinates to exclude from 898 updates, ex. android.core or androidx.core:core 899 beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher 900 Returns: 901 True if successful, false otherwise. 902 """ 903 if build_id: 904 repo_file = 'top-of-tree-m2repository-all-%s.zip' % build_id.fs_id 905 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp, None) 906 else: 907 repo_dir = fetch_and_extract(target, None, None, beyond_corp, local_file) 908 if not repo_dir: 909 print_e('Failed to extract AndroidX repository') 910 return False 911 912 prepend_path = os.path.relpath('update_prebuilts/prepend_androidx_license', start=temp_dir) 913 914 # Transform the repo archive into a Makefile-compatible format. 915 if not transform_maven_repos([repo_dir], androidx_dir, write_pom2bp_cmd=False, 916 extract_res=False, include=include, exclude=exclude, 917 prepend=prepend_path): 918 return False 919 920 # Import JavaPlugins.bp in Android.bp. 921 makefile = os.path.join(androidx_dir, 'Android.bp') 922 with open(makefile, 'a+') as f: 923 f.write('\nbuild = ["JavaPlugins.bp"]\n') 924 925 # Keep OWNERs file, JavaPlugins.bp file, and TEST_MAPPING files untouched. 926 files_to_restore = [androidx_owners, java_plugins_bp_path, test_mapping_file, 927 drop_config_toml, compose_test_mapping_file] 928 for file_to_restore in files_to_restore: 929 # Ignore any output or error - these files are not gauranteed to exist, but 930 # if they do, we want to restore them. 931 subprocess.call(['git', 'restore', file_to_restore], 932 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 933 934 return True 935 936 937def update_jetifier(target, build_id, beyond_corp): 938 """ 939 Fetches and extracts Jetifier tool prebuilts. 940 941 Args: 942 target: Android build server target name 943 build_id: Android build server ID 944 beyond_corp: Whether to use BeyondCorp-compatible artifact fetcher 945 Return: 946 Whether the prebuilt was successfully updated. 947 """ 948 repo_file = 'jetifier-standalone.zip' 949 repo_dir = fetch_and_extract(target, build_id.url_id, repo_file, beyond_corp) 950 if not repo_dir: 951 print_e('Failed to extract Jetifier') 952 return False 953 954 rm(jetifier_dir) 955 mv(os.path.join(repo_dir, 'jetifier-standalone'), jetifier_dir) 956 os.chmod(os.path.join(jetifier_dir, 'bin', 'jetifier-standalone'), 0o755) 957 return True 958 959 960def update_constraint(local_file): 961 """ 962 Extracts ConstraintLayout library prebuilts. 963 964 Args: 965 local_file: local Maven repository ZIP containing library artifacts 966 Return: 967 Whether the prebuilts were successfully updated. 968 """ 969 repo_dir = extract_artifact(local_file) 970 if not repo_dir: 971 print_e('Failed to extract Constraint Layout') 972 return False 973 return transform_maven_repos([repo_dir], os.path.join(extras_dir, 'constraint-layout-x'), 974 extract_res=False) 975 976 977def update_material(local_file): 978 """ 979 Extracts Material Design Components library prebuilts. 980 981 Args: 982 local_file: local Maven repository ZIP containing library artifacts 983 Return: 984 Whether the prebuilts were successfully updated. 985 """ 986 design_dir = extract_artifact(local_file) 987 if not design_dir: 988 print_e('Failed to extract Material Design Components') 989 return False 990 return transform_maven_repos([design_dir], os.path.join(extras_dir, 'material-design-x'), 991 extract_res=False) 992 993 994def fetch_artifact(target, build_id, artifact_path, beyond_corp, local_mode): 995 if not local_mode: 996 return buildserver_fetch_artifact(target, build_id, artifact_path, beyond_corp) 997 998 copy_from = os.path.join(repo_root_dir.resolve(), 'out/dist', artifact_path) 999 copy_to = os.path.join('.', os.path.dirname(artifact_path)) 1000 print(f'Copying {copy_from} to {copy_to}...') 1001 result_path = None 1002 try: 1003 if not os.path.exists(copy_to): 1004 os.makedirs(copy_to) 1005 copied = 0 1006 for file in glob.glob(copy_from): 1007 result_path = shutil.copy(file, copy_to) 1008 copied += 1 1009 # Multiple files, return destination folder. 1010 if copied > 1: 1011 result_path = artifact_path 1012 except Exception as e: 1013 print(f'Error: {e} occured while copying') 1014 raise 1015 return result_path 1016 1017 1018def fetch_artifacts(target, build_id, artifact_dict, beyond_corp, local_mode): 1019 if not local_mode: 1020 return buildserver_fetch_artifacts(target, build_id, artifact_dict, beyond_corp) 1021 1022 for artifact, target_path in artifact_dict.items(): 1023 artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp, local_mode) 1024 if not artifact_path: 1025 return False 1026 mv(artifact_path, target_path) 1027 return True 1028 1029 1030def update_framework(target, build_id, sdk_dir, beyond_corp, local_mode): 1031 api_scope_list = ['public', 'system', 'test', 'module-lib', 'system-server'] 1032 if sdk_dir == 'current': 1033 api_scope_list.append('core') 1034 1035 for api_scope in api_scope_list: 1036 target_dir = os.path.join(sdk_dir, api_scope) 1037 if api_scope == 'core': 1038 artifact_to_path = {'core.current.stubs.jar': os.path.join(target_dir, 'android.jar')} 1039 else: 1040 artifact_to_path = { 1041 'apistubs/android/' + api_scope + '/*.jar': os.path.join(target_dir, '*'), 1042 } 1043 if api_scope == 'public' or api_scope == 'module-lib': 1044 # Distinct core-for-system-modules.jar files are only provided 1045 # for the public and module-lib API surfaces. 1046 artifact_to_path[ 1047 'system-modules/' + api_scope + '/core-for-system-modules.jar'] = os.path.join( 1048 target_dir, '*') 1049 1050 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp, local_mode): 1051 return False 1052 1053 if api_scope == 'public': 1054 # Fetch a few artifacts from the public sdk. 1055 if local_mode: 1056 artifact = 'android-sdk*.zip' 1057 else: 1058 artifact = f'sdk-repo-linux-platforms-{build_id.fs_id}.zip' 1059 artifact_path = fetch_artifact(target, build_id.url_id, artifact, beyond_corp, local_mode) 1060 if not artifact_path: 1061 return False 1062 1063 with zipfile.ZipFile(artifact_path) as zipFile: 1064 extra_files = [ 1065 'android.jar', 1066 'framework.aidl', 1067 'uiautomator.jar'] 1068 for filename in extra_files: 1069 matches = list(filter(lambda path: filename in path, zipFile.namelist())) 1070 if len(matches) != 1: 1071 print_e('Expected 1 file named \'%s\' in zip %s, found %d' % 1072 (filename, zipFile.filename, len(matches))) 1073 return False 1074 zip_path = matches[0] 1075 src_path = zipFile.extract(zip_path) 1076 dst_path = os.path.join(target_dir, filename) 1077 mv(src_path, dst_path) 1078 1079 # Fetch the lint api databases 1080 lint_database_artifacts = {} 1081 for api_scope in ['public', 'system', 'module-lib', 'system-server']: 1082 data_folder = 'data' if api_scope == 'public' else api_scope + '-data' 1083 lint_database_artifacts[os.path.join(data_folder, 'api-versions.xml')] = os.path.join(sdk_dir, api_scope, 'data', 'api-versions.xml') 1084 lint_database_artifacts[os.path.join(data_folder, 'annotations.zip')] = os.path.join(sdk_dir, api_scope, 'data', 'annotations.zip') 1085 fetch_artifacts(target, build_id, lint_database_artifacts, beyond_corp, local_mode) 1086 1087 return True 1088 1089 1090def update_makefile(build_id): 1091 template = '"%s",\n\ 1092 "current"' 1093 makefile = os.path.join(git_dir, 'Android.bp') 1094 1095 with open(makefile, 'r+') as f: 1096 contents = f.read().replace('"current"', template % build_id) 1097 f.seek(0) 1098 f.write(contents) 1099 1100 return True 1101 1102 1103def finalize_sdk(target, build_id, sdk_version, beyond_corp, local_mode): 1104 target_finalize_dir = sdk_version 1105 1106 for api_scope in ['public', 'system', 'test', 'module-lib', 'system-server']: 1107 artifact_to_path = {f'apistubs/android/{api_scope}/api/*.txt': os.path.join( 1108 target_finalize_dir, api_scope, 'api', '*')} 1109 1110 if not fetch_artifacts(target, build_id, artifact_to_path, beyond_corp, local_mode): 1111 return False 1112 1113 return update_framework(target, build_id, target_finalize_dir, beyond_corp, local_mode) and update_makefile( 1114 target_finalize_dir) 1115 1116 1117def update_framework_current(target, build_id, beyond_corp, local_mode): 1118 return update_framework(target, build_id, current_path, beyond_corp, local_mode) 1119 1120 1121def update_buildtools(target, arch, build_id, beyond_corp): 1122 artifact_path = fetch_and_extract(target, build_id.url_id, 1123 f'sdk-repo-{arch}-build-tools-{build_id.fs_id}.zip', 1124 beyond_corp) 1125 if not artifact_path: 1126 return False 1127 1128 top_level_dir = os.listdir(artifact_path)[0] 1129 src_path = os.path.join(artifact_path, top_level_dir) 1130 dst_path = os.path.join(buildtools_dir, arch) 1131 1132 # There are a few libraries that have been manually added to the 1133 # build tools, copy them from the destination back to the source 1134 # before the destination is overwritten. 1135 files_to_save = ( 1136 'lib64/libconscrypt_openjdk_jni.dylib', 1137 'lib64/libconscrypt_openjdk_jni.so', 1138 'bin/lib64/libwinpthread-1.dll', 1139 ) 1140 for file in files_to_save: 1141 src_file = os.path.join(dst_path, file) 1142 dst_file = os.path.join(src_path, file) 1143 if os.path.exists(dst_path): 1144 mv(src_file, dst_file) 1145 1146 mv(src_path, dst_path) 1147 1148 # Move all top-level files to /bin and make them executable 1149 bin_path = os.path.join(dst_path, 'bin') 1150 top_level_files = filter(lambda e: os.path.isfile(os.path.join(dst_path, e)), os.listdir(dst_path)) 1151 for file in top_level_files: 1152 src_file = os.path.join(dst_path, file) 1153 dst_file = os.path.join(bin_path, file) 1154 mv(src_file, dst_file) 1155 os.chmod(dst_file, 0o755) 1156 1157 # Make the files under lld-bin executable 1158 lld_bin_files = os.listdir(os.path.join(dst_path, 'lld-bin')) 1159 for file in lld_bin_files: 1160 os.chmod(os.path.join(dst_path, 'lld-bin', file), 0o755) 1161 1162 # Remove renderscript 1163 rm(os.path.join(dst_path, 'renderscript')) 1164 1165 return True 1166 1167 1168def has_uncommitted_changes(): 1169 try: 1170 # Make sure we don't overwrite any pending changes. 1171 diff_command = f'cd {git_dir} && git diff --quiet' 1172 subprocess.check_call(diff_command, shell=True) 1173 subprocess.check_call(f'{diff_command} --cached', shell=True) 1174 return False 1175 except subprocess.CalledProcessError: 1176 return True 1177 1178def check_string_major_minor_format(s): 1179 # Verify that the string follows the major.minor format: 1180 # Examples of allowed strings: 1.0, 2.34 1181 # Examples of disallowed strings: 0.1, 1.01 1182 if not re.match(r'[1-9][0-9]*\.(0|[1-9][0-9]*)', s): 1183 raise ValueError('string not a major.minor version (e.g. 1.0, 2.34, 56.7890)') 1184 return s 1185 1186 1187def main(): 1188 parser = argparse.ArgumentParser( 1189 description='Update current prebuilts') 1190 parser.add_argument( 1191 'source', nargs='?', 1192 help='Build server build ID or local Maven ZIP file') 1193 parser.add_argument( 1194 '-m', '--material', action='store_true', 1195 help='If specified, updates only Material Design Components') 1196 parser.add_argument( 1197 '-c', '--constraint', action='store_true', 1198 help='If specified, updates only Constraint Layout') 1199 parser.add_argument( 1200 '-j', '--jetifier', action='store_true', 1201 help='If specified, updates only Jetifier') 1202 parser.add_argument( 1203 '-p', '--platform', action='store_true', 1204 help='If specified, updates only the Android Platform') 1205 parser.add_argument( 1206 '-f', '--finalize_sdk', type=check_string_major_minor_format, 1207 help='Finalize the build as the specified SDK version. Must be used together with -e') 1208 parser.add_argument( 1209 '-e', '--finalize_extension', type=int, 1210 help='Finalize the build as the specified extension SDK version. Must be used together with -f') 1211 parser.add_argument('--bug', type=int, help='The bug number to add to the commit message.') 1212 parser.add_argument( 1213 '--sdk_target', 1214 default=framework_sdk_target, 1215 help='If specified, the name of the build target from which to retrieve the SDK when -p or -f ' 1216 'is specified.') 1217 parser.add_argument( 1218 '-b', '--buildtools', action='store_true', 1219 help='If specified, updates only the Build Tools') 1220 parser.add_argument( 1221 '-x', '--androidx', action='store_true', 1222 help='If specified, updates only the Jetpack (androidx) libraries excluding those covered by ' 1223 'other arguments') 1224 parser.add_argument( 1225 '--include', action='append', default=[], 1226 help='If specified with -x, includes the specified Jetpack library Maven group or artifact for ' 1227 'updates. Applied before exclude.') 1228 parser.add_argument( 1229 '--exclude', action='append', default=[], 1230 help='If specified with -x, excludes the specified Jetpack library Maven group or artifact ' 1231 'from updates') 1232 parser.add_argument( 1233 '-g', '--gmaven', action='store_true', 1234 help='If specified, updates only the artifact from GMaven libraries excluding those covered by ' 1235 'other arguments') 1236 parser.add_argument( 1237 '--commit-first', action='store_true', 1238 help='If specified, then if uncommited changes exist, commit before continuing') 1239 parser.add_argument( 1240 '--beyond-corp', action='store_true', 1241 help='If specified, then fetch artifacts with tooling that works on BeyondCorp devices') 1242 parser.add_argument( 1243 '--local_mode', action="store_true", 1244 help='Local mode: use locally built artifacts and don\'t upload the result to Gerrit.') 1245 rm(temp_dir) 1246 1247 args = parser.parse_args() 1248 1249 # Validate combinations of arguments. 1250 if not args.source and (args.platform or args.buildtools or args.jetifier 1251 or args.androidx or args.material or args.finalize_sdk 1252 or args.constraint): 1253 parser.error('You must specify a build ID or local Maven ZIP file') 1254 sys.exit(1) 1255 if not (args.gmaven or args.platform or args.buildtools or args.jetifier 1256 or args.androidx or args.material or args.finalize_sdk 1257 or args.finalize_extension or args.constraint): 1258 parser.error('You must specify at least one target to update') 1259 sys.exit(1) 1260 if args.local_mode and not args.finalize_sdk: 1261 parser.error('Local mode can only be used when finalizing an SDK.') 1262 sys.exit(1) 1263 if (args.finalize_sdk is None) != (args.finalize_extension is None): 1264 parser.error('Either both or neither of -e and -f must be specified.') 1265 sys.exit(1) 1266 if args.finalize_sdk and not args.bug: 1267 parser.error('Specifying a bug ID with --bug is required when finalizing an SDK.') 1268 sys.exit(1) 1269 1270 # Validate the build environment for POM-dependent targets. 1271 if (args.constraint or args.material or args.androidx or args.gmaven) \ 1272 and which('pom2bp') is None: 1273 parser.error('Cannot find pom2bp in path; please run lunch to set up build environment. ' 1274 'You may also need to run \'m pom2bp\' if it hasn\'t been built already.') 1275 sys.exit(1) 1276 1277 # Validate include/exclude arguments. 1278 if args.exclude: 1279 invalid_spec = find_invalid_spec(args.exclude) 1280 if invalid_spec: 1281 parser.error('Unknown artifact specification in exclude: ' + invalid_spec) 1282 sys.exit(1) 1283 if args.include: 1284 invalid_spec = find_invalid_spec(args.include) 1285 if invalid_spec: 1286 parser.error('Unknown artifact specification in include: ' + invalid_spec) 1287 sys.exit(1) 1288 1289 # Validate the git status. 1290 if not args.local_mode and has_uncommitted_changes(): 1291 if args.commit_first: 1292 subprocess.check_call(f'cd {git_dir} && git add -u', shell=True) 1293 subprocess.check_call(f'cd {git_dir} && git commit -m \'save working state\'', 1294 shell=True) 1295 if not args.local_mode and has_uncommitted_changes(): 1296 self_file = os.path.basename(__file__) 1297 print_e(f'FAIL: There are uncommitted changes here. Please commit or stash before ' 1298 f'continuing, because {self_file} will run "git reset --hard" if execution fails') 1299 sys.exit(1) 1300 1301 if args.bug: 1302 commit_msg_suffix = f'\n\nBug: {args.bug}' 1303 else: 1304 commit_msg_suffix = '' 1305 1306 # Are we fetching a build ID or using a local file? 1307 build_id = None 1308 file = None 1309 if args.source: 1310 build_id = parse_build_id(args.source) 1311 if build_id is None: 1312 file = args.source 1313 1314 try: 1315 components = [] 1316 if args.constraint: 1317 if update_constraint(file): 1318 components.append('Constraint Layout') 1319 else: 1320 print_e('Failed to update Constraint Layout, aborting...') 1321 sys.exit(1) 1322 if args.material: 1323 if update_material(file): 1324 components.append('Material Design Components') 1325 else: 1326 print_e('Failed to update Material Design Components, aborting...') 1327 sys.exit(1) 1328 if args.gmaven: 1329 updated_artifacts = update_gmaven(gmaven_artifacts) 1330 if updated_artifacts: 1331 components.append('\n'.join(updated_artifacts)) 1332 else: 1333 print_e('Failed to update GMaven, aborting...') 1334 sys.exit(1) 1335 if args.androidx: 1336 if update_androidx('androidx', build_id, file, args.include, args.exclude, 1337 args.beyond_corp): 1338 components.append('AndroidX') 1339 else: 1340 print_e('Failed to update AndroidX, aborting...') 1341 sys.exit(1) 1342 if args.jetifier: 1343 if update_jetifier('androidx', build_id, args.beyond_corp): 1344 components.append('Jetifier') 1345 else: 1346 print_e('Failed to update Jetifier, aborting...') 1347 sys.exit(1) 1348 if args.platform or args.finalize_sdk: 1349 if update_framework_current(args.sdk_target, build_id, args.beyond_corp, args.local_mode): 1350 components.append('platform SDK') 1351 else: 1352 print_e('Failed to update platform SDK, aborting...') 1353 sys.exit(1) 1354 if args.finalize_sdk: 1355 n = args.finalize_sdk 1356 if not finalize_sdk(args.sdk_target, build_id, n, args.beyond_corp, args.local_mode): 1357 print_e('Failed to finalize SDK %d, aborting...' % n) 1358 sys.exit(1) 1359 1360 if not args.local_mode: 1361 # HACK: extension sdk finalization will create a new branch, hiding this commit. 1362 # Let's create it in advance for now. 1363 # TODO(b/228451704) do a proper fix? 1364 branch_name = 'finalize-%d' % args.finalize_extension 1365 subprocess.check_output(['repo', 'start', branch_name]) 1366 # We commit the finalized dir separately from the current sdk update. 1367 msg = f'Import final sdk version {n} from build {build_id.url_id}{commit_msg_suffix}' 1368 subprocess.check_call(['git', 'add', '%d' % n]) 1369 subprocess.check_call(['git', 'add', 'Android.bp']) 1370 subprocess.check_call(['git', 'commit', '-m', msg]) 1371 1372 # Finalize extension sdk level 1373 readme = (f'Finalized together with ' 1374 f'Android {args.finalize_sdk} (all modules)') 1375 cmd = extension_sdk_finalization_cmd.format( 1376 readme=readme, 1377 bug=args.bug, 1378 extension_version=args.finalize_extension, 1379 build_id=build_id.url_id, 1380 local_mode='--local_mode' if args.local_mode else '') 1381 subprocess.check_call(shlex.split(cmd), cwd=repo_root_dir.resolve()) 1382 if args.buildtools: 1383 if update_buildtools('sdk-sdk_mac', 'darwin', build_id, args.beyond_corp) \ 1384 and update_buildtools('sdk', 'linux', build_id, args.beyond_corp) \ 1385 and update_buildtools('sdk', 'windows', build_id, args.beyond_corp): 1386 components.append('build tools') 1387 else: 1388 print_e('Failed to update build tools, aborting...') 1389 sys.exit(1) 1390 1391 if args.local_mode: 1392 print('Updated prebuilts using locally built artifacts. Don\'t submit or use for anything besides local testing.') 1393 sys.exit(0) 1394 1395 # Build the git commit. 1396 subprocess.check_call(['git', 'add', current_path, buildtools_dir]) 1397 1398 # Build the commit message. 1399 components_msg = ', '.join(components) 1400 argv_msg = ' '.join(sys.argv) 1401 if not args.source and args.gmaven: 1402 src_msg = 'GMaven' 1403 elif not args.source.isnumeric(): 1404 src_msg = 'local Maven ZIP' 1405 else: 1406 src_msg = f'build {build_id.url_id}' 1407 msg = f'Import {components_msg} from {src_msg}\n\n{argv_msg}{commit_msg_suffix}' 1408 1409 # Create the git commit. 1410 subprocess.check_call(['git', 'commit', '-q', '-m', msg]) 1411 1412 if args.finalize_sdk: 1413 print('NOTE: Created three commits:') 1414 subprocess.check_call(['git', 'log', '-3', '--oneline']) 1415 else: 1416 print('Created commit:') 1417 subprocess.check_call(['git', 'log', '-1', '--oneline']) 1418 print('Remember to test this change before uploading it to Gerrit!') 1419 1420 except Exception as e: 1421 print(f'ERROR: {e} occured while updating prebuilts') 1422 raise 1423 finally: 1424 if args.local_mode: 1425 print('No cleaning up in local mode, manual cleanup required.') 1426 else: 1427 # Revert all stray files, including the downloaded zip. 1428 try: 1429 with open(os.devnull, 'w') as bitbucket: 1430 subprocess.check_call(['git', 'add', '-Af', '.'], stdout=bitbucket) 1431 subprocess.check_call( 1432 ['git', 'commit', '-m', 'COMMIT TO REVERT - RESET ME!!!', '--allow-empty'], 1433 stdout=bitbucket) 1434 subprocess.check_call(['git', 'reset', '--hard', 'HEAD~1'], stdout=bitbucket) 1435 except subprocess.CalledProcessError: 1436 print_e('ERROR: Failed cleaning up, manual cleanup required!!!') 1437 1438 1439# Add automatic entries to maven_to_make. 1440populate_maven_to_make(maven_to_make) 1441 1442if __name__ == '__main__': 1443 main() 1444