1#!/usr/bin/env python3 2# 3# Copyright 2012 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Compile Android resources into an intermediate APK. 8 9This can also generate an R.txt, and an .srcjar file containing the proper 10final R.java class for all resource packages the APK depends on. 11 12This will crunch images with aapt2. 13""" 14 15import argparse 16import collections 17import contextlib 18import filecmp 19import hashlib 20import logging 21import os 22import pathlib 23import re 24import shutil 25import subprocess 26import sys 27import textwrap 28from xml.etree import ElementTree 29 30from util import build_utils 31from util import diff_utils 32from util import manifest_utils 33from util import parallel 34from util import protoresources 35from util import resource_utils 36import action_helpers # build_utils adds //build to sys.path. 37import zip_helpers 38 39 40# Pngs that we shouldn't convert to webp. Please add rationale when updating. 41_PNG_WEBP_EXCLUSION_PATTERN = re.compile('|'.join([ 42 # Crashes on Galaxy S5 running L (https://crbug.com/807059). 43 r'.*star_gray\.png', 44 # Android requires pngs for 9-patch images. 45 r'.*\.9\.png', 46 # Daydream requires pngs for icon files. 47 r'.*daydream_icon_.*\.png' 48])) 49 50 51def _ParseArgs(args): 52 """Parses command line options. 53 54 Returns: 55 An options object as from argparse.ArgumentParser.parse_args() 56 """ 57 parser = argparse.ArgumentParser(description=__doc__) 58 59 input_opts = parser.add_argument_group('Input options') 60 output_opts = parser.add_argument_group('Output options') 61 62 input_opts.add_argument('--include-resources', 63 action='append', 64 required=True, 65 help='Paths to arsc resource files used to link ' 66 'against. Can be specified multiple times.') 67 input_opts.add_argument( 68 '--dependencies-res-zips', 69 default=[], 70 help='Resources zip archives from dependents. Required to ' 71 'resolve @type/foo references into dependent libraries.') 72 input_opts.add_argument( 73 '--extra-res-packages', 74 help='Additional package names to generate R.java files for.') 75 input_opts.add_argument( 76 '--aapt2-path', required=True, help='Path to the Android aapt2 tool.') 77 input_opts.add_argument( 78 '--android-manifest', required=True, help='AndroidManifest.xml path.') 79 input_opts.add_argument( 80 '--r-java-root-package-name', 81 default='base', 82 help='Short package name for this target\'s root R java file (ex. ' 83 'input of "base" would become gen.base_module). Defaults to "base".') 84 group = input_opts.add_mutually_exclusive_group() 85 group.add_argument( 86 '--shared-resources', 87 action='store_true', 88 help='Make all resources in R.java non-final and allow the resource IDs ' 89 'to be reset to a different package index when the apk is loaded by ' 90 'another application at runtime.') 91 group.add_argument( 92 '--app-as-shared-lib', 93 action='store_true', 94 help='Same as --shared-resources, but also ensures all resource IDs are ' 95 'directly usable from the APK loaded as an application.') 96 input_opts.add_argument( 97 '--package-id', 98 type=int, 99 help='Decimal integer representing custom package ID for resources ' 100 '(instead of 127==0x7f). Cannot be used with --shared-resources.') 101 input_opts.add_argument( 102 '--package-name', 103 help='Package name that will be used to create R class.') 104 input_opts.add_argument( 105 '--rename-manifest-package', help='Package name to force AAPT to use.') 106 input_opts.add_argument( 107 '--arsc-package-name', 108 help='Package name to set in manifest of resources.arsc file. This is ' 109 'only used for apks under test.') 110 input_opts.add_argument( 111 '--shared-resources-allowlist', 112 help='An R.txt file acting as a allowlist for resources that should be ' 113 'non-final and have their package ID changed at runtime in R.java. ' 114 'Implies and overrides --shared-resources.') 115 input_opts.add_argument( 116 '--shared-resources-allowlist-locales', 117 default='[]', 118 help='Optional GN-list of locales. If provided, all strings corresponding' 119 ' to this locale list will be kept in the final output for the ' 120 'resources identified through --shared-resources-allowlist, even ' 121 'if --locale-allowlist is being used.') 122 input_opts.add_argument( 123 '--use-resource-ids-path', 124 help='Use resource IDs generated by aapt --emit-ids.') 125 input_opts.add_argument( 126 '--debuggable', 127 action='store_true', 128 help='Whether to add android:debuggable="true".') 129 input_opts.add_argument('--static-library-version', 130 help='Version code for static library.') 131 input_opts.add_argument('--version-code', help='Version code for apk.') 132 input_opts.add_argument('--version-name', help='Version name for apk.') 133 input_opts.add_argument( 134 '--min-sdk-version', required=True, help='android:minSdkVersion for APK.') 135 input_opts.add_argument( 136 '--target-sdk-version', 137 required=True, 138 help="android:targetSdkVersion for APK.") 139 input_opts.add_argument( 140 '--max-sdk-version', 141 help="android:maxSdkVersion expected in AndroidManifest.xml.") 142 input_opts.add_argument( 143 '--manifest-package', help='Package name of the AndroidManifest.xml.') 144 input_opts.add_argument( 145 '--locale-allowlist', 146 default='[]', 147 help='GN list of languages to include. All other language configs will ' 148 'be stripped out. List may include a combination of Android locales ' 149 'or Chrome locales.') 150 input_opts.add_argument( 151 '--resource-exclusion-regex', 152 default='', 153 help='File-based filter for resources (applied before compiling)') 154 input_opts.add_argument( 155 '--resource-exclusion-exceptions', 156 default='[]', 157 help='GN list of globs that say which files to include even ' 158 'when --resource-exclusion-regex is set.') 159 input_opts.add_argument( 160 '--dependencies-res-zip-overlays', 161 help='GN list with subset of --dependencies-res-zips to use overlay ' 162 'semantics for.') 163 input_opts.add_argument( 164 '--values-filter-rules', 165 help='GN list of source_glob:regex for filtering resources after they ' 166 'are compiled. Use this to filter out entries within values/ files.') 167 input_opts.add_argument('--png-to-webp', action='store_true', 168 help='Convert png files to webp format.') 169 170 input_opts.add_argument('--webp-binary', default='', 171 help='Path to the cwebp binary.') 172 input_opts.add_argument( 173 '--webp-cache-dir', help='The directory to store webp image cache.') 174 input_opts.add_argument( 175 '--is-bundle-module', 176 action='store_true', 177 help='Whether resources are being generated for a bundle module.') 178 input_opts.add_argument( 179 '--uses-split', 180 help='Value to set uses-split to in the AndroidManifest.xml.') 181 input_opts.add_argument( 182 '--verification-version-code-offset', 183 help='Subtract this from versionCode for expectation files') 184 input_opts.add_argument( 185 '--verification-library-version-offset', 186 help='Subtract this from static-library version for expectation files') 187 188 action_helpers.add_depfile_arg(output_opts) 189 output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') 190 output_opts.add_argument('--proto-path', help='Apk output for proto format.') 191 output_opts.add_argument( 192 '--info-path', help='Path to output info file for the partial apk.') 193 output_opts.add_argument( 194 '--srcjar-out', 195 help='Path to srcjar to contain generated R.java.') 196 output_opts.add_argument('--r-text-out', 197 help='Path to store the generated R.txt file.') 198 output_opts.add_argument( 199 '--proguard-file', help='Path to proguard.txt generated file.') 200 output_opts.add_argument( 201 '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') 202 203 diff_utils.AddCommandLineFlags(parser) 204 options = parser.parse_args(args) 205 206 options.include_resources = action_helpers.parse_gn_list( 207 options.include_resources) 208 options.dependencies_res_zips = action_helpers.parse_gn_list( 209 options.dependencies_res_zips) 210 options.extra_res_packages = action_helpers.parse_gn_list( 211 options.extra_res_packages) 212 options.locale_allowlist = action_helpers.parse_gn_list( 213 options.locale_allowlist) 214 options.shared_resources_allowlist_locales = action_helpers.parse_gn_list( 215 options.shared_resources_allowlist_locales) 216 options.resource_exclusion_exceptions = action_helpers.parse_gn_list( 217 options.resource_exclusion_exceptions) 218 options.dependencies_res_zip_overlays = action_helpers.parse_gn_list( 219 options.dependencies_res_zip_overlays) 220 options.values_filter_rules = action_helpers.parse_gn_list( 221 options.values_filter_rules) 222 223 if not options.arsc_path and not options.proto_path: 224 parser.error('One of --arsc-path or --proto-path is required.') 225 226 if options.package_id and options.shared_resources: 227 parser.error('--package-id and --shared-resources are mutually exclusive') 228 229 if options.static_library_version and (options.static_library_version != 230 options.version_code): 231 assert options.static_library_version == options.version_code, ( 232 f'static_library_version={options.static_library_version} must equal ' 233 f'version_code={options.version_code}. Please verify the version code ' 234 'map for this target is defined correctly.') 235 236 return options 237 238 239def _IterFiles(root_dir): 240 for root, _, files in os.walk(root_dir): 241 for f in files: 242 yield os.path.join(root, f) 243 244 245def _RenameLocaleResourceDirs(resource_dirs, path_info): 246 """Rename locale resource directories into standard names when necessary. 247 248 This is necessary to deal with the fact that older Android releases only 249 support ISO 639-1 two-letter codes, and sometimes even obsolete versions 250 of them. 251 252 In practice it means: 253 * 3-letter ISO 639-2 qualifiers are renamed under a corresponding 254 2-letter one. E.g. for Filipino, strings under values-fil/ will be moved 255 to a new corresponding values-tl/ sub-directory. 256 257 * Modern ISO 639-1 codes will be renamed to their obsolete variant 258 for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/). 259 260 * Norwegian macrolanguage strings will be renamed to Bokmal (main 261 Norway language). See http://crbug.com/920960. In practice this 262 means that 'values-no/ -> values-nb/' unless 'values-nb/' already 263 exists. 264 265 * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1 266 locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS'). 267 268 Args: 269 resource_dirs: list of top-level resource directories. 270 """ 271 for resource_dir in resource_dirs: 272 ignore_dirs = {} 273 for path in _IterFiles(resource_dir): 274 locale = resource_utils.FindLocaleInStringResourceFilePath(path) 275 if not locale: 276 continue 277 cr_locale = resource_utils.ToChromiumLocaleName(locale) 278 if not cr_locale: 279 continue # Unsupported Android locale qualifier!? 280 locale2 = resource_utils.ToAndroidLocaleName(cr_locale) 281 if locale != locale2: 282 path2 = path.replace('/values-%s/' % locale, '/values-%s/' % locale2) 283 if path == path2: 284 raise Exception('Could not substitute locale %s for %s in %s' % 285 (locale, locale2, path)) 286 287 # Ignore rather than rename when the destination resources config 288 # already exists. 289 # e.g. some libraries provide both values-nb/ and values-no/. 290 # e.g. material design provides: 291 # * res/values-rUS/values-rUS.xml 292 # * res/values-b+es+419/values-b+es+419.xml 293 config_dir = os.path.dirname(path2) 294 already_has_renamed_config = ignore_dirs.get(config_dir) 295 if already_has_renamed_config is None: 296 # Cache the result of the first time the directory is encountered 297 # since subsequent encounters will find the directory already exists 298 # (due to the rename). 299 already_has_renamed_config = os.path.exists(config_dir) 300 ignore_dirs[config_dir] = already_has_renamed_config 301 if already_has_renamed_config: 302 continue 303 304 build_utils.MakeDirectory(os.path.dirname(path2)) 305 shutil.move(path, path2) 306 path_info.RegisterRename( 307 os.path.relpath(path, resource_dir), 308 os.path.relpath(path2, resource_dir)) 309 310 311def _ToAndroidLocales(locale_allowlist): 312 """Converts the list of Chrome locales to Android config locale qualifiers. 313 314 Args: 315 locale_allowlist: A list of Chromium locale names. 316 Returns: 317 A set of matching Android config locale qualifier names. 318 """ 319 ret = set() 320 for locale in locale_allowlist: 321 locale = resource_utils.ToAndroidLocaleName(locale) 322 if locale is None or ('-' in locale and '-r' not in locale): 323 raise Exception('Unsupported Chromium locale name: %s' % locale) 324 ret.add(locale) 325 # Always keep non-regional fall-backs. 326 language = locale.split('-')[0] 327 ret.add(language) 328 329 return ret 330 331 332def _MoveImagesToNonMdpiFolders(res_root, path_info): 333 """Move images from drawable-*-mdpi-* folders to drawable-* folders. 334 335 Why? http://crbug.com/289843 336 """ 337 for src_dir_name in os.listdir(res_root): 338 src_components = src_dir_name.split('-') 339 if src_components[0] != 'drawable' or 'mdpi' not in src_components: 340 continue 341 src_dir = os.path.join(res_root, src_dir_name) 342 if not os.path.isdir(src_dir): 343 continue 344 dst_components = [c for c in src_components if c != 'mdpi'] 345 assert dst_components != src_components 346 dst_dir_name = '-'.join(dst_components) 347 dst_dir = os.path.join(res_root, dst_dir_name) 348 build_utils.MakeDirectory(dst_dir) 349 for src_file_name in os.listdir(src_dir): 350 src_file = os.path.join(src_dir, src_file_name) 351 dst_file = os.path.join(dst_dir, src_file_name) 352 assert not os.path.lexists(dst_file) 353 shutil.move(src_file, dst_file) 354 path_info.RegisterRename( 355 os.path.relpath(src_file, res_root), 356 os.path.relpath(dst_file, res_root)) 357 358 359def _DeterminePlatformVersion(aapt2_path, jar_candidates): 360 def maybe_extract_version(j): 361 try: 362 return resource_utils.ExtractBinaryManifestValues(aapt2_path, j) 363 except build_utils.CalledProcessError: 364 return None 365 366 def is_sdk_jar(jar_name): 367 if jar_name in ('android.jar', 'android_system.jar'): 368 return True 369 # Robolectric jar looks a bit different. 370 return 'android-all' in jar_name and 'robolectric' in jar_name 371 372 android_sdk_jars = [ 373 j for j in jar_candidates if is_sdk_jar(os.path.basename(j)) 374 ] 375 extract_all = [maybe_extract_version(j) for j in android_sdk_jars] 376 extract_all = [x for x in extract_all if x] 377 if len(extract_all) == 0: 378 raise Exception( 379 'Unable to find android SDK jar among candidates: %s' 380 % ', '.join(android_sdk_jars)) 381 if len(extract_all) > 1: 382 raise Exception( 383 'Found multiple android SDK jars among candidates: %s' 384 % ', '.join(android_sdk_jars)) 385 platform_version_code, platform_version_name = extract_all.pop()[:2] 386 return platform_version_code, platform_version_name 387 388 389def _FixManifest(options, temp_dir): 390 """Fix the APK's AndroidManifest.xml. 391 392 This adds any missing namespaces for 'android' and 'tools', and 393 sets certains elements like 'platformBuildVersionCode' or 394 'android:debuggable' depending on the content of |options|. 395 396 Args: 397 options: The command-line arguments tuple. 398 temp_dir: A temporary directory where the fixed manifest will be written to. 399 Returns: 400 Tuple of: 401 * Manifest path within |temp_dir|. 402 * Original package_name. 403 * Manifest package name. 404 """ 405 doc, manifest_node, app_node = manifest_utils.ParseManifest( 406 options.android_manifest) 407 408 # merge_manifest.py also sets package & <uses-sdk>. We may want to ensure 409 # manifest merger is always enabled and remove these command-line arguments. 410 manifest_utils.SetUsesSdk(manifest_node, options.target_sdk_version, 411 options.min_sdk_version, options.max_sdk_version) 412 orig_package = manifest_node.get('package') or options.manifest_package 413 fixed_package = (options.arsc_package_name or options.manifest_package 414 or orig_package) 415 manifest_node.set('package', fixed_package) 416 417 platform_version_code, platform_version_name = _DeterminePlatformVersion( 418 options.aapt2_path, options.include_resources) 419 manifest_node.set('platformBuildVersionCode', platform_version_code) 420 manifest_node.set('platformBuildVersionName', platform_version_name) 421 if options.version_code: 422 manifest_utils.NamespacedSet(manifest_node, 'versionCode', 423 options.version_code) 424 if options.version_name: 425 manifest_utils.NamespacedSet(manifest_node, 'versionName', 426 options.version_name) 427 if options.debuggable: 428 manifest_utils.NamespacedSet(app_node, 'debuggable', 'true') 429 430 if options.uses_split: 431 uses_split = ElementTree.SubElement(manifest_node, 'uses-split') 432 manifest_utils.NamespacedSet(uses_split, 'name', options.uses_split) 433 434 # Make sure the min-sdk condition is not less than the min-sdk of the bundle. 435 for min_sdk_node in manifest_node.iter('{%s}min-sdk' % 436 manifest_utils.DIST_NAMESPACE): 437 dist_value = '{%s}value' % manifest_utils.DIST_NAMESPACE 438 if int(min_sdk_node.get(dist_value)) < int(options.min_sdk_version): 439 min_sdk_node.set(dist_value, options.min_sdk_version) 440 441 debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') 442 manifest_utils.SaveManifest(doc, debug_manifest_path) 443 return debug_manifest_path, orig_package, fixed_package 444 445 446def _CreateKeepPredicate(resource_exclusion_regex, 447 resource_exclusion_exceptions): 448 """Return a predicate lambda to determine which resource files to keep. 449 450 Args: 451 resource_exclusion_regex: A regular expression describing all resources 452 to exclude, except if they are mip-maps, or if they are listed 453 in |resource_exclusion_exceptions|. 454 resource_exclusion_exceptions: A list of glob patterns corresponding 455 to exceptions to the |resource_exclusion_regex|. 456 Returns: 457 A lambda that takes a path, and returns true if the corresponding file 458 must be kept. 459 """ 460 predicate = lambda path: os.path.basename(path)[0] != '.' 461 if resource_exclusion_regex == '': 462 # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways. 463 return predicate 464 465 # A simple predicate that only removes (returns False for) paths covered by 466 # the exclusion regex or listed as exceptions. 467 return lambda path: ( 468 not re.search(resource_exclusion_regex, path) or 469 build_utils.MatchesGlob(path, resource_exclusion_exceptions)) 470 471 472def _ComputeSha1(path): 473 with open(path, 'rb') as f: 474 data = f.read() 475 return hashlib.sha1(data).hexdigest() 476 477 478def _ConvertToWebPSingle(png_path, cwebp_binary, cwebp_version, webp_cache_dir): 479 sha1_hash = _ComputeSha1(png_path) 480 481 # The set of arguments that will appear in the cache key. 482 quality_args = ['-m', '6', '-q', '100', '-lossless'] 483 484 webp_cache_path = os.path.join( 485 webp_cache_dir, '{}-{}-{}'.format(sha1_hash, cwebp_version, 486 ''.join(quality_args))) 487 # No need to add .webp. Android can load images fine without them. 488 webp_path = os.path.splitext(png_path)[0] 489 490 cache_hit = os.path.exists(webp_cache_path) 491 if cache_hit: 492 os.link(webp_cache_path, webp_path) 493 else: 494 # We place the generated webp image to webp_path, instead of in the 495 # webp_cache_dir to avoid concurrency issues. 496 args = [cwebp_binary, png_path, '-o', webp_path, '-quiet'] + quality_args 497 subprocess.check_call(args) 498 499 try: 500 os.link(webp_path, webp_cache_path) 501 except OSError: 502 # Because of concurrent run, a webp image may already exists in 503 # webp_cache_path. 504 pass 505 506 os.remove(png_path) 507 original_dir = os.path.dirname(os.path.dirname(png_path)) 508 rename_tuple = (os.path.relpath(png_path, original_dir), 509 os.path.relpath(webp_path, original_dir)) 510 return rename_tuple, cache_hit 511 512 513def _ConvertToWebP(cwebp_binary, png_paths, path_info, webp_cache_dir): 514 cwebp_version = subprocess.check_output([cwebp_binary, '-version']).rstrip() 515 shard_args = [(f, ) for f in png_paths 516 if not _PNG_WEBP_EXCLUSION_PATTERN.match(f)] 517 518 build_utils.MakeDirectory(webp_cache_dir) 519 results = parallel.BulkForkAndCall(_ConvertToWebPSingle, 520 shard_args, 521 cwebp_binary=cwebp_binary, 522 cwebp_version=cwebp_version, 523 webp_cache_dir=webp_cache_dir) 524 total_cache_hits = 0 525 for rename_tuple, cache_hit in results: 526 path_info.RegisterRename(*rename_tuple) 527 total_cache_hits += int(cache_hit) 528 529 logging.debug('png->webp cache: %d/%d', total_cache_hits, len(shard_args)) 530 531 532def _RemoveImageExtensions(directory, path_info): 533 """Remove extensions from image files in the passed directory. 534 535 This reduces binary size but does not affect android's ability to load the 536 images. 537 """ 538 for f in _IterFiles(directory): 539 if (f.endswith('.png') or f.endswith('.webp')) and not f.endswith('.9.png'): 540 path_with_extension = f 541 path_no_extension = os.path.splitext(path_with_extension)[0] 542 if path_no_extension != path_with_extension: 543 shutil.move(path_with_extension, path_no_extension) 544 path_info.RegisterRename( 545 os.path.relpath(path_with_extension, directory), 546 os.path.relpath(path_no_extension, directory)) 547 548 549def _CompileSingleDep(index, dep_subdir, keep_predicate, aapt2_path, 550 partials_dir): 551 unique_name = '{}_{}'.format(index, os.path.basename(dep_subdir)) 552 partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name)) 553 554 compile_command = [ 555 aapt2_path, 556 'compile', 557 # TODO(wnwen): Turn this on once aapt2 forces 9-patch to be crunched. 558 # '--no-crunch', 559 '--dir', 560 dep_subdir, 561 '-o', 562 partial_path 563 ] 564 565 # There are resources targeting API-versions lower than our minapi. For 566 # various reasons it's easier to let aapt2 ignore these than for us to 567 # remove them from our build (e.g. it's from a 3rd party library). 568 build_utils.CheckOutput( 569 compile_command, 570 stderr_filter=lambda output: build_utils.FilterLines( 571 output, r'ignoring configuration .* for (styleable|attribute)')) 572 573 # Filtering these files is expensive, so only apply filters to the partials 574 # that have been explicitly targeted. 575 if keep_predicate: 576 logging.debug('Applying .arsc filtering to %s', dep_subdir) 577 protoresources.StripUnwantedResources(partial_path, keep_predicate) 578 return partial_path 579 580 581def _CreateValuesKeepPredicate(exclusion_rules, dep_subdir): 582 patterns = [ 583 x[1] for x in exclusion_rules 584 if build_utils.MatchesGlob(dep_subdir, [x[0]]) 585 ] 586 if not patterns: 587 return None 588 589 regexes = [re.compile(p) for p in patterns] 590 return lambda x: not any(r.search(x) for r in regexes) 591 592 593def _CompileDeps(aapt2_path, dep_subdirs, dep_subdir_overlay_set, temp_dir, 594 exclusion_rules): 595 partials_dir = os.path.join(temp_dir, 'partials') 596 build_utils.MakeDirectory(partials_dir) 597 598 job_params = [(i, dep_subdir, 599 _CreateValuesKeepPredicate(exclusion_rules, dep_subdir)) 600 for i, dep_subdir in enumerate(dep_subdirs)] 601 602 # Filtering is slow, so ensure jobs with keep_predicate are started first. 603 job_params.sort(key=lambda x: not x[2]) 604 partials = list( 605 parallel.BulkForkAndCall(_CompileSingleDep, 606 job_params, 607 aapt2_path=aapt2_path, 608 partials_dir=partials_dir)) 609 610 partials_cmd = list() 611 for i, partial in enumerate(partials): 612 dep_subdir = job_params[i][1] 613 if dep_subdir in dep_subdir_overlay_set: 614 partials_cmd += ['-R'] 615 partials_cmd += [partial] 616 return partials_cmd 617 618 619def _CreateResourceInfoFile(path_info, info_path, dependencies_res_zips): 620 for zip_file in dependencies_res_zips: 621 zip_info_file_path = zip_file + '.info' 622 if os.path.exists(zip_info_file_path): 623 path_info.MergeInfoFile(zip_info_file_path) 624 path_info.Write(info_path) 625 626 627def _RemoveUnwantedLocalizedStrings(dep_subdirs, options): 628 """Remove localized strings that should not go into the final output. 629 630 Args: 631 dep_subdirs: List of resource dependency directories. 632 options: Command-line options namespace. 633 """ 634 # Collect locale and file paths from the existing subdirs. 635 # The following variable maps Android locale names to 636 # sets of corresponding xml file paths. 637 locale_to_files_map = collections.defaultdict(set) 638 for directory in dep_subdirs: 639 for f in _IterFiles(directory): 640 locale = resource_utils.FindLocaleInStringResourceFilePath(f) 641 if locale: 642 locale_to_files_map[locale].add(f) 643 644 all_locales = set(locale_to_files_map) 645 646 # Set A: wanted locales, either all of them or the 647 # list provided by --locale-allowlist. 648 wanted_locales = all_locales 649 if options.locale_allowlist: 650 wanted_locales = _ToAndroidLocales(options.locale_allowlist) 651 652 # Set B: shared resources locales, which is either set A 653 # or the list provided by --shared-resources-allowlist-locales 654 shared_resources_locales = wanted_locales 655 shared_names_allowlist = set() 656 if options.shared_resources_allowlist_locales: 657 shared_names_allowlist = set( 658 resource_utils.GetRTxtStringResourceNames( 659 options.shared_resources_allowlist)) 660 661 shared_resources_locales = _ToAndroidLocales( 662 options.shared_resources_allowlist_locales) 663 664 # Remove any file that belongs to a locale not covered by 665 # either A or B. 666 removable_locales = (all_locales - wanted_locales - shared_resources_locales) 667 for locale in removable_locales: 668 for path in locale_to_files_map[locale]: 669 os.remove(path) 670 671 # For any locale in B but not in A, only keep the shared 672 # resource strings in each file. 673 for locale in shared_resources_locales - wanted_locales: 674 for path in locale_to_files_map[locale]: 675 resource_utils.FilterAndroidResourceStringsXml( 676 path, lambda x: x in shared_names_allowlist) 677 678 # For any locale in A but not in B, only keep the strings 679 # that are _not_ from shared resources in the file. 680 for locale in wanted_locales - shared_resources_locales: 681 for path in locale_to_files_map[locale]: 682 resource_utils.FilterAndroidResourceStringsXml( 683 path, lambda x: x not in shared_names_allowlist) 684 685 686def _FilterResourceFiles(dep_subdirs, keep_predicate): 687 # Create a function that selects which resource files should be packaged 688 # into the final output. Any file that does not pass the predicate will 689 # be removed below. 690 png_paths = [] 691 for directory in dep_subdirs: 692 for f in _IterFiles(directory): 693 if not keep_predicate(f): 694 os.remove(f) 695 elif f.endswith('.png'): 696 png_paths.append(f) 697 698 return png_paths 699 700 701def _PackageApk(options, build): 702 """Compile and link resources with aapt2. 703 704 Args: 705 options: The command-line options. 706 build: BuildContext object. 707 Returns: 708 The manifest package name for the APK. 709 """ 710 logging.debug('Extracting resource .zips') 711 dep_subdirs = [] 712 dep_subdir_overlay_set = set() 713 for dependency_res_zip in options.dependencies_res_zips: 714 extracted_dep_subdirs = resource_utils.ExtractDeps([dependency_res_zip], 715 build.deps_dir) 716 dep_subdirs += extracted_dep_subdirs 717 if dependency_res_zip in options.dependencies_res_zip_overlays: 718 dep_subdir_overlay_set.update(extracted_dep_subdirs) 719 720 logging.debug('Applying locale transformations') 721 path_info = resource_utils.ResourceInfoFile() 722 _RenameLocaleResourceDirs(dep_subdirs, path_info) 723 724 logging.debug('Applying file-based exclusions') 725 keep_predicate = _CreateKeepPredicate(options.resource_exclusion_regex, 726 options.resource_exclusion_exceptions) 727 png_paths = _FilterResourceFiles(dep_subdirs, keep_predicate) 728 729 if options.locale_allowlist or options.shared_resources_allowlist_locales: 730 logging.debug('Applying locale-based string exclusions') 731 _RemoveUnwantedLocalizedStrings(dep_subdirs, options) 732 733 if png_paths and options.png_to_webp: 734 logging.debug('Converting png->webp') 735 _ConvertToWebP(options.webp_binary, png_paths, path_info, 736 options.webp_cache_dir) 737 logging.debug('Applying drawable transformations') 738 for directory in dep_subdirs: 739 _MoveImagesToNonMdpiFolders(directory, path_info) 740 _RemoveImageExtensions(directory, path_info) 741 742 logging.debug('Running aapt2 compile') 743 exclusion_rules = [x.split(':', 1) for x in options.values_filter_rules] 744 partials = _CompileDeps(options.aapt2_path, dep_subdirs, 745 dep_subdir_overlay_set, build.temp_dir, 746 exclusion_rules) 747 748 link_command = [ 749 options.aapt2_path, 750 'link', 751 '--auto-add-overlay', 752 '--no-version-vectors', 753 '--output-text-symbols', 754 build.r_txt_path, 755 ] 756 757 for j in options.include_resources: 758 link_command += ['-I', j] 759 if options.proguard_file: 760 link_command += ['--proguard', build.proguard_path] 761 link_command += ['--proguard-minimal-keep-rules'] 762 if options.emit_ids_out: 763 link_command += ['--emit-ids', build.emit_ids_path] 764 765 # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib 766 # can be used with recent versions of aapt2. 767 if options.shared_resources: 768 link_command.append('--shared-lib') 769 770 if int(options.min_sdk_version) > 21: 771 link_command.append('--no-xml-namespaces') 772 773 if options.package_id: 774 link_command += [ 775 '--package-id', 776 '0x%02x' % options.package_id, 777 '--allow-reserved-package-id', 778 ] 779 780 fixed_manifest, desired_manifest_package_name, fixed_manifest_package = ( 781 _FixManifest(options, build.temp_dir)) 782 if options.rename_manifest_package: 783 desired_manifest_package_name = options.rename_manifest_package 784 785 link_command += [ 786 '--manifest', fixed_manifest, '--rename-manifest-package', 787 desired_manifest_package_name 788 ] 789 790 if options.package_id is not None: 791 package_id = options.package_id 792 elif options.shared_resources: 793 package_id = 0 794 else: 795 package_id = 0x7f 796 _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, 797 fixed_manifest_package, package_id) 798 link_command += ['--stable-ids', build.stable_ids_path] 799 800 link_command += partials 801 802 # We always create a binary arsc file first, then convert to proto, so flags 803 # such as --shared-lib can be supported. 804 link_command += ['-o', build.arsc_path] 805 806 logging.debug('Starting: aapt2 link') 807 link_proc = subprocess.Popen(link_command) 808 809 # Create .res.info file in parallel. 810 if options.info_path: 811 logging.debug('Creating .res.info file') 812 _CreateResourceInfoFile(path_info, build.info_path, 813 options.dependencies_res_zips) 814 815 exit_code = link_proc.wait() 816 assert exit_code == 0, f'aapt2 link cmd failed with {exit_code=}' 817 logging.debug('Finished: aapt2 link') 818 819 if options.shared_resources: 820 logging.debug('Resolving styleables in R.txt') 821 # Need to resolve references because unused resource removal tool does not 822 # support references in R.txt files. 823 resource_utils.ResolveStyleableReferences(build.r_txt_path) 824 825 if exit_code: 826 raise subprocess.CalledProcessError(exit_code, link_command) 827 828 if options.proguard_file and (options.shared_resources 829 or options.app_as_shared_lib): 830 # Make sure the R class associated with the manifest package does not have 831 # its onResourcesLoaded method obfuscated or removed, so that the framework 832 # can call it in the case where the APK is being loaded as a library. 833 with open(build.proguard_path, 'a') as proguard_file: 834 keep_rule = ''' 835 -keep,allowoptimization class {package}.R {{ 836 public static void onResourcesLoaded(int); 837 }} 838 '''.format(package=desired_manifest_package_name) 839 proguard_file.write(textwrap.dedent(keep_rule)) 840 841 logging.debug('Running aapt2 convert') 842 build_utils.CheckOutput([ 843 options.aapt2_path, 'convert', '--output-format', 'proto', '-o', 844 build.proto_path, build.arsc_path 845 ]) 846 847 # Workaround for b/147674078. This is only needed for WebLayer and does not 848 # affect WebView usage, since WebView does not used dynamic attributes. 849 if options.shared_resources: 850 logging.debug('Hardcoding dynamic attributes') 851 protoresources.HardcodeSharedLibraryDynamicAttributes( 852 build.proto_path, options.is_bundle_module, 853 options.shared_resources_allowlist) 854 855 build_utils.CheckOutput([ 856 options.aapt2_path, 'convert', '--output-format', 'binary', '-o', 857 build.arsc_path, build.proto_path 858 ]) 859 860 # Sanity check that the created resources have the expected package ID. 861 logging.debug('Performing sanity check') 862 _, actual_package_id = resource_utils.ExtractArscPackage( 863 options.aapt2_path, 864 build.arsc_path if options.arsc_path else build.proto_path) 865 # When there are no resources, ExtractArscPackage returns (None, None), in 866 # this case there is no need to check for matching package ID. 867 if actual_package_id is not None and actual_package_id != package_id: 868 raise Exception('Invalid package ID 0x%x (expected 0x%x)' % 869 (actual_package_id, package_id)) 870 871 return desired_manifest_package_name 872 873 874def _CreateStableIdsFile(in_path, out_path, package_name, package_id): 875 """Transforms a file generated by --emit-ids from another package. 876 877 --stable-ids is generally meant to be used by different versions of the same 878 package. To make it work for other packages, we need to transform the package 879 name references to match the package that resources are being generated for. 880 """ 881 if in_path: 882 data = pathlib.Path(in_path).read_text() 883 else: 884 # Force IDs to use 0x01 for the type byte in order to ensure they are 885 # different from IDs generated by other apps. https://crbug.com/1293336 886 data = 'pkg:id/fake_resource_id = 0x7f010000\n' 887 # Replace "pkg:" with correct package name. 888 data = re.sub(r'^.*?:', package_name + ':', data, flags=re.MULTILINE) 889 # Replace "0x7f" with correct package id. 890 data = re.sub(r'0x..', '0x%02x' % package_id, data) 891 pathlib.Path(out_path).write_text(data) 892 893 894def _WriteOutputs(options, build): 895 possible_outputs = [ 896 (options.srcjar_out, build.srcjar_path), 897 (options.r_text_out, build.r_txt_path), 898 (options.arsc_path, build.arsc_path), 899 (options.proto_path, build.proto_path), 900 (options.proguard_file, build.proguard_path), 901 (options.emit_ids_out, build.emit_ids_path), 902 (options.info_path, build.info_path), 903 ] 904 905 for final, temp in possible_outputs: 906 # Write file only if it's changed. 907 if final and not (os.path.exists(final) and filecmp.cmp(final, temp)): 908 shutil.move(temp, final) 909 910 911def _CreateNormalizedManifestForVerification(options): 912 with build_utils.TempDir() as tempdir: 913 fixed_manifest, _, _ = _FixManifest(options, tempdir) 914 with open(fixed_manifest) as f: 915 return manifest_utils.NormalizeManifest( 916 f.read(), options.verification_version_code_offset, 917 options.verification_library_version_offset) 918 919 920def main(args): 921 build_utils.InitLogging('RESOURCE_DEBUG') 922 args = build_utils.ExpandFileArgs(args) 923 options = _ParseArgs(args) 924 925 if options.expected_file: 926 actual_data = _CreateNormalizedManifestForVerification(options) 927 diff_utils.CheckExpectations(actual_data, options) 928 if options.only_verify_expectations: 929 return 930 931 path = options.arsc_path or options.proto_path 932 debug_temp_resources_dir = os.environ.get('TEMP_RESOURCES_DIR') 933 if debug_temp_resources_dir: 934 path = os.path.join(debug_temp_resources_dir, os.path.basename(path)) 935 else: 936 # Use a deterministic temp directory since .pb files embed the absolute 937 # path of resources: crbug.com/939984 938 path = path + '.tmpdir' 939 build_utils.DeleteDirectory(path) 940 941 with resource_utils.BuildContext( 942 temp_dir=path, keep_files=bool(debug_temp_resources_dir)) as build: 943 944 manifest_package_name = _PackageApk(options, build) 945 946 # If --shared-resources-allowlist is used, all the resources listed in the 947 # corresponding R.txt file will be non-final, and an onResourcesLoaded() 948 # will be generated to adjust them at runtime. 949 # 950 # Otherwise, if --shared-resources is used, the all resources will be 951 # non-final, and an onResourcesLoaded() method will be generated too. 952 # 953 # Otherwise, all resources will be final, and no method will be generated. 954 # 955 rjava_build_options = resource_utils.RJavaBuildOptions() 956 if options.shared_resources_allowlist: 957 rjava_build_options.ExportSomeResources( 958 options.shared_resources_allowlist) 959 rjava_build_options.GenerateOnResourcesLoaded() 960 if options.shared_resources: 961 # The final resources will only be used in WebLayer, so hardcode the 962 # package ID to be what WebLayer expects. 963 rjava_build_options.SetFinalPackageId( 964 protoresources.SHARED_LIBRARY_HARDCODED_ID) 965 elif options.shared_resources or options.app_as_shared_lib: 966 rjava_build_options.ExportAllResources() 967 rjava_build_options.GenerateOnResourcesLoaded() 968 969 custom_root_package_name = options.r_java_root_package_name 970 grandparent_custom_package_name = None 971 972 # Always generate an R.java file for the package listed in 973 # AndroidManifest.xml because this is where Android framework looks to find 974 # onResourcesLoaded() for shared library apks. While not actually necessary 975 # for application apks, it also doesn't hurt. 976 apk_package_name = manifest_package_name 977 978 if options.package_name and not options.arsc_package_name: 979 # Feature modules have their own custom root package name and should 980 # inherit from the appropriate base module package. This behaviour should 981 # not be present for test apks with an apk under test. Thus, 982 # arsc_package_name is used as it is only defined for test apks with an 983 # apk under test. 984 custom_root_package_name = options.package_name 985 grandparent_custom_package_name = options.r_java_root_package_name 986 # Feature modules have the same manifest package as the base module but 987 # they should not create an R.java for said manifest package because it 988 # will be created in the base module. 989 apk_package_name = None 990 991 if options.srcjar_out: 992 logging.debug('Creating R.srcjar') 993 resource_utils.CreateRJavaFiles(build.srcjar_dir, apk_package_name, 994 build.r_txt_path, 995 options.extra_res_packages, 996 rjava_build_options, options.srcjar_out, 997 custom_root_package_name, 998 grandparent_custom_package_name) 999 with action_helpers.atomic_output(build.srcjar_path) as f: 1000 zip_helpers.zip_directory(f, build.srcjar_dir) 1001 1002 logging.debug('Copying outputs') 1003 _WriteOutputs(options, build) 1004 1005 if options.depfile: 1006 assert options.srcjar_out, 'Update first output below and remove assert.' 1007 depfile_deps = (options.dependencies_res_zips + 1008 options.dependencies_res_zip_overlays + 1009 options.include_resources) 1010 action_helpers.write_depfile(options.depfile, options.srcjar_out, 1011 depfile_deps) 1012 1013 1014if __name__ == '__main__': 1015 main(sys.argv[1:]) 1016