1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import collections 9import logging 10import os 11import re 12import shutil 13import shlex 14import sys 15import tempfile 16import zipfile 17 18from util import build_utils 19from util import md5_check 20import action_helpers # build_utils adds //build to sys.path. 21import zip_helpers 22 23 24_DEX_XMX = '2G' # Increase this when __final_dex OOMs. 25 26DEFAULT_IGNORE_WARNINGS = ( 27 # Warning: Running R8 version main (build engineering), which cannot be 28 # represented as a semantic version. Using an artificial version newer than 29 # any known version for selecting Proguard configurations embedded under 30 # META-INF/. This means that all rules with a '-upto-' qualifier will be 31 # excluded and all rules with a -from- qualifier will be included. 32 r'Running R8 version main', ) 33 34_IGNORE_SERVICE_ENTRIES = ( 35 # ServiceLoader call is used only for ProtoBuf full (non-lite). 36 # BaseGeneratedExtensionRegistryLite$Loader conflicts with 37 # ChromeGeneratedExtensionRegistryLite$Loader. 38 'META-INF/services/com.google.protobuf.GeneratedExtensionRegistryLoader', 39 # Uses ServiceLoader to find all implementing classes, so multiple are 40 # expected. 41 'META-INF/services/androidx.appsearch.app.AppSearchDocumentClassMap', 42) 43 44INTERFACE_DESUGARING_WARNINGS = (r'default or static interface methods', ) 45 46_SKIPPED_CLASS_FILE_NAMES = ( 47 'module-info.class', # Explicitly skipped by r8/utils/FileUtils#isClassFile 48) 49 50 51def _ParseArgs(args): 52 args = build_utils.ExpandFileArgs(args) 53 parser = argparse.ArgumentParser() 54 55 action_helpers.add_depfile_arg(parser) 56 parser.add_argument('--output', required=True, help='Dex output path.') 57 parser.add_argument( 58 '--class-inputs', 59 action='append', 60 help='GN-list of .jars with .class files.') 61 parser.add_argument( 62 '--class-inputs-filearg', 63 action='append', 64 help='GN-list of .jars with .class files (added to depfile).') 65 parser.add_argument( 66 '--dex-inputs', action='append', help='GN-list of .jars with .dex files.') 67 parser.add_argument( 68 '--dex-inputs-filearg', 69 action='append', 70 help='GN-list of .jars with .dex files (added to depfile).') 71 parser.add_argument( 72 '--incremental-dir', 73 help='Path of directory to put intermediate dex files.') 74 parser.add_argument('--library', 75 action='store_true', 76 help='Allow numerous dex files within output.') 77 parser.add_argument('--r8-jar-path', required=True, help='Path to R8 jar.') 78 parser.add_argument('--skip-custom-d8', 79 action='store_true', 80 help='When rebuilding the CustomD8 jar, this may be ' 81 'necessary to avoid incompatibility with the new r8 ' 82 'jar.') 83 parser.add_argument('--custom-d8-jar-path', 84 required=True, 85 help='Path to our customized d8 jar.') 86 parser.add_argument('--desugar-dependencies', 87 help='Path to store desugar dependencies.') 88 parser.add_argument('--desugar', action='store_true') 89 parser.add_argument( 90 '--bootclasspath', 91 action='append', 92 help='GN-list of bootclasspath. Needed for --desugar') 93 parser.add_argument('--show-desugar-default-interface-warnings', 94 action='store_true', 95 help='Enable desugaring warnings.') 96 parser.add_argument( 97 '--classpath', 98 action='append', 99 help='GN-list of full classpath. Needed for --desugar') 100 parser.add_argument('--release', 101 action='store_true', 102 help='Run D8 in release mode.') 103 parser.add_argument( 104 '--min-api', help='Minimum Android API level compatibility.') 105 parser.add_argument('--force-enable-assertions', 106 action='store_true', 107 help='Forcefully enable javac generated assertion code.') 108 parser.add_argument('--assertion-handler', 109 help='The class name of the assertion handler class.') 110 parser.add_argument('--warnings-as-errors', 111 action='store_true', 112 help='Treat all warnings as errors.') 113 parser.add_argument('--dump-inputs', 114 action='store_true', 115 help='Use when filing D8 bugs to capture inputs.' 116 ' Stores inputs to d8inputs.zip') 117 options = parser.parse_args(args) 118 119 if options.force_enable_assertions and options.assertion_handler: 120 parser.error('Cannot use both --force-enable-assertions and ' 121 '--assertion-handler') 122 123 options.class_inputs = action_helpers.parse_gn_list(options.class_inputs) 124 options.class_inputs_filearg = action_helpers.parse_gn_list( 125 options.class_inputs_filearg) 126 options.bootclasspath = action_helpers.parse_gn_list(options.bootclasspath) 127 options.classpath = action_helpers.parse_gn_list(options.classpath) 128 options.dex_inputs = action_helpers.parse_gn_list(options.dex_inputs) 129 options.dex_inputs_filearg = action_helpers.parse_gn_list( 130 options.dex_inputs_filearg) 131 132 return options 133 134 135def CreateStderrFilter(filters): 136 def filter_stderr(output): 137 # Set this when debugging R8 output. 138 if os.environ.get('R8_SHOW_ALL_OUTPUT', '0') != '0': 139 return output 140 141 # All missing definitions are logged as a single warning, but start on a 142 # new line like "Missing class ...". 143 warnings = re.split(r'^(?=Warning|Error|Missing (?:class|field|method))', 144 output, 145 flags=re.MULTILINE) 146 preamble, *warnings = warnings 147 148 combined_pattern = '|'.join(filters) 149 preamble = build_utils.FilterLines(preamble, combined_pattern) 150 151 compiled_re = re.compile(combined_pattern, re.DOTALL) 152 warnings = [w for w in warnings if not compiled_re.search(w)] 153 154 return preamble + ''.join(warnings) 155 156 return filter_stderr 157 158 159def _RunD8(dex_cmd, input_paths, output_path, warnings_as_errors, 160 show_desugar_default_interface_warnings): 161 dex_cmd = dex_cmd + ['--output', output_path] + input_paths 162 163 # Missing deps can happen for prebuilts that are missing transitive deps 164 # and have set enable_bytecode_checks=false. 165 filters = list(DEFAULT_IGNORE_WARNINGS) 166 if not show_desugar_default_interface_warnings: 167 filters += INTERFACE_DESUGARING_WARNINGS 168 169 stderr_filter = CreateStderrFilter(filters) 170 171 is_debug = logging.getLogger().isEnabledFor(logging.DEBUG) 172 173 # Avoid deleting the flag file when DEX_DEBUG is set in case the flag file 174 # needs to be examined after the build. 175 with tempfile.NamedTemporaryFile(mode='w', delete=not is_debug) as flag_file: 176 # Chosen arbitrarily. Needed to avoid command-line length limits. 177 MAX_ARGS = 50 178 orig_dex_cmd = dex_cmd 179 if len(dex_cmd) > MAX_ARGS: 180 # Add all flags to D8 (anything after the first --) as well as all 181 # positional args at the end to the flag file. 182 for idx, cmd in enumerate(dex_cmd): 183 if cmd.startswith('--'): 184 flag_file.write('\n'.join(dex_cmd[idx:])) 185 flag_file.flush() 186 dex_cmd = dex_cmd[:idx] 187 dex_cmd.append('@' + flag_file.name) 188 break 189 190 # stdout sometimes spams with things like: 191 # Stripped invalid locals information from 1 method. 192 try: 193 build_utils.CheckOutput(dex_cmd, 194 stderr_filter=stderr_filter, 195 fail_on_output=warnings_as_errors) 196 except Exception as e: 197 if isinstance(e, build_utils.CalledProcessError): 198 output = e.output # pylint: disable=no-member 199 if "global synthetic for 'Record desugaring'" in output: 200 sys.stderr.write('Java records are not supported.\n') 201 sys.stderr.write( 202 'See https://chromium.googlesource.com/chromium/src/+/' + 203 'main/styleguide/java/java.md#Records\n') 204 sys.exit(1) 205 if orig_dex_cmd is not dex_cmd: 206 sys.stderr.write('Full command: ' + shlex.join(orig_dex_cmd) + '\n') 207 raise 208 209 210def _ZipAligned(dex_files, output_path, services_map): 211 """Creates a .dex.jar with 4-byte aligned files. 212 213 Args: 214 dex_files: List of dex files. 215 output_path: The output file in which to write the zip. 216 services_map: map of path->data for META-INF/services 217 """ 218 with zipfile.ZipFile(output_path, 'w') as z: 219 for i, dex_file in enumerate(dex_files): 220 name = 'classes{}.dex'.format(i + 1 if i > 0 else '') 221 zip_helpers.add_to_zip_hermetic(z, name, src_path=dex_file, alignment=4) 222 for path, data in sorted(services_map.items()): 223 zip_helpers.add_to_zip_hermetic(z, path, data=data, alignment=4) 224 225 226def _CreateServicesMap(service_jars): 227 ret = {} 228 origins = {} 229 for jar_path in service_jars: 230 with zipfile.ZipFile(jar_path, 'r') as z: 231 for n in z.namelist(): 232 if n.startswith('META-INF/services/') and not n.endswith('/'): 233 if n in _IGNORE_SERVICE_ENTRIES: 234 continue 235 data = z.read(n) 236 if ret.get(n, data) == data: 237 ret[n] = data 238 origins[n] = jar_path 239 else: 240 # We should arguably just concat the files here, but Chrome's own 241 # uses (via ServiceLoaderUtil) all assume only one entry. 242 raise Exception(f"""\ 243Conflicting contents for: {n} 244{origins[n]}: 245{ret[n]} 246{jar_path}: 247{data} 248 249If this entry can be safely ignored (because the ServiceLoader.load() call is \ 250never hit), update _IGNORE_SERVICE_ENTRIES in dex.py. 251""") 252 return ret 253 254 255def _CreateFinalDex(d8_inputs, 256 output, 257 tmp_dir, 258 dex_cmd, 259 options=None, 260 service_jars=None): 261 tmp_dex_output = os.path.join(tmp_dir, 'tmp_dex_output.zip') 262 needs_dexing = not all(f.endswith('.dex') for f in d8_inputs) 263 needs_dexmerge = output.endswith('.dex') or not (options and options.library) 264 services_map = _CreateServicesMap(service_jars or []) 265 if needs_dexing or needs_dexmerge: 266 tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir') 267 os.mkdir(tmp_dex_dir) 268 269 _RunD8(dex_cmd, d8_inputs, tmp_dex_dir, 270 (not options or options.warnings_as_errors), 271 (options and options.show_desugar_default_interface_warnings)) 272 logging.debug('Performed dex merging') 273 274 dex_files = [os.path.join(tmp_dex_dir, f) for f in os.listdir(tmp_dex_dir)] 275 276 if output.endswith('.dex'): 277 if len(dex_files) > 1: 278 raise Exception('%d files created, expected 1' % len(dex_files)) 279 tmp_dex_output = dex_files[0] 280 else: 281 _ZipAligned(sorted(dex_files), tmp_dex_output, services_map) 282 else: 283 # Skip dexmerger. Just put all incrementals into the .jar individually. 284 _ZipAligned(sorted(d8_inputs), tmp_dex_output, services_map) 285 logging.debug('Quick-zipped %d files', len(d8_inputs)) 286 287 # The dex file is complete and can be moved out of tmp_dir. 288 shutil.move(tmp_dex_output, output) 289 290 291def _IntermediateDexFilePathsFromInputJars(class_inputs, incremental_dir): 292 """Returns list of intermediate dex file paths, .jar files with services.""" 293 dex_files = [] 294 service_jars = set() 295 for jar in class_inputs: 296 with zipfile.ZipFile(jar, 'r') as z: 297 for subpath in z.namelist(): 298 if _IsClassFile(subpath): 299 subpath = subpath[:-5] + 'dex' 300 dex_files.append(os.path.join(incremental_dir, subpath)) 301 elif subpath.startswith('META-INF/services/'): 302 service_jars.add(jar) 303 return dex_files, sorted(service_jars) 304 305 306def _DeleteStaleIncrementalDexFiles(dex_dir, dex_files): 307 """Deletes intermediate .dex files that are no longer needed.""" 308 all_files = build_utils.FindInDirectory(dex_dir) 309 desired_files = set(dex_files) 310 for path in all_files: 311 if path not in desired_files: 312 os.unlink(path) 313 314 315def _ParseDesugarDeps(desugar_dependencies_file): 316 # pylint: disable=line-too-long 317 """Returns a dict of dependent/dependency mapping parsed from the file. 318 319 Example file format: 320 $ tail out/Debug/gen/base/base_java__dex.desugardeps 321 org/chromium/base/task/SingleThreadTaskRunnerImpl.class 322 <- org/chromium/base/task/SingleThreadTaskRunner.class 323 <- org/chromium/base/task/TaskRunnerImpl.class 324 org/chromium/base/task/TaskRunnerImpl.class 325 <- org/chromium/base/task/TaskRunner.class 326 org/chromium/base/task/TaskRunnerImplJni$1.class 327 <- obj/base/jni_java.turbine.jar:org/jni_zero/JniStaticTestMocker.class 328 org/chromium/base/task/TaskRunnerImplJni.class 329 <- org/chromium/base/task/TaskRunnerImpl$Natives.class 330 """ 331 # pylint: enable=line-too-long 332 dependents_from_dependency = collections.defaultdict(set) 333 if desugar_dependencies_file and os.path.exists(desugar_dependencies_file): 334 with open(desugar_dependencies_file, 'r') as f: 335 dependent = None 336 for line in f: 337 line = line.rstrip() 338 if line.startswith(' <- '): 339 dependency = line[len(' <- '):] 340 # Note that this is a reversed mapping from the one in CustomD8.java. 341 dependents_from_dependency[dependency].add(dependent) 342 else: 343 dependent = line 344 return dependents_from_dependency 345 346 347def _ComputeRequiredDesugarClasses(changes, desugar_dependencies_file, 348 class_inputs, classpath): 349 dependents_from_dependency = _ParseDesugarDeps(desugar_dependencies_file) 350 required_classes = set() 351 # Gather classes that need to be re-desugared from changes in the classpath. 352 for jar in classpath: 353 for subpath in changes.IterChangedSubpaths(jar): 354 dependency = '{}:{}'.format(jar, subpath) 355 required_classes.update(dependents_from_dependency[dependency]) 356 357 for jar in class_inputs: 358 for subpath in changes.IterChangedSubpaths(jar): 359 required_classes.update(dependents_from_dependency[subpath]) 360 361 return required_classes 362 363 364def _IsClassFile(path): 365 if os.path.basename(path) in _SKIPPED_CLASS_FILE_NAMES: 366 return False 367 return path.endswith('.class') 368 369 370def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set): 371 classes_list = [] 372 for jar in class_inputs: 373 if changes: 374 changed_class_list = (set(changes.IterChangedSubpaths(jar)) 375 | required_classes_set) 376 predicate = lambda x: x in changed_class_list and _IsClassFile(x) 377 else: 378 predicate = _IsClassFile 379 380 classes_list.extend( 381 build_utils.ExtractAll(jar, path=tmp_dir, predicate=predicate)) 382 return classes_list 383 384 385def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd): 386 # Create temporary directory for classes to be extracted to. 387 tmp_extract_dir = os.path.join(tmp_dir, 'tmp_extract_dir') 388 os.mkdir(tmp_extract_dir) 389 390 # Do a full rebuild when changes occur in non-input files. 391 allowed_changed = set(options.class_inputs) 392 allowed_changed.update(options.dex_inputs) 393 allowed_changed.update(options.classpath) 394 strings_changed = changes.HasStringChanges() 395 non_direct_input_changed = next( 396 (p for p in changes.IterChangedPaths() if p not in allowed_changed), None) 397 398 if strings_changed or non_direct_input_changed: 399 logging.debug('Full dex required: strings_changed=%s path_changed=%s', 400 strings_changed, non_direct_input_changed) 401 changes = None 402 403 if changes is None: 404 required_desugar_classes_set = set() 405 else: 406 required_desugar_classes_set = _ComputeRequiredDesugarClasses( 407 changes, options.desugar_dependencies, options.class_inputs, 408 options.classpath) 409 logging.debug('Class files needing re-desugar: %d', 410 len(required_desugar_classes_set)) 411 class_files = _ExtractClassFiles(changes, tmp_extract_dir, 412 options.class_inputs, 413 required_desugar_classes_set) 414 logging.debug('Extracted class files: %d', len(class_files)) 415 416 # If the only change is deleting a file, class_files will be empty. 417 if class_files: 418 # Dex necessary classes into intermediate dex files. 419 dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file'] 420 if options.desugar_dependencies and not options.skip_custom_d8: 421 # Adding os.sep to remove the entire prefix. 422 dex_cmd += ['--file-tmp-prefix', tmp_extract_dir + os.sep] 423 if changes is None and os.path.exists(options.desugar_dependencies): 424 # Since incremental dexing only ever adds to the desugar_dependencies 425 # file, whenever full dexes are required the .desugardeps files need to 426 # be manually removed. 427 os.unlink(options.desugar_dependencies) 428 _RunD8(dex_cmd, class_files, options.incremental_dir, 429 options.warnings_as_errors, 430 options.show_desugar_default_interface_warnings) 431 logging.debug('Dexed class files.') 432 433 434def _OnStaleMd5(changes, options, final_dex_inputs, service_jars, dex_cmd): 435 logging.debug('_OnStaleMd5') 436 with build_utils.TempDir() as tmp_dir: 437 if options.incremental_dir: 438 # Create directory for all intermediate dex files. 439 if not os.path.exists(options.incremental_dir): 440 os.makedirs(options.incremental_dir) 441 442 _DeleteStaleIncrementalDexFiles(options.incremental_dir, final_dex_inputs) 443 logging.debug('Stale files deleted') 444 _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd) 445 446 _CreateFinalDex(final_dex_inputs, 447 options.output, 448 tmp_dir, 449 dex_cmd, 450 options=options, 451 service_jars=service_jars) 452 453 454def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar, 455 min_api): 456 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) + [ 457 '-cp', 458 r8_jar_path, 459 'com.android.tools.r8.D8', 460 '--min-api', 461 min_api, 462 ] 463 with build_utils.TempDir() as tmp_dir: 464 _CreateFinalDex(src_paths, 465 dest_dex_jar, 466 tmp_dir, 467 dex_cmd, 468 service_jars=src_paths) 469 470 471def main(args): 472 build_utils.InitLogging('DEX_DEBUG') 473 options = _ParseArgs(args) 474 475 options.class_inputs += options.class_inputs_filearg 476 options.dex_inputs += options.dex_inputs_filearg 477 478 input_paths = ([ 479 build_utils.JAVA_PATH_FOR_INPUTS, options.r8_jar_path, 480 options.custom_d8_jar_path 481 ] + options.class_inputs + options.dex_inputs) 482 483 depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg 484 485 output_paths = [options.output] 486 487 track_subpaths_allowlist = [] 488 if options.incremental_dir: 489 final_dex_inputs, service_jars = _IntermediateDexFilePathsFromInputJars( 490 options.class_inputs, options.incremental_dir) 491 output_paths += final_dex_inputs 492 track_subpaths_allowlist += options.class_inputs 493 else: 494 final_dex_inputs = list(options.class_inputs) 495 service_jars = final_dex_inputs 496 service_jars += options.dex_inputs 497 final_dex_inputs += options.dex_inputs 498 499 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) 500 501 if options.dump_inputs: 502 dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip'] 503 504 if not options.skip_custom_d8: 505 dex_cmd += [ 506 '-cp', 507 '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path), 508 'org.chromium.build.CustomD8', 509 ] 510 else: 511 dex_cmd += [ 512 '-cp', 513 options.r8_jar_path, 514 'com.android.tools.r8.D8', 515 ] 516 517 if options.release: 518 dex_cmd += ['--release'] 519 if options.min_api: 520 dex_cmd += ['--min-api', options.min_api] 521 522 if not options.desugar: 523 dex_cmd += ['--no-desugaring'] 524 elif options.classpath: 525 # The classpath is used by D8 to for interface desugaring. 526 if options.desugar_dependencies and not options.skip_custom_d8: 527 dex_cmd += ['--desugar-dependencies', options.desugar_dependencies] 528 if track_subpaths_allowlist: 529 track_subpaths_allowlist += options.classpath 530 depfile_deps += options.classpath 531 input_paths += options.classpath 532 # Still pass the entire classpath in case a new dependency is needed by 533 # desugar, so that desugar_dependencies will be updated for the next build. 534 for path in options.classpath: 535 dex_cmd += ['--classpath', path] 536 537 if options.classpath: 538 dex_cmd += ['--lib', build_utils.JAVA_HOME] 539 for path in options.bootclasspath: 540 dex_cmd += ['--lib', path] 541 depfile_deps += options.bootclasspath 542 input_paths += options.bootclasspath 543 544 545 if options.assertion_handler: 546 dex_cmd += ['--force-assertions-handler:' + options.assertion_handler] 547 if options.force_enable_assertions: 548 dex_cmd += ['--force-enable-assertions'] 549 550 # The changes feature from md5_check allows us to only re-dex the class files 551 # that have changed and the class files that need to be re-desugared by D8. 552 md5_check.CallAndWriteDepfileIfStale( 553 lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, 554 service_jars, dex_cmd), 555 options, 556 input_paths=input_paths, 557 input_strings=dex_cmd + [str(bool(options.incremental_dir))], 558 output_paths=output_paths, 559 pass_changes=True, 560 track_subpaths_allowlist=track_subpaths_allowlist, 561 depfile_deps=depfile_deps) 562 563 564if __name__ == '__main__': 565 sys.exit(main(sys.argv[1:])) 566