xref: /aosp_15_r20/external/angle/build/android/gyp/dex.py (revision 8975f5c5ed3d1c378011245431ada316dfb6f244)
1#!/usr/bin/env python3
2#
3# Copyright 2013 The Chromium Authors
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import argparse
8import collections
9import logging
10import os
11import re
12import shutil
13import shlex
14import sys
15import tempfile
16import zipfile
17
18from util import build_utils
19from util import md5_check
20import action_helpers  # build_utils adds //build to sys.path.
21import zip_helpers
22
23
24_DEX_XMX = '2G'  # Increase this when __final_dex OOMs.
25
26DEFAULT_IGNORE_WARNINGS = (
27    # Warning: Running R8 version main (build engineering), which cannot be
28    # represented as a semantic version. Using an artificial version newer than
29    # any known version for selecting Proguard configurations embedded under
30    # META-INF/. This means that all rules with a '-upto-' qualifier will be
31    # excluded and all rules with a -from- qualifier will be included.
32    r'Running R8 version main', )
33
34_IGNORE_SERVICE_ENTRIES = (
35    # ServiceLoader call is used only for ProtoBuf full (non-lite).
36    # BaseGeneratedExtensionRegistryLite$Loader conflicts with
37    # ChromeGeneratedExtensionRegistryLite$Loader.
38    'META-INF/services/com.google.protobuf.GeneratedExtensionRegistryLoader',
39    # Uses ServiceLoader to find all implementing classes, so multiple are
40    # expected.
41    'META-INF/services/androidx.appsearch.app.AppSearchDocumentClassMap',
42)
43
44INTERFACE_DESUGARING_WARNINGS = (r'default or static interface methods', )
45
46_SKIPPED_CLASS_FILE_NAMES = (
47    'module-info.class',  # Explicitly skipped by r8/utils/FileUtils#isClassFile
48)
49
50
51def _ParseArgs(args):
52  args = build_utils.ExpandFileArgs(args)
53  parser = argparse.ArgumentParser()
54
55  action_helpers.add_depfile_arg(parser)
56  parser.add_argument('--output', required=True, help='Dex output path.')
57  parser.add_argument(
58      '--class-inputs',
59      action='append',
60      help='GN-list of .jars with .class files.')
61  parser.add_argument(
62      '--class-inputs-filearg',
63      action='append',
64      help='GN-list of .jars with .class files (added to depfile).')
65  parser.add_argument(
66      '--dex-inputs', action='append', help='GN-list of .jars with .dex files.')
67  parser.add_argument(
68      '--dex-inputs-filearg',
69      action='append',
70      help='GN-list of .jars with .dex files (added to depfile).')
71  parser.add_argument(
72      '--incremental-dir',
73      help='Path of directory to put intermediate dex files.')
74  parser.add_argument('--library',
75                      action='store_true',
76                      help='Allow numerous dex files within output.')
77  parser.add_argument('--r8-jar-path', required=True, help='Path to R8 jar.')
78  parser.add_argument('--skip-custom-d8',
79                      action='store_true',
80                      help='When rebuilding the CustomD8 jar, this may be '
81                      'necessary to avoid incompatibility with the new r8 '
82                      'jar.')
83  parser.add_argument('--custom-d8-jar-path',
84                      required=True,
85                      help='Path to our customized d8 jar.')
86  parser.add_argument('--desugar-dependencies',
87                      help='Path to store desugar dependencies.')
88  parser.add_argument('--desugar', action='store_true')
89  parser.add_argument(
90      '--bootclasspath',
91      action='append',
92      help='GN-list of bootclasspath. Needed for --desugar')
93  parser.add_argument('--show-desugar-default-interface-warnings',
94                      action='store_true',
95                      help='Enable desugaring warnings.')
96  parser.add_argument(
97      '--classpath',
98      action='append',
99      help='GN-list of full classpath. Needed for --desugar')
100  parser.add_argument('--release',
101                      action='store_true',
102                      help='Run D8 in release mode.')
103  parser.add_argument(
104      '--min-api', help='Minimum Android API level compatibility.')
105  parser.add_argument('--force-enable-assertions',
106                      action='store_true',
107                      help='Forcefully enable javac generated assertion code.')
108  parser.add_argument('--assertion-handler',
109                      help='The class name of the assertion handler class.')
110  parser.add_argument('--warnings-as-errors',
111                      action='store_true',
112                      help='Treat all warnings as errors.')
113  parser.add_argument('--dump-inputs',
114                      action='store_true',
115                      help='Use when filing D8 bugs to capture inputs.'
116                      ' Stores inputs to d8inputs.zip')
117  options = parser.parse_args(args)
118
119  if options.force_enable_assertions and options.assertion_handler:
120    parser.error('Cannot use both --force-enable-assertions and '
121                 '--assertion-handler')
122
123  options.class_inputs = action_helpers.parse_gn_list(options.class_inputs)
124  options.class_inputs_filearg = action_helpers.parse_gn_list(
125      options.class_inputs_filearg)
126  options.bootclasspath = action_helpers.parse_gn_list(options.bootclasspath)
127  options.classpath = action_helpers.parse_gn_list(options.classpath)
128  options.dex_inputs = action_helpers.parse_gn_list(options.dex_inputs)
129  options.dex_inputs_filearg = action_helpers.parse_gn_list(
130      options.dex_inputs_filearg)
131
132  return options
133
134
135def CreateStderrFilter(filters):
136  def filter_stderr(output):
137    # Set this when debugging R8 output.
138    if os.environ.get('R8_SHOW_ALL_OUTPUT', '0') != '0':
139      return output
140
141    # All missing definitions are logged as a single warning, but start on a
142    # new line like "Missing class ...".
143    warnings = re.split(r'^(?=Warning|Error|Missing (?:class|field|method))',
144                        output,
145                        flags=re.MULTILINE)
146    preamble, *warnings = warnings
147
148    combined_pattern = '|'.join(filters)
149    preamble = build_utils.FilterLines(preamble, combined_pattern)
150
151    compiled_re = re.compile(combined_pattern, re.DOTALL)
152    warnings = [w for w in warnings if not compiled_re.search(w)]
153
154    return preamble + ''.join(warnings)
155
156  return filter_stderr
157
158
159def _RunD8(dex_cmd, input_paths, output_path, warnings_as_errors,
160           show_desugar_default_interface_warnings):
161  dex_cmd = dex_cmd + ['--output', output_path] + input_paths
162
163  # Missing deps can happen for prebuilts that are missing transitive deps
164  # and have set enable_bytecode_checks=false.
165  filters = list(DEFAULT_IGNORE_WARNINGS)
166  if not show_desugar_default_interface_warnings:
167    filters += INTERFACE_DESUGARING_WARNINGS
168
169  stderr_filter = CreateStderrFilter(filters)
170
171  is_debug = logging.getLogger().isEnabledFor(logging.DEBUG)
172
173  # Avoid deleting the flag file when DEX_DEBUG is set in case the flag file
174  # needs to be examined after the build.
175  with tempfile.NamedTemporaryFile(mode='w', delete=not is_debug) as flag_file:
176    # Chosen arbitrarily. Needed to avoid command-line length limits.
177    MAX_ARGS = 50
178    orig_dex_cmd = dex_cmd
179    if len(dex_cmd) > MAX_ARGS:
180      # Add all flags to D8 (anything after the first --) as well as all
181      # positional args at the end to the flag file.
182      for idx, cmd in enumerate(dex_cmd):
183        if cmd.startswith('--'):
184          flag_file.write('\n'.join(dex_cmd[idx:]))
185          flag_file.flush()
186          dex_cmd = dex_cmd[:idx]
187          dex_cmd.append('@' + flag_file.name)
188          break
189
190    # stdout sometimes spams with things like:
191    # Stripped invalid locals information from 1 method.
192    try:
193      build_utils.CheckOutput(dex_cmd,
194                              stderr_filter=stderr_filter,
195                              fail_on_output=warnings_as_errors)
196    except Exception as e:
197      if isinstance(e, build_utils.CalledProcessError):
198        output = e.output  # pylint: disable=no-member
199        if "global synthetic for 'Record desugaring'" in output:
200          sys.stderr.write('Java records are not supported.\n')
201          sys.stderr.write(
202              'See https://chromium.googlesource.com/chromium/src/+/' +
203              'main/styleguide/java/java.md#Records\n')
204          sys.exit(1)
205      if orig_dex_cmd is not dex_cmd:
206        sys.stderr.write('Full command: ' + shlex.join(orig_dex_cmd) + '\n')
207      raise
208
209
210def _ZipAligned(dex_files, output_path, services_map):
211  """Creates a .dex.jar with 4-byte aligned files.
212
213  Args:
214    dex_files: List of dex files.
215    output_path: The output file in which to write the zip.
216    services_map: map of path->data for META-INF/services
217  """
218  with zipfile.ZipFile(output_path, 'w') as z:
219    for i, dex_file in enumerate(dex_files):
220      name = 'classes{}.dex'.format(i + 1 if i > 0 else '')
221      zip_helpers.add_to_zip_hermetic(z, name, src_path=dex_file, alignment=4)
222    for path, data in sorted(services_map.items()):
223      zip_helpers.add_to_zip_hermetic(z, path, data=data, alignment=4)
224
225
226def _CreateServicesMap(service_jars):
227  ret = {}
228  origins = {}
229  for jar_path in service_jars:
230    with zipfile.ZipFile(jar_path, 'r') as z:
231      for n in z.namelist():
232        if n.startswith('META-INF/services/') and not n.endswith('/'):
233          if n in _IGNORE_SERVICE_ENTRIES:
234            continue
235          data = z.read(n)
236          if ret.get(n, data) == data:
237            ret[n] = data
238            origins[n] = jar_path
239          else:
240            # We should arguably just concat the files here, but Chrome's own
241            # uses (via ServiceLoaderUtil) all assume only one entry.
242            raise Exception(f"""\
243Conflicting contents for: {n}
244{origins[n]}:
245{ret[n]}
246{jar_path}:
247{data}
248
249If this entry can be safely ignored (because the ServiceLoader.load() call is \
250never hit), update _IGNORE_SERVICE_ENTRIES in dex.py.
251""")
252  return ret
253
254
255def _CreateFinalDex(d8_inputs,
256                    output,
257                    tmp_dir,
258                    dex_cmd,
259                    options=None,
260                    service_jars=None):
261  tmp_dex_output = os.path.join(tmp_dir, 'tmp_dex_output.zip')
262  needs_dexing = not all(f.endswith('.dex') for f in d8_inputs)
263  needs_dexmerge = output.endswith('.dex') or not (options and options.library)
264  services_map = _CreateServicesMap(service_jars or [])
265  if needs_dexing or needs_dexmerge:
266    tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir')
267    os.mkdir(tmp_dex_dir)
268
269    _RunD8(dex_cmd, d8_inputs, tmp_dex_dir,
270           (not options or options.warnings_as_errors),
271           (options and options.show_desugar_default_interface_warnings))
272    logging.debug('Performed dex merging')
273
274    dex_files = [os.path.join(tmp_dex_dir, f) for f in os.listdir(tmp_dex_dir)]
275
276    if output.endswith('.dex'):
277      if len(dex_files) > 1:
278        raise Exception('%d files created, expected 1' % len(dex_files))
279      tmp_dex_output = dex_files[0]
280    else:
281      _ZipAligned(sorted(dex_files), tmp_dex_output, services_map)
282  else:
283    # Skip dexmerger. Just put all incrementals into the .jar individually.
284    _ZipAligned(sorted(d8_inputs), tmp_dex_output, services_map)
285    logging.debug('Quick-zipped %d files', len(d8_inputs))
286
287  # The dex file is complete and can be moved out of tmp_dir.
288  shutil.move(tmp_dex_output, output)
289
290
291def _IntermediateDexFilePathsFromInputJars(class_inputs, incremental_dir):
292  """Returns list of intermediate dex file paths, .jar files with services."""
293  dex_files = []
294  service_jars = set()
295  for jar in class_inputs:
296    with zipfile.ZipFile(jar, 'r') as z:
297      for subpath in z.namelist():
298        if _IsClassFile(subpath):
299          subpath = subpath[:-5] + 'dex'
300          dex_files.append(os.path.join(incremental_dir, subpath))
301        elif subpath.startswith('META-INF/services/'):
302          service_jars.add(jar)
303  return dex_files, sorted(service_jars)
304
305
306def _DeleteStaleIncrementalDexFiles(dex_dir, dex_files):
307  """Deletes intermediate .dex files that are no longer needed."""
308  all_files = build_utils.FindInDirectory(dex_dir)
309  desired_files = set(dex_files)
310  for path in all_files:
311    if path not in desired_files:
312      os.unlink(path)
313
314
315def _ParseDesugarDeps(desugar_dependencies_file):
316  # pylint: disable=line-too-long
317  """Returns a dict of dependent/dependency mapping parsed from the file.
318
319  Example file format:
320  $ tail out/Debug/gen/base/base_java__dex.desugardeps
321  org/chromium/base/task/SingleThreadTaskRunnerImpl.class
322    <-  org/chromium/base/task/SingleThreadTaskRunner.class
323    <-  org/chromium/base/task/TaskRunnerImpl.class
324  org/chromium/base/task/TaskRunnerImpl.class
325    <-  org/chromium/base/task/TaskRunner.class
326  org/chromium/base/task/TaskRunnerImplJni$1.class
327    <-  obj/base/jni_java.turbine.jar:org/jni_zero/JniStaticTestMocker.class
328  org/chromium/base/task/TaskRunnerImplJni.class
329    <-  org/chromium/base/task/TaskRunnerImpl$Natives.class
330  """
331  # pylint: enable=line-too-long
332  dependents_from_dependency = collections.defaultdict(set)
333  if desugar_dependencies_file and os.path.exists(desugar_dependencies_file):
334    with open(desugar_dependencies_file, 'r') as f:
335      dependent = None
336      for line in f:
337        line = line.rstrip()
338        if line.startswith('  <-  '):
339          dependency = line[len('  <-  '):]
340          # Note that this is a reversed mapping from the one in CustomD8.java.
341          dependents_from_dependency[dependency].add(dependent)
342        else:
343          dependent = line
344  return dependents_from_dependency
345
346
347def _ComputeRequiredDesugarClasses(changes, desugar_dependencies_file,
348                                   class_inputs, classpath):
349  dependents_from_dependency = _ParseDesugarDeps(desugar_dependencies_file)
350  required_classes = set()
351  # Gather classes that need to be re-desugared from changes in the classpath.
352  for jar in classpath:
353    for subpath in changes.IterChangedSubpaths(jar):
354      dependency = '{}:{}'.format(jar, subpath)
355      required_classes.update(dependents_from_dependency[dependency])
356
357  for jar in class_inputs:
358    for subpath in changes.IterChangedSubpaths(jar):
359      required_classes.update(dependents_from_dependency[subpath])
360
361  return required_classes
362
363
364def _IsClassFile(path):
365  if os.path.basename(path) in _SKIPPED_CLASS_FILE_NAMES:
366    return False
367  return path.endswith('.class')
368
369
370def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set):
371  classes_list = []
372  for jar in class_inputs:
373    if changes:
374      changed_class_list = (set(changes.IterChangedSubpaths(jar))
375                            | required_classes_set)
376      predicate = lambda x: x in changed_class_list and _IsClassFile(x)
377    else:
378      predicate = _IsClassFile
379
380    classes_list.extend(
381        build_utils.ExtractAll(jar, path=tmp_dir, predicate=predicate))
382  return classes_list
383
384
385def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd):
386  # Create temporary directory for classes to be extracted to.
387  tmp_extract_dir = os.path.join(tmp_dir, 'tmp_extract_dir')
388  os.mkdir(tmp_extract_dir)
389
390  # Do a full rebuild when changes occur in non-input files.
391  allowed_changed = set(options.class_inputs)
392  allowed_changed.update(options.dex_inputs)
393  allowed_changed.update(options.classpath)
394  strings_changed = changes.HasStringChanges()
395  non_direct_input_changed = next(
396      (p for p in changes.IterChangedPaths() if p not in allowed_changed), None)
397
398  if strings_changed or non_direct_input_changed:
399    logging.debug('Full dex required: strings_changed=%s path_changed=%s',
400                  strings_changed, non_direct_input_changed)
401    changes = None
402
403  if changes is None:
404    required_desugar_classes_set = set()
405  else:
406    required_desugar_classes_set = _ComputeRequiredDesugarClasses(
407        changes, options.desugar_dependencies, options.class_inputs,
408        options.classpath)
409    logging.debug('Class files needing re-desugar: %d',
410                  len(required_desugar_classes_set))
411  class_files = _ExtractClassFiles(changes, tmp_extract_dir,
412                                   options.class_inputs,
413                                   required_desugar_classes_set)
414  logging.debug('Extracted class files: %d', len(class_files))
415
416  # If the only change is deleting a file, class_files will be empty.
417  if class_files:
418    # Dex necessary classes into intermediate dex files.
419    dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file']
420    if options.desugar_dependencies and not options.skip_custom_d8:
421      # Adding os.sep to remove the entire prefix.
422      dex_cmd += ['--file-tmp-prefix', tmp_extract_dir + os.sep]
423      if changes is None and os.path.exists(options.desugar_dependencies):
424        # Since incremental dexing only ever adds to the desugar_dependencies
425        # file, whenever full dexes are required the .desugardeps files need to
426        # be manually removed.
427        os.unlink(options.desugar_dependencies)
428    _RunD8(dex_cmd, class_files, options.incremental_dir,
429           options.warnings_as_errors,
430           options.show_desugar_default_interface_warnings)
431    logging.debug('Dexed class files.')
432
433
434def _OnStaleMd5(changes, options, final_dex_inputs, service_jars, dex_cmd):
435  logging.debug('_OnStaleMd5')
436  with build_utils.TempDir() as tmp_dir:
437    if options.incremental_dir:
438      # Create directory for all intermediate dex files.
439      if not os.path.exists(options.incremental_dir):
440        os.makedirs(options.incremental_dir)
441
442      _DeleteStaleIncrementalDexFiles(options.incremental_dir, final_dex_inputs)
443      logging.debug('Stale files deleted')
444      _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd)
445
446    _CreateFinalDex(final_dex_inputs,
447                    options.output,
448                    tmp_dir,
449                    dex_cmd,
450                    options=options,
451                    service_jars=service_jars)
452
453
454def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar,
455                                  min_api):
456  dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) + [
457      '-cp',
458      r8_jar_path,
459      'com.android.tools.r8.D8',
460      '--min-api',
461      min_api,
462  ]
463  with build_utils.TempDir() as tmp_dir:
464    _CreateFinalDex(src_paths,
465                    dest_dex_jar,
466                    tmp_dir,
467                    dex_cmd,
468                    service_jars=src_paths)
469
470
471def main(args):
472  build_utils.InitLogging('DEX_DEBUG')
473  options = _ParseArgs(args)
474
475  options.class_inputs += options.class_inputs_filearg
476  options.dex_inputs += options.dex_inputs_filearg
477
478  input_paths = ([
479      build_utils.JAVA_PATH_FOR_INPUTS, options.r8_jar_path,
480      options.custom_d8_jar_path
481  ] + options.class_inputs + options.dex_inputs)
482
483  depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg
484
485  output_paths = [options.output]
486
487  track_subpaths_allowlist = []
488  if options.incremental_dir:
489    final_dex_inputs, service_jars = _IntermediateDexFilePathsFromInputJars(
490        options.class_inputs, options.incremental_dir)
491    output_paths += final_dex_inputs
492    track_subpaths_allowlist += options.class_inputs
493  else:
494    final_dex_inputs = list(options.class_inputs)
495    service_jars = final_dex_inputs
496  service_jars += options.dex_inputs
497  final_dex_inputs += options.dex_inputs
498
499  dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX)
500
501  if options.dump_inputs:
502    dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip']
503
504  if not options.skip_custom_d8:
505    dex_cmd += [
506        '-cp',
507        '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path),
508        'org.chromium.build.CustomD8',
509    ]
510  else:
511    dex_cmd += [
512        '-cp',
513        options.r8_jar_path,
514        'com.android.tools.r8.D8',
515    ]
516
517  if options.release:
518    dex_cmd += ['--release']
519  if options.min_api:
520    dex_cmd += ['--min-api', options.min_api]
521
522  if not options.desugar:
523    dex_cmd += ['--no-desugaring']
524  elif options.classpath:
525    # The classpath is used by D8 to for interface desugaring.
526    if options.desugar_dependencies and not options.skip_custom_d8:
527      dex_cmd += ['--desugar-dependencies', options.desugar_dependencies]
528      if track_subpaths_allowlist:
529        track_subpaths_allowlist += options.classpath
530    depfile_deps += options.classpath
531    input_paths += options.classpath
532    # Still pass the entire classpath in case a new dependency is needed by
533    # desugar, so that desugar_dependencies will be updated for the next build.
534    for path in options.classpath:
535      dex_cmd += ['--classpath', path]
536
537  if options.classpath:
538    dex_cmd += ['--lib', build_utils.JAVA_HOME]
539    for path in options.bootclasspath:
540      dex_cmd += ['--lib', path]
541    depfile_deps += options.bootclasspath
542    input_paths += options.bootclasspath
543
544
545  if options.assertion_handler:
546    dex_cmd += ['--force-assertions-handler:' + options.assertion_handler]
547  if options.force_enable_assertions:
548    dex_cmd += ['--force-enable-assertions']
549
550  # The changes feature from md5_check allows us to only re-dex the class files
551  # that have changed and the class files that need to be re-desugared by D8.
552  md5_check.CallAndWriteDepfileIfStale(
553      lambda changes: _OnStaleMd5(changes, options, final_dex_inputs,
554                                  service_jars, dex_cmd),
555      options,
556      input_paths=input_paths,
557      input_strings=dex_cmd + [str(bool(options.incremental_dir))],
558      output_paths=output_paths,
559      pass_changes=True,
560      track_subpaths_allowlist=track_subpaths_allowlist,
561      depfile_deps=depfile_deps)
562
563
564if __name__ == '__main__':
565  sys.exit(main(sys.argv[1:]))
566