1#!/usr/bin/env python3 2# Copyright 2017 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Wraps bin/helper/bytecode_processor and expands @FileArgs.""" 7 8import argparse 9import collections 10import logging 11import pathlib 12import shlex 13import sys 14from typing import Dict, List, Optional, Set, Tuple 15 16from util import build_utils 17from util import dep_utils 18from util import jar_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_SRC_PATH = pathlib.Path(build_utils.DIR_SOURCE_ROOT).resolve() 23sys.path.append(str(_SRC_PATH / 'build/gn_ast')) 24from gn_editor import NO_VALID_GN_STR 25 26 27def _ShouldIgnoreDep(dep_name: str): 28 if 'gen.base_module.R' in dep_name: 29 return True 30 return False 31 32 33def _ParseDepGraph(jar_path: str): 34 output = jar_utils.run_jdeps(pathlib.Path(jar_path)) 35 assert output is not None, f'Unable to parse jdep for {jar_path}' 36 dep_graph = collections.defaultdict(set) 37 # pylint: disable=line-too-long 38 # Example output: 39 # java.javac.jar -> java.base 40 # java.javac.jar -> not found 41 # org.chromium.chrome.browser.tabmodel.AsyncTabParamsManagerFactory -> java.lang.Object java.base 42 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus not found 43 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.base.ApplicationStatus$ActivityStateListener not found 44 # org.chromium.chrome.browser.tabmodel.TabWindowManagerImpl -> org.chromium.chrome.browser.tab.Tab not found 45 # pylint: enable=line-too-long 46 for line in output.splitlines(): 47 parsed = line.split() 48 # E.g. java.javac.jar -> java.base 49 if len(parsed) <= 3: 50 continue 51 # E.g. java.javac.jar -> not found 52 if parsed[2] == 'not' and parsed[3] == 'found': 53 continue 54 if parsed[1] != '->': 55 continue 56 dep_from = parsed[0] 57 dep_to = parsed[2] 58 dep_graph[dep_from].add(dep_to) 59 return dep_graph 60 61 62def _EnsureDirectClasspathIsComplete( 63 *, 64 input_jar: str, 65 gn_target: str, 66 output_dir: str, 67 sdk_classpath_jars: List[str], 68 direct_classpath_jars: List[str], 69 full_classpath_jars: List[str], 70 full_classpath_gn_targets: List[str], 71 warnings_as_errors: bool, 72 auto_add_deps: bool, 73): 74 logging.info('Parsing %d direct classpath jars', len(sdk_classpath_jars)) 75 sdk_classpath_deps = set() 76 for jar in sdk_classpath_jars: 77 deps = jar_utils.extract_full_class_names_from_jar(jar) 78 sdk_classpath_deps.update(deps) 79 80 logging.info('Parsing %d direct classpath jars', len(direct_classpath_jars)) 81 direct_classpath_deps = set() 82 for jar in direct_classpath_jars: 83 deps = jar_utils.extract_full_class_names_from_jar(jar) 84 direct_classpath_deps.update(deps) 85 86 logging.info('Parsing %d full classpath jars', len(full_classpath_jars)) 87 full_classpath_deps = set() 88 dep_to_target = collections.defaultdict(set) 89 for jar, target in zip(full_classpath_jars, full_classpath_gn_targets): 90 deps = jar_utils.extract_full_class_names_from_jar(jar) 91 full_classpath_deps.update(deps) 92 for dep in deps: 93 dep_to_target[dep].add(target) 94 95 transitive_deps = full_classpath_deps - direct_classpath_deps 96 97 missing_class_to_caller: Dict[str, str] = {} 98 dep_graph = _ParseDepGraph(input_jar) 99 logging.info('Finding missing deps from %d classes', len(dep_graph)) 100 # dep_graph.keys() is a list of all the classes in the current input_jar. Skip 101 # all of these to avoid checking dependencies in the same target (e.g. A 102 # depends on B, but both A and B are in input_jar). 103 # Since the bundle will always have access to classes in the current android 104 # sdk, those should not be considered missing. 105 seen_deps = set(dep_graph.keys()) | sdk_classpath_deps 106 for dep_from, deps_to in dep_graph.items(): 107 for dep_to in deps_to - seen_deps: 108 if _ShouldIgnoreDep(dep_to): 109 continue 110 seen_deps.add(dep_to) 111 if dep_to in transitive_deps: 112 # Allow clobbering since it doesn't matter which specific class depends 113 # on |dep_to|. 114 missing_class_to_caller[dep_to] = dep_from 115 116 if missing_class_to_caller: 117 _ProcessMissingDirectClasspathDeps(missing_class_to_caller, dep_to_target, 118 gn_target, output_dir, 119 warnings_as_errors, auto_add_deps) 120 121 122def _ProcessMissingDirectClasspathDeps( 123 missing_class_to_caller: Dict[str, str], 124 dep_to_target: Dict[str, Set[str]], 125 gn_target: str, 126 output_dir: str, 127 warnings_as_errors: bool, 128 auto_add_deps: bool, 129): 130 potential_targets_to_missing_classes: Dict[ 131 Tuple, List[str]] = collections.defaultdict(list) 132 for missing_class in missing_class_to_caller: 133 potential_targets = tuple(sorted(dep_to_target[missing_class])) 134 potential_targets_to_missing_classes[potential_targets].append( 135 missing_class) 136 137 deps_to_add_programatically = _DisambiguateMissingDeps( 138 potential_targets_to_missing_classes, output_dir=output_dir) 139 140 cmd = dep_utils.CreateAddDepsCommand(gn_target, 141 sorted(deps_to_add_programatically)) 142 143 if not auto_add_deps: 144 _PrintAndMaybeExit(potential_targets_to_missing_classes, 145 missing_class_to_caller, gn_target, warnings_as_errors, 146 cmd) 147 else: 148 failed = False 149 try: 150 stdout = build_utils.CheckOutput(cmd, 151 cwd=build_utils.DIR_SOURCE_ROOT, 152 fail_on_output=warnings_as_errors) 153 if f'Unable to find {gn_target}' in stdout: 154 # This can happen if a target's deps are stored in a variable instead 155 # of a list and then simply assigned: `deps = deps_variable`. These 156 # need to be manually added to the `deps_variable`. 157 failed = True 158 except build_utils.CalledProcessError as e: 159 if NO_VALID_GN_STR in e.output: 160 failed = True 161 else: 162 raise 163 164 build_file_path = dep_utils.GnTargetToBuildFilePath(gn_target) 165 if failed: 166 print(f'Unable to auto-add missing dep(s) to {build_file_path}.') 167 _PrintAndMaybeExit(potential_targets_to_missing_classes, 168 missing_class_to_caller, gn_target, warnings_as_errors) 169 else: 170 gn_target_name = gn_target.split(':', 1)[-1] 171 print(f'Successfully updated "{gn_target_name}" in {build_file_path} ' 172 f'with missing direct deps: {deps_to_add_programatically}') 173 174 175def _DisambiguateMissingDeps( 176 potential_targets_to_missing_classes: Dict[Tuple, List[str]], 177 output_dir: str, 178): 179 deps_to_add_programatically = set() 180 class_lookup_index = None 181 # Run this even when len(potential_targets) == 1, because we need to look for 182 # java_group()s with preferred_deps=true. 183 for (potential_targets, 184 missing_classes) in potential_targets_to_missing_classes.items(): 185 # Dict for ordered dict. 186 potential_targets = {t: True for t in potential_targets} 187 # Rather than just picking any of the potential targets, we want to use 188 # dep_utils.ClassLookupIndex to ensure we respect the preferred dep if any 189 # exists for the missing deps. It is necessary to obtain the preferred dep 190 # status of these potential targets by matching them to a ClassEntry. 191 target_name_to_class_entry: Dict[str, dep_utils.ClassEntry] = {} 192 for missing_class in missing_classes: 193 # Lazily create the ClassLookupIndex in case all potential_targets lists 194 # are only 1 element in length. 195 if class_lookup_index is None: 196 class_lookup_index = dep_utils.ClassLookupIndex( 197 pathlib.Path(output_dir), should_build=False) 198 for class_entry in class_lookup_index.match(missing_class): 199 target_name_to_class_entry[class_entry.target] = class_entry 200 # Ensure preferred deps are always included in the options. 201 if (class_entry.preferred_dep 202 and class_entry.target not in potential_targets): 203 potential_targets[class_entry.target] = True 204 potential_class_entries = [ 205 target_name_to_class_entry[t] for t in potential_targets 206 ] 207 potential_class_entries = dep_utils.DisambiguateDeps( 208 potential_class_entries) 209 deps_to_add_programatically.add(potential_class_entries[0].target) 210 return deps_to_add_programatically 211 212 213def _PrintAndMaybeExit( 214 potential_targets_to_missing_classes: Dict[Tuple, List[str]], 215 missing_class_to_caller: Dict[str, str], 216 gn_target: str, 217 warnings_as_errors: bool, 218 cmd: Optional[List[str]] = None, 219): 220 print('=' * 30 + ' Dependency Checks Failed ' + '=' * 30) 221 print(f'Target: {gn_target}') 222 print('Direct classpath is incomplete. To fix, add deps on:') 223 for (potential_targets, 224 missing_classes) in potential_targets_to_missing_classes.items(): 225 if len(potential_targets) == 1: 226 print(f' * {potential_targets[0]}') 227 else: 228 print(f' * One of {", ".join(potential_targets)}') 229 for missing_class in missing_classes: 230 caller = missing_class_to_caller[missing_class] 231 print(f' ** {missing_class} (needed by {caller})') 232 if cmd: 233 print('\nHint: Run the following command to add the missing deps:') 234 print(f' {shlex.join(cmd)}\n') 235 if warnings_as_errors: 236 sys.exit(1) 237 238 239def main(argv): 240 build_utils.InitLogging('BYTECODE_PROCESSOR_DEBUG') 241 argv = build_utils.ExpandFileArgs(argv[1:]) 242 parser = argparse.ArgumentParser() 243 parser.add_argument('--target-name', help='Fully qualified GN target name.') 244 parser.add_argument('--use-build-server', 245 action='store_true', 246 help='Always use the build server.') 247 parser.add_argument('--experimental-build-server', 248 action='store_true', 249 help='Use experimental build server features.') 250 parser.add_argument('--gn-target', required=True) 251 parser.add_argument('--input-jar', required=True) 252 parser.add_argument('--direct-classpath-jars') 253 parser.add_argument('--sdk-classpath-jars') 254 parser.add_argument('--full-classpath-jars') 255 parser.add_argument('--full-classpath-gn-targets') 256 parser.add_argument('--chromium-output-dir') 257 parser.add_argument('--stamp') 258 parser.add_argument('--warnings-as-errors', 259 action='store_true', 260 help='Treat all warnings as errors.') 261 parser.add_argument( 262 '--auto-add-deps', 263 action='store_true', 264 help='Attempt to automatically add missing deps to the corresponding ' 265 'BUILD.gn file.') 266 args = parser.parse_args(argv) 267 268 if server_utils.MaybeRunCommand(name=args.target_name, 269 argv=sys.argv, 270 stamp_file=args.stamp, 271 force=args.use_build_server, 272 experimental=args.experimental_build_server): 273 return 274 275 args.sdk_classpath_jars = action_helpers.parse_gn_list( 276 args.sdk_classpath_jars) 277 args.direct_classpath_jars = action_helpers.parse_gn_list( 278 args.direct_classpath_jars) 279 args.full_classpath_jars = action_helpers.parse_gn_list( 280 args.full_classpath_jars) 281 args.full_classpath_gn_targets = [ 282 dep_utils.ReplaceGmsPackageIfNeeded(t) 283 for t in action_helpers.parse_gn_list(args.full_classpath_gn_targets) 284 ] 285 286 logging.info('Processed args for %s, starting direct classpath check.', 287 args.target_name) 288 _EnsureDirectClasspathIsComplete( 289 input_jar=args.input_jar, 290 gn_target=args.gn_target, 291 output_dir=args.chromium_output_dir, 292 sdk_classpath_jars=args.sdk_classpath_jars, 293 direct_classpath_jars=args.direct_classpath_jars, 294 full_classpath_jars=args.full_classpath_jars, 295 full_classpath_gn_targets=args.full_classpath_gn_targets, 296 warnings_as_errors=args.warnings_as_errors, 297 auto_add_deps=args.auto_add_deps) 298 logging.info('Check completed.') 299 300 if args.stamp: 301 build_utils.Touch(args.stamp) 302 303 304if __name__ == '__main__': 305 sys.exit(main(sys.argv)) 306