1# Copyright 2012 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4"""Entry point for "intermediates" command.""" 5 6import base64 7import collections 8import dataclasses 9import hashlib 10import os 11import pickle 12import re 13import shutil 14from string import Template 15import subprocess 16import sys 17import tempfile 18import textwrap 19import zipfile 20 21_FILE_DIR = os.path.dirname(__file__) 22_CHROMIUM_SRC = os.path.join(_FILE_DIR, os.pardir, os.pardir) 23_BUILD_ANDROID_GYP = os.path.join(_CHROMIUM_SRC, 'build', 'android', 'gyp') 24 25# Item 0 of sys.path is the directory of the main file; item 1 is PYTHONPATH 26# (if set); item 2 is system libraries. 27sys.path.insert(1, _BUILD_ANDROID_GYP) 28 29from codegen import called_by_native_header 30from codegen import convert_type 31from codegen import header_common 32from codegen import natives_header 33from codegen import placeholder_gen_jni_java 34from codegen import placeholder_java_type 35from codegen import proxy_impl_java 36import common 37import java_types 38import parse 39import proxy 40 41# Use 100 columns rather than 80 because it makes many lines more readable. 42_WRAP_LINE_LENGTH = 100 43# WrapOutput() is fairly slow. Pre-creating TextWrappers helps a bit. 44_WRAPPERS_BY_INDENT = [ 45 textwrap.TextWrapper(width=_WRAP_LINE_LENGTH, 46 expand_tabs=False, 47 replace_whitespace=False, 48 subsequent_indent=' ' * (indent + 4), 49 break_long_words=False) for indent in range(50) 50] # 50 chosen experimentally. 51 52 53class NativeMethod: 54 """Describes a C/C++ method that is called by Java.""" 55 def __init__(self, parsed_method, *, java_class, is_proxy): 56 self.java_class = java_class 57 self.name = parsed_method.name 58 self.signature = parsed_method.signature 59 self.is_proxy = is_proxy 60 self.static = is_proxy or parsed_method.static 61 self.native_class_name = parsed_method.native_class_name 62 63 # Proxy methods don't have a native prefix so the first letter is 64 # lowercase. But we still want the CPP declaration to use upper camel 65 # case for the method name. 66 self.cpp_name = common.capitalize(self.name) 67 self.cpp_impl_name = f'JNI_{java_class.name}_{self.cpp_name}' 68 self.is_test_only = NameIsTestOnly(self.name) 69 70 if self.is_proxy: 71 self.needs_implicit_array_element_class_param = ( 72 proxy.needs_implicit_array_element_class_param(self.return_type)) 73 self.proxy_signature = self.signature.to_proxy() 74 if self.needs_implicit_array_element_class_param: 75 self.proxy_signature = proxy.add_implicit_array_element_class_param( 76 self.proxy_signature) 77 self.proxy_name, self.hashed_proxy_name = proxy.create_method_names( 78 java_class, self.name, self.is_test_only) 79 else: 80 self.needs_implicit_array_element_class_param = False 81 self.proxy_signature = self.signature 82 83 first_param = self.params and self.params[0] 84 if (first_param and first_param.java_type.is_primitive() 85 and first_param.java_type.primitive_name == 'long' 86 and first_param.name.startswith('native')): 87 if parsed_method.native_class_name: 88 self.first_param_cpp_type = parsed_method.native_class_name 89 else: 90 self.first_param_cpp_type = first_param.name[len('native'):] 91 else: 92 self.first_param_cpp_type = None 93 94 @property 95 def return_type(self): 96 return self.signature.return_type 97 98 @property 99 def proxy_return_type(self): 100 return self.proxy_signature.return_type 101 102 @property 103 def params(self): 104 return self.signature.param_list 105 106 @property 107 def proxy_params(self): 108 return self.proxy_signature.param_list 109 110 @property 111 def param_types(self): 112 return self.signature.param_types 113 114 @property 115 def proxy_param_types(self): 116 return self.proxy_signature.param_types 117 118class CalledByNative: 119 """Describes a Java method that is called from C++""" 120 def __init__(self, 121 parsed_called_by_native, 122 *, 123 is_system_class, 124 unchecked=False): 125 self.name = parsed_called_by_native.name 126 self.signature = parsed_called_by_native.signature 127 self.static = parsed_called_by_native.static 128 self.unchecked = parsed_called_by_native.unchecked or unchecked 129 self.java_class = parsed_called_by_native.java_class 130 self.is_system_class = is_system_class 131 132 # Computed once we know if overloads exist. 133 self.method_id_function_name = None 134 135 @property 136 def is_constructor(self): 137 return self.name == '<init>' 138 139 @property 140 def return_type(self): 141 return self.signature.return_type 142 143 @property 144 def params(self): 145 return self.signature.param_list 146 147 148def NameIsTestOnly(name): 149 return name.endswith(('ForTest', 'ForTests', 'ForTesting')) 150 151 152def _MangleMethodName(type_resolver, name, param_types): 153 mangled_types = [] 154 for java_type in param_types: 155 if java_type.primitive_name: 156 part = java_type.primitive_name 157 else: 158 part = type_resolver.contextualize(java_type.java_class).replace('.', '_') 159 mangled_types.append(part + ('Array' * java_type.array_dimensions)) 160 161 return f'{name}__' + '__'.join(mangled_types) 162 163 164def _AssignMethodIdFunctionNames(type_resolver, called_by_natives): 165 # Mangle names for overloads with different number of parameters. 166 def key(called_by_native): 167 return (called_by_native.java_class.full_name_with_slashes, 168 called_by_native.name, len(called_by_native.params)) 169 170 method_counts = collections.Counter(key(x) for x in called_by_natives) 171 172 for called_by_native in called_by_natives: 173 if called_by_native.is_constructor: 174 method_id_function_name = 'Constructor' 175 else: 176 method_id_function_name = called_by_native.name 177 178 if method_counts[key(called_by_native)] > 1: 179 method_id_function_name = _MangleMethodName( 180 type_resolver, method_id_function_name, 181 called_by_native.signature.param_types) 182 183 called_by_native.method_id_function_name = method_id_function_name 184 185 186class JniObject: 187 """Uses the given java source file to generate the JNI header file.""" 188 189 def __init__(self, parsed_file, options, *, from_javap): 190 self.options = options 191 self.filename = parsed_file.filename 192 self.type_resolver = parsed_file.type_resolver 193 self.module_name = parsed_file.module_name 194 self.proxy_interface = parsed_file.proxy_interface 195 self.proxy_visibility = parsed_file.proxy_visibility 196 self.constant_fields = parsed_file.constant_fields 197 # --per-file-natives is not available in all parsers. 198 self.per_file_natives = getattr(options, 'per_file_natives', False) 199 200 # These are different only for legacy reasons. 201 if from_javap: 202 self.jni_namespace = options.namespace or 'JNI_' + self.java_class.name 203 else: 204 self.jni_namespace = parsed_file.jni_namespace or options.namespace 205 206 natives = [] 207 for parsed_method in parsed_file.proxy_methods: 208 natives.append( 209 NativeMethod(parsed_method, java_class=self.java_class, 210 is_proxy=True)) 211 212 for parsed_method in parsed_file.non_proxy_methods: 213 natives.append( 214 NativeMethod(parsed_method, 215 java_class=self.java_class, 216 is_proxy=False)) 217 218 self.natives = natives 219 220 called_by_natives = [] 221 for parsed_called_by_native in parsed_file.called_by_natives: 222 called_by_natives.append( 223 CalledByNative(parsed_called_by_native, 224 unchecked=from_javap and options.unchecked_exceptions, 225 is_system_class=from_javap)) 226 227 _AssignMethodIdFunctionNames(parsed_file.type_resolver, called_by_natives) 228 self.called_by_natives = called_by_natives 229 230 # from-jar does not define these flags. 231 if natives: 232 self.final_gen_jni_class = proxy.get_gen_jni_class( 233 short=options.use_proxy_hash or options.enable_jni_multiplexing, 234 name_prefix=self.module_name, 235 package_prefix=options.package_prefix) 236 else: 237 self.final_gen_jni_class = None 238 239 @property 240 def java_class(self): 241 return self.type_resolver.java_class 242 243 @property 244 def proxy_natives(self): 245 return [n for n in self.natives if n.is_proxy] 246 247 @property 248 def non_proxy_natives(self): 249 return [n for n in self.natives if not n.is_proxy] 250 251 def GetClassesToBeImported(self): 252 classes = set() 253 for p in self.proxy_natives: 254 for t in list(p.param_types) + [p.return_type]: 255 class_obj = t.java_class 256 if class_obj is None: 257 # Primitive types will be None. 258 continue 259 if class_obj.full_name_with_slashes.startswith('java/lang/'): 260 # java.lang** are never imported. 261 continue 262 classes.add(class_obj) 263 264 return sorted(classes) 265 266 def RemoveTestOnlyNatives(self): 267 self.natives = [n for n in self.natives if not n.is_test_only] 268 269 def GetStubName(self, native): 270 """Return the name of the stub function for a native method.""" 271 if native.is_proxy: 272 if self.options.use_proxy_hash: 273 method_name = common.escape_class_name(native.hashed_proxy_name) 274 else: 275 method_name = common.escape_class_name(native.proxy_name) 276 if self.per_file_natives: 277 return 'Java_' + common.escape_class_name( 278 f'{self.java_class.full_name_with_slashes}Jni/native{common.capitalize(native.name)}' 279 ) 280 else: 281 return 'Java_%s_%s' % (common.escape_class_name( 282 self.final_gen_jni_class.full_name_with_slashes), method_name) 283 284 escaped_name = common.escape_class_name( 285 self.java_class.full_name_with_slashes) 286 return f'Java_{escaped_name}_native{native.cpp_name}' 287 288 289def _UsesConvertType(java_type): 290 # Array conversions do not need to be declared and primitive conversions 291 # are just static_cast. 292 return bool(java_type.converted_type() and not java_type.is_array() 293 and not java_type.is_primitive()) 294 295 296def _CollectConvertTypeTypes(natives, called_by_natives): 297 java_to_cpp_types = [] 298 cpp_to_java_types = [] 299 300 for native in natives: 301 java_to_cpp_types.extend(param.java_type for param in native.params 302 if _UsesConvertType(param.java_type)) 303 if _UsesConvertType(native.return_type): 304 cpp_to_java_types.append(native.return_type) 305 306 for called_by_native in called_by_natives: 307 cpp_to_java_types.extend(param.java_type 308 for param in called_by_native.params 309 if _UsesConvertType(param.java_type)) 310 if _UsesConvertType(called_by_native.return_type): 311 java_to_cpp_types.append(called_by_native.return_type) 312 313 return java_to_cpp_types, cpp_to_java_types 314 315 316def _CollectReferencedClasses(jni_obj): 317 ret = set() 318 # @CalledByNatives can appear on nested classes, so check each one. 319 for called_by_native in jni_obj.called_by_natives: 320 ret.add(called_by_native.java_class) 321 for param in called_by_native.params: 322 java_type = param.java_type 323 if java_type.is_object_array() and java_type.converted_type(): 324 ret.add(java_type.java_class) 325 326 327 # Find any classes needed for @JniType conversions. 328 for native in jni_obj.proxy_natives: 329 return_type = native.return_type 330 if return_type.is_object_array() and return_type.converted_type(): 331 ret.add(return_type.java_class) 332 return sorted(ret) 333 334 335class InlHeaderFileGenerator: 336 337 def __init__(self, jni_obj): 338 self.jni_obj = jni_obj 339 self.namespace = jni_obj.jni_namespace 340 java_class = jni_obj.java_class 341 self.java_class = java_class 342 self.class_name = java_class.name 343 self.module_name = jni_obj.module_name 344 self.natives = jni_obj.natives 345 self.called_by_natives = jni_obj.called_by_natives 346 self.constant_fields = jni_obj.constant_fields 347 self.type_resolver = jni_obj.type_resolver 348 self.options = jni_obj.options 349 350 def GetContent(self): 351 """Returns the content of the JNI binding file.""" 352 template = Template("""\ 353${PREAMBLE}\ 354${CLASS_ACCESSORS}\ 355${CONVERSION_FUNCTION_DECLARATIONS}\ 356${OPEN_NAMESPACE}\ 357${CONSTANTS_ENUMS}\ 358${NATIVES}\ 359${CALLED_BY_NATIVES}\ 360${CLOSE_NAMESPACE}\ 361${EPILOGUE}\ 362""") 363 java_classes = _CollectReferencedClasses(self.jni_obj) 364 preamble, epilogue = header_common.header_preamble( 365 GetScriptName(), 366 self.java_class, 367 system_includes=['jni.h'], 368 user_includes=['third_party/jni_zero/jni_export.h'] + 369 self.options.extra_includes) 370 class_accessors = header_common.class_accessors(java_classes, 371 self.jni_obj.module_name) 372 java_to_cpp_types, cpp_to_java_types = _CollectConvertTypeTypes( 373 self.natives, self.called_by_natives) 374 conversion_declarations = convert_type.conversion_declarations( 375 java_to_cpp_types, cpp_to_java_types) 376 constants_enums = called_by_native_header.constants_enums( 377 self.java_class, self.constant_fields) 378 379 called_by_natives_code = called_by_native_header.methods( 380 self.called_by_natives) 381 natives_code = natives_header.methods(self.jni_obj) 382 383 values = { 384 'PREAMBLE': preamble, 385 'EPILOGUE': epilogue, 386 'CLASS_ACCESSORS': class_accessors, 387 'CONVERSION_FUNCTION_DECLARATIONS': conversion_declarations, 388 'CONSTANTS_ENUMS': constants_enums, 389 'CALLED_BY_NATIVES': called_by_natives_code, 390 'NATIVES': natives_code, 391 'OPEN_NAMESPACE': self.GetOpenNamespaceString(), 392 'CLOSE_NAMESPACE': self.GetCloseNamespaceString(), 393 } 394 395 return template.substitute(values) 396 397 def GetOpenNamespaceString(self): 398 if self.namespace: 399 all_namespaces = [ 400 'namespace %s {' % ns for ns in self.namespace.split('::') 401 ] 402 return '\n'.join(all_namespaces) + '\n' 403 return '' 404 405 def GetCloseNamespaceString(self): 406 if self.namespace: 407 all_namespaces = [ 408 '} // namespace %s' % ns for ns in self.namespace.split('::') 409 ] 410 all_namespaces.reverse() 411 return '\n' + '\n'.join(all_namespaces) 412 return '' 413 414 415def WrapOutput(output): 416 ret = [] 417 for line in output.splitlines(): 418 # Do not wrap preprocessor directives or comments. 419 if len(line) < _WRAP_LINE_LENGTH or line[0] == '#' or line.startswith('//'): 420 ret.append(line) 421 else: 422 # Assumes that the line is not already indented as a continuation line, 423 # which is not always true (oh well). 424 first_line_indent = (len(line) - len(line.lstrip())) 425 wrapper = _WRAPPERS_BY_INDENT[first_line_indent] 426 ret.extend(wrapper.wrap(line)) 427 ret += [''] 428 return '\n'.join(ret) 429 430 431def GetScriptName(): 432 return '//third_party/jni_zero/jni_zero.py' 433 434 435def _RemoveStaleHeaders(path, output_names): 436 if not os.path.isdir(path): 437 return 438 # Do not remove output files so that timestamps on declared outputs are not 439 # modified unless their contents are changed (avoids reverse deps needing to 440 # be rebuilt). 441 preserve = set(output_names) 442 for root, _, files in os.walk(path): 443 for f in files: 444 if f not in preserve: 445 file_path = os.path.join(root, f) 446 if os.path.isfile(file_path) and file_path.endswith('.h'): 447 os.remove(file_path) 448 449 450def _CheckSameModule(jni_objs): 451 files_by_module = collections.defaultdict(list) 452 for jni_obj in jni_objs: 453 if jni_obj.proxy_natives: 454 files_by_module[jni_obj.module_name].append(jni_obj.filename) 455 if len(files_by_module) > 1: 456 sys.stderr.write( 457 'Multiple values for @NativeMethods(moduleName) is not supported.\n') 458 for module_name, filenames in files_by_module.items(): 459 sys.stderr.write(f'module_name={module_name}\n') 460 for filename in filenames: 461 sys.stderr.write(f' {filename}\n') 462 sys.exit(1) 463 464 465def _CheckNotEmpty(jni_objs): 466 has_empty = False 467 for jni_obj in jni_objs: 468 if not (jni_obj.natives or jni_obj.called_by_natives): 469 has_empty = True 470 sys.stderr.write(f'No native methods found in {jni_obj.filename}.\n') 471 if has_empty: 472 sys.exit(1) 473 474 475def _RunJavap(javap_path, class_file): 476 p = subprocess.run([javap_path, '-s', '-constants', class_file], 477 text=True, 478 capture_output=True, 479 check=True) 480 return p.stdout 481 482 483def _ParseClassFiles(jar_file, class_files, args): 484 # Parse javap output. 485 ret = [] 486 with tempfile.TemporaryDirectory() as temp_dir: 487 with zipfile.ZipFile(jar_file) as z: 488 z.extractall(temp_dir, class_files) 489 for class_file in class_files: 490 class_file = os.path.join(temp_dir, class_file) 491 contents = _RunJavap(args.javap, class_file) 492 parsed_file = parse.parse_javap(class_file, contents) 493 ret.append(JniObject(parsed_file, args, from_javap=True)) 494 return ret 495 496 497def _CreateSrcJar(srcjar_path, 498 gen_jni_class, 499 jni_objs, 500 *, 501 script_name, 502 per_file_natives=False): 503 with common.atomic_output(srcjar_path) as f: 504 with zipfile.ZipFile(f, 'w') as srcjar: 505 for jni_obj in jni_objs: 506 if not jni_obj.proxy_natives: 507 continue 508 content = proxy_impl_java.Generate(jni_obj, 509 gen_jni_class=gen_jni_class, 510 script_name=script_name, 511 per_file_natives=per_file_natives) 512 zip_path = f'{jni_obj.java_class.class_without_prefix.full_name_with_slashes}Jni.java' 513 common.add_to_zip_hermetic(srcjar, zip_path, data=content) 514 515 if not per_file_natives: 516 content = placeholder_gen_jni_java.Generate(jni_objs, 517 gen_jni_class=gen_jni_class, 518 script_name=script_name) 519 zip_path = f'{gen_jni_class.full_name_with_slashes}.java' 520 common.add_to_zip_hermetic(srcjar, zip_path, data=content) 521 522 523def _CreatePlaceholderSrcJar(srcjar_path, jni_objs, *, script_name): 524 already_added = set() 525 with common.atomic_output(srcjar_path) as f: 526 with zipfile.ZipFile(f, 'w') as srcjar: 527 for jni_obj in jni_objs: 528 if not jni_obj.proxy_natives: 529 continue 530 main_class = jni_obj.type_resolver.java_class 531 zip_path = main_class.class_without_prefix.full_name_with_slashes + '.java' 532 content = placeholder_java_type.Generate( 533 main_class, 534 jni_obj.type_resolver.nested_classes, 535 script_name=script_name, 536 proxy_interface=jni_obj.proxy_interface, 537 proxy_natives=jni_obj.proxy_natives) 538 common.add_to_zip_hermetic(srcjar, zip_path, data=content) 539 already_added.add(zip_path) 540 # In rare circumstances, another file in our generate_jni list will 541 # import the FooJni from another class within the same generate_jni 542 # target. We want to make sure we don't make placeholders for these, but 543 # we do want placeholders for all BarJni classes that aren't a part of 544 # this generate_jni. 545 fake_zip_path = main_class.class_without_prefix.full_name_with_slashes + 'Jni.java' 546 already_added.add(fake_zip_path) 547 548 placeholders = collections.defaultdict(list) 549 # Doing this in 2 phases to ensure that the Jni classes (the ones that 550 # can have @NativeMethods) all get added first, so we don't accidentally 551 # write a stubbed version of the class if it's imported by another class. 552 for jni_obj in jni_objs: 553 for java_class in jni_obj.GetClassesToBeImported(): 554 if java_class.full_name_with_slashes.startswith('java/'): 555 continue 556 # TODO(mheikal): handle more than 1 nesting layer. 557 if java_class.is_nested(): 558 placeholders[java_class.get_outer_class()].append(java_class) 559 elif java_class not in placeholders: 560 placeholders[java_class] = [] 561 for java_class, nested_classes in placeholders.items(): 562 zip_path = java_class.class_without_prefix.full_name_with_slashes + '.java' 563 if zip_path not in already_added: 564 content = placeholder_java_type.Generate(java_class, 565 nested_classes, 566 script_name=script_name) 567 common.add_to_zip_hermetic(srcjar, zip_path, data=content) 568 already_added.add(zip_path) 569 570 571def _WriteHeaders(jni_objs, output_names, output_dir): 572 for jni_obj, header_name in zip(jni_objs, output_names): 573 output_file = os.path.join(output_dir, header_name) 574 content = InlHeaderFileGenerator(jni_obj).GetContent() 575 576 with common.atomic_output(output_file, 'w') as f: 577 f.write(content) 578 579 580def GenerateFromSource(parser, args): 581 # Remove existing headers so that moving .java source files but not updating 582 # the corresponding C++ include will be a compile failure (otherwise 583 # incremental builds will usually not catch this). 584 _RemoveStaleHeaders(args.output_dir, args.output_names) 585 586 try: 587 parsed_files = [ 588 parse.parse_java_file(f, package_prefix=args.package_prefix) 589 for f in args.input_files 590 ] 591 jni_objs = [JniObject(x, args, from_javap=False) for x in parsed_files] 592 _CheckNotEmpty(jni_objs) 593 _CheckSameModule(jni_objs) 594 except parse.ParseError as e: 595 sys.stderr.write(f'{e}\n') 596 sys.exit(1) 597 598 _WriteHeaders(jni_objs, args.output_names, args.output_dir) 599 600 jni_objs_with_proxy_natives = [x for x in jni_objs if x.proxy_natives] 601 # Write .srcjar 602 if args.srcjar_path: 603 if jni_objs_with_proxy_natives: 604 gen_jni_class = proxy.get_gen_jni_class( 605 short=False, 606 name_prefix=jni_objs_with_proxy_natives[0].module_name, 607 package_prefix=args.package_prefix) 608 _CreateSrcJar(args.srcjar_path, 609 gen_jni_class, 610 jni_objs_with_proxy_natives, 611 script_name=GetScriptName(), 612 per_file_natives=args.per_file_natives) 613 else: 614 # Only @CalledByNatives. 615 zipfile.ZipFile(args.srcjar_path, 'w').close() 616 if args.jni_pickle: 617 with common.atomic_output(args.jni_pickle, 'wb') as f: 618 pickle.dump(parsed_files, f) 619 620 if args.placeholder_srcjar_path: 621 if jni_objs_with_proxy_natives: 622 _CreatePlaceholderSrcJar(args.placeholder_srcjar_path, 623 jni_objs_with_proxy_natives, 624 script_name=GetScriptName()) 625 else: 626 zipfile.ZipFile(args.placeholder_srcjar_path, 'w').close() 627 628 629def GenerateFromJar(parser, args): 630 if not args.javap: 631 args.javap = shutil.which('javap') 632 if not args.javap: 633 parser.error('Could not find "javap" on your PATH. Use --javap to ' 634 'specify its location.') 635 636 # Remove existing headers so that moving .java source files but not updating 637 # the corresponding C++ include will be a compile failure (otherwise 638 # incremental builds will usually not catch this). 639 _RemoveStaleHeaders(args.output_dir, args.output_names) 640 641 try: 642 jni_objs = _ParseClassFiles(args.jar_file, args.input_files, args) 643 except parse.ParseError as e: 644 sys.stderr.write(f'{e}\n') 645 sys.exit(1) 646 647 _WriteHeaders(jni_objs, args.output_names, args.output_dir) 648