1#!/usr/bin/env python 2# Copyright 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""The frontend for the Mojo bindings system.""" 7 8 9import argparse 10import pickle 11import hashlib 12import importlib 13import json 14import os 15import pprint 16import re 17import struct 18import sys 19 20# Disable lint check for finding modules: 21# pylint: disable=F0401 22 23def _GetDirAbove(dirname): 24 """Returns the directory "above" this file containing |dirname| (which must 25 also be "above" this file).""" 26 path = os.path.abspath(__file__) 27 while True: 28 path, tail = os.path.split(path) 29 assert tail 30 if tail == dirname: 31 return path 32 33# Manually check for the command-line flag. (This isn't quite right, since it 34# ignores, e.g., "--", but it's close enough.) 35if "--use_bundled_pylibs" in sys.argv[1:]: 36 sys.path.insert(0, os.path.join(_GetDirAbove("mojo"), "third_party")) 37 38sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 39 "pylib")) 40 41from mojom.error import Error 42import mojom.fileutil as fileutil 43from mojom.generate import template_expander 44from mojom.generate import translate 45from mojom.generate.generator import AddComputedData, WriteFile 46from mojom.parse.conditional_features import RemoveDisabledDefinitions 47from mojom.parse.parser import Parse 48 49 50_BUILTIN_GENERATORS = { 51 "c++": "mojom_cpp_generator", 52 "javascript": "mojom_js_generator", 53 "java": "mojom_java_generator", 54} 55 56 57def LoadGenerators(generators_string): 58 if not generators_string: 59 return [] # No generators. 60 61 generators = {} 62 for generator_name in [s.strip() for s in generators_string.split(",")]: 63 language = generator_name.lower() 64 if language not in _BUILTIN_GENERATORS: 65 print("Unknown generator name %s" % generator_name) 66 sys.exit(1) 67 generator_module = importlib.import_module( 68 "mojo.public.tools.bindings.generators.%s" % _BUILTIN_GENERATORS[language]) 69 generators[language] = generator_module 70 return generators 71 72 73def MakeImportStackMessage(imported_filename_stack): 74 """Make a (human-readable) message listing a chain of imports. (Returned 75 string begins with a newline (if nonempty) and does not end with one.)""" 76 return ''.join( 77 reversed(["\n %s was imported by %s" % (a, b) for (a, b) in \ 78 zip(imported_filename_stack[1:], imported_filename_stack)])) 79 80 81class RelativePath(object): 82 """Represents a path relative to the source tree.""" 83 def __init__(self, path, source_root): 84 self.path = path 85 self.source_root = source_root 86 87 def relative_path(self): 88 return os.path.relpath(os.path.abspath(self.path), 89 os.path.abspath(self.source_root)) 90 91 92def FindImportFile(rel_dir, file_name, search_rel_dirs): 93 """Finds |file_name| in either |rel_dir| or |search_rel_dirs|. Returns a 94 RelativePath with first file found, or an arbitrary non-existent file 95 otherwise.""" 96 for rel_search_dir in [rel_dir] + search_rel_dirs: 97 path = os.path.join(rel_search_dir.path, file_name) 98 if os.path.isfile(path): 99 return RelativePath(path, rel_search_dir.source_root) 100 return RelativePath(os.path.join(rel_dir.path, file_name), 101 rel_dir.source_root) 102 103 104def ScrambleMethodOrdinals(interfaces, salt): 105 already_generated = set() 106 for interface in interfaces: 107 i = 0 108 already_generated.clear() 109 for method in interface.methods: 110 while True: 111 i = i + 1 112 if i == 1000000: 113 raise Exception("Could not generate %d method ordinals for %s" % 114 (len(interface.methods), interface.mojom_name)) 115 # Generate a scrambled method.ordinal value. The algorithm doesn't have 116 # to be very strong, cryptographically. It just needs to be non-trivial 117 # to guess the results without the secret salt, in order to make it 118 # harder for a compromised process to send fake Mojo messages. 119 sha256 = hashlib.sha256(salt) 120 sha256.update(interface.mojom_name) 121 sha256.update(str(i)) 122 # Take the first 4 bytes as a little-endian uint32. 123 ordinal = struct.unpack('<L', sha256.digest()[:4])[0] 124 # Trim to 31 bits, so it always fits into a Java (signed) int. 125 ordinal = ordinal & 0x7fffffff 126 if ordinal in already_generated: 127 continue 128 already_generated.add(ordinal) 129 method.ordinal = ordinal 130 method.ordinal_comment = ( 131 'The %s value is based on sha256(salt + "%s%d").' % 132 (ordinal, interface.mojom_name, i)) 133 break 134 135 136def ReadFileContents(filename): 137 with open(filename, 'rb') as f: 138 return f.read() 139 140 141class MojomProcessor(object): 142 """Parses mojom files and creates ASTs for them. 143 144 Attributes: 145 _processed_files: {Dict[str, mojom.generate.module.Module]} Mapping from 146 relative mojom filename paths to the module AST for that mojom file. 147 """ 148 def __init__(self, should_generate): 149 self._should_generate = should_generate 150 self._processed_files = {} 151 self._typemap = {} 152 153 def LoadTypemaps(self, typemaps): 154 # Support some very simple single-line comments in typemap JSON. 155 comment_expr = r"^\s*//.*$" 156 def no_comments(line): 157 return not re.match(comment_expr, line) 158 for filename in typemaps: 159 with open(filename) as f: 160 typemaps = json.loads("".join(filter(no_comments, f.readlines()))) 161 for language, typemap in typemaps.items(): 162 language_map = self._typemap.get(language, {}) 163 language_map.update(typemap) 164 self._typemap[language] = language_map 165 166 def _GenerateModule(self, args, remaining_args, generator_modules, 167 rel_filename, imported_filename_stack): 168 # Return the already-generated module. 169 if rel_filename.path in self._processed_files: 170 return self._processed_files[rel_filename.path] 171 172 if rel_filename.path in imported_filename_stack: 173 print("%s: Error: Circular dependency" % rel_filename.path + 174 MakeImportStackMessage(imported_filename_stack + [rel_filename.path])) 175 sys.exit(1) 176 177 tree = _UnpickleAST(_FindPicklePath(rel_filename, args.gen_directories + 178 [args.output_dir])) 179 180 dirname = os.path.dirname(rel_filename.path) 181 182 # Process all our imports first and collect the module object for each. 183 # We use these to generate proper type info. 184 imports = {} 185 for parsed_imp in tree.import_list: 186 rel_import_file = FindImportFile( 187 RelativePath(dirname, rel_filename.source_root), 188 parsed_imp.import_filename, args.import_directories) 189 imports[parsed_imp.import_filename] = self._GenerateModule( 190 args, remaining_args, generator_modules, rel_import_file, 191 imported_filename_stack + [rel_filename.path]) 192 193 # Set the module path as relative to the source root. 194 # Normalize to unix-style path here to keep the generators simpler. 195 module_path = rel_filename.relative_path().replace('\\', '/') 196 197 module = translate.OrderedModule(tree, module_path, imports) 198 199 if args.scrambled_message_id_salt_paths: 200 salt = ''.join( 201 map(ReadFileContents, args.scrambled_message_id_salt_paths)) 202 ScrambleMethodOrdinals(module.interfaces, salt) 203 204 if self._should_generate(rel_filename.path): 205 AddComputedData(module) 206 for language, generator_module in generator_modules.items(): 207 generator = generator_module.Generator( 208 module, args.output_dir, typemap=self._typemap.get(language, {}), 209 variant=args.variant, bytecode_path=args.bytecode_path, 210 for_blink=args.for_blink, 211 use_once_callback=args.use_once_callback, 212 js_bindings_mode=args.js_bindings_mode, 213 export_attribute=args.export_attribute, 214 export_header=args.export_header, 215 generate_non_variant_code=args.generate_non_variant_code, 216 support_lazy_serialization=args.support_lazy_serialization, 217 disallow_native_types=args.disallow_native_types, 218 disallow_interfaces=args.disallow_interfaces, 219 generate_message_ids=args.generate_message_ids, 220 generate_fuzzing=args.generate_fuzzing) 221 filtered_args = [] 222 if hasattr(generator_module, 'GENERATOR_PREFIX'): 223 prefix = '--' + generator_module.GENERATOR_PREFIX + '_' 224 filtered_args = [arg for arg in remaining_args 225 if arg.startswith(prefix)] 226 generator.GenerateFiles(filtered_args) 227 228 # Save result. 229 self._processed_files[rel_filename.path] = module 230 return module 231 232 233def _Generate(args, remaining_args): 234 if args.variant == "none": 235 args.variant = None 236 237 for idx, import_dir in enumerate(args.import_directories): 238 tokens = import_dir.split(":") 239 if len(tokens) >= 2: 240 args.import_directories[idx] = RelativePath(tokens[0], tokens[1]) 241 else: 242 args.import_directories[idx] = RelativePath(tokens[0], args.depth) 243 generator_modules = LoadGenerators(args.generators_string) 244 245 fileutil.EnsureDirectoryExists(args.output_dir) 246 247 processor = MojomProcessor(lambda filename: filename in args.filename) 248 processor.LoadTypemaps(set(args.typemaps)) 249 250 if args.filelist: 251 with open(args.filelist) as f: 252 args.filename.extend(f.read().split()) 253 254 for filename in args.filename: 255 processor._GenerateModule(args, remaining_args, generator_modules, 256 RelativePath(filename, args.depth), []) 257 258 return 0 259 260 261def _FindPicklePath(rel_filename, search_dirs): 262 filename, _ = os.path.splitext(rel_filename.relative_path()) 263 pickle_path = filename + '.p' 264 for search_dir in search_dirs: 265 path = os.path.join(search_dir, pickle_path) 266 if os.path.isfile(path): 267 return path 268 raise Exception("%s: Error: Could not find file in %r" % (pickle_path, search_dirs)) 269 270 271def _GetPicklePath(rel_filename, output_dir): 272 filename, _ = os.path.splitext(rel_filename.relative_path()) 273 pickle_path = filename + '.p' 274 return os.path.join(output_dir, pickle_path) 275 276 277def _PickleAST(ast, output_file): 278 full_dir = os.path.dirname(output_file) 279 fileutil.EnsureDirectoryExists(full_dir) 280 281 try: 282 WriteFile(pickle.dumps(ast, protocol=0), output_file) 283 except (IOError, pickle.PicklingError) as e: 284 print("%s: Error: %s" % (output_file, str(e))) 285 sys.exit(1) 286 287def _UnpickleAST(input_file): 288 try: 289 with open(input_file, "rb") as f: 290 return pickle.load(f) 291 except (IOError, pickle.UnpicklingError) as e: 292 print("%s: Error: %s" % (input_file, str(e))) 293 sys.exit(1) 294 295def _ParseFile(args, rel_filename): 296 try: 297 with open(rel_filename.path) as f: 298 source = f.read() 299 except IOError as e: 300 print("%s: Error: %s" % (rel_filename.path, e.strerror)) 301 sys.exit(1) 302 303 try: 304 tree = Parse(source, rel_filename.path) 305 RemoveDisabledDefinitions(tree, args.enabled_features) 306 except Error as e: 307 print("%s: Error: %s" % (rel_filename.path, str(e))) 308 sys.exit(1) 309 _PickleAST(tree, _GetPicklePath(rel_filename, args.output_dir)) 310 311 312def _Parse(args, _): 313 fileutil.EnsureDirectoryExists(args.output_dir) 314 315 if args.filelist: 316 with open(args.filelist) as f: 317 args.filename.extend(f.read().split()) 318 319 for filename in args.filename: 320 _ParseFile(args, RelativePath(filename, args.depth)) 321 return 0 322 323 324def _Precompile(args, _): 325 generator_modules = LoadGenerators(",".join(_BUILTIN_GENERATORS.keys())) 326 327 template_expander.PrecompileTemplates(generator_modules, args.output_dir) 328 return 0 329 330def _VerifyImportDeps(args, __): 331 fileutil.EnsureDirectoryExists(args.gen_dir) 332 333 if args.filelist: 334 with open(args.filelist) as f: 335 args.filename.extend(f.read().split()) 336 337 for filename in args.filename: 338 rel_path = RelativePath(filename, args.depth) 339 tree = _UnpickleAST(_GetPicklePath(rel_path, args.gen_dir)) 340 341 mojom_imports = set( 342 parsed_imp.import_filename for parsed_imp in tree.import_list 343 ) 344 345 # read the paths from the file 346 f_deps = open(args.deps_file, 'r') 347 deps_sources = set() 348 for deps_path in f_deps: 349 deps_path = deps_path.rstrip('\n') 350 f_sources = open(deps_path, 'r') 351 352 for source_file in f_sources: 353 source_dir = deps_path.split(args.gen_dir + "/", 1)[1] 354 full_source_path = os.path.dirname(source_dir) + "/" + \ 355 source_file 356 deps_sources.add(full_source_path.rstrip('\n')) 357 358 if (not deps_sources.issuperset(mojom_imports)): 359 print(">>> [%s] Missing dependencies for the following imports: %s" % ( 360 args.filename[0], 361 list(mojom_imports.difference(deps_sources)))) 362 sys.exit(1) 363 364 source_filename, _ = os.path.splitext(rel_path.relative_path()) 365 output_file = source_filename + '.v' 366 output_file_path = os.path.join(args.gen_dir, output_file) 367 WriteFile("", output_file_path) 368 369 return 0 370 371def main(): 372 parser = argparse.ArgumentParser( 373 description="Generate bindings from mojom files.") 374 parser.add_argument("--use_bundled_pylibs", action="store_true", 375 help="use Python modules bundled in the SDK") 376 377 subparsers = parser.add_subparsers() 378 379 parse_parser = subparsers.add_parser( 380 "parse", description="Parse mojom to AST and remove disabled definitions." 381 " Pickle pruned AST into output_dir.") 382 parse_parser.add_argument("filename", nargs="*", help="mojom input file") 383 parse_parser.add_argument("--filelist", help="mojom input file list") 384 parse_parser.add_argument( 385 "-o", 386 "--output_dir", 387 dest="output_dir", 388 default=".", 389 help="output directory for generated files") 390 parse_parser.add_argument( 391 "-d", "--depth", dest="depth", default=".", help="depth from source root") 392 parse_parser.add_argument( 393 "--enable_feature", 394 dest = "enabled_features", 395 default=[], 396 action="append", 397 help="Controls which definitions guarded by an EnabledIf attribute " 398 "will be enabled. If an EnabledIf attribute does not specify a value " 399 "that matches one of the enabled features, it will be disabled.") 400 parse_parser.set_defaults(func=_Parse) 401 402 generate_parser = subparsers.add_parser( 403 "generate", description="Generate bindings from mojom files.") 404 generate_parser.add_argument("filename", nargs="*", 405 help="mojom input file") 406 generate_parser.add_argument("--filelist", help="mojom input file list") 407 generate_parser.add_argument("-d", "--depth", dest="depth", default=".", 408 help="depth from source root") 409 generate_parser.add_argument("-o", "--output_dir", dest="output_dir", 410 default=".", 411 help="output directory for generated files") 412 generate_parser.add_argument("-g", "--generators", 413 dest="generators_string", 414 metavar="GENERATORS", 415 default="c++,javascript,java", 416 help="comma-separated list of generators") 417 generate_parser.add_argument( 418 "--gen_dir", dest="gen_directories", action="append", metavar="directory", 419 default=[], help="add a directory to be searched for the syntax trees.") 420 generate_parser.add_argument( 421 "-I", dest="import_directories", action="append", metavar="directory", 422 default=[], 423 help="add a directory to be searched for import files. The depth from " 424 "source root can be specified for each import by appending it after " 425 "a colon") 426 generate_parser.add_argument("--typemap", action="append", metavar="TYPEMAP", 427 default=[], dest="typemaps", 428 help="apply TYPEMAP to generated output") 429 generate_parser.add_argument("--variant", dest="variant", default=None, 430 help="output a named variant of the bindings") 431 generate_parser.add_argument( 432 "--bytecode_path", required=True, help=( 433 "the path from which to load template bytecode; to generate template " 434 "bytecode, run %s precompile BYTECODE_PATH" % os.path.basename( 435 sys.argv[0]))) 436 generate_parser.add_argument("--for_blink", action="store_true", 437 help="Use WTF types as generated types for mojo " 438 "string/array/map.") 439 generate_parser.add_argument( 440 "--use_once_callback", action="store_true", 441 help="Use base::OnceCallback instead of base::RepeatingCallback.") 442 generate_parser.add_argument( 443 "--js_bindings_mode", choices=["new", "both", "old"], default="new", 444 help="This option only affects the JavaScript bindings. The value could " 445 "be: \"new\" - generate only the new-style JS bindings, which use the " 446 "new module loading approach and the core api exposed by Web IDL; " 447 "\"both\" - generate both the old- and new-style bindings; \"old\" - " 448 "generate only the old-style bindings.") 449 generate_parser.add_argument( 450 "--export_attribute", default="", 451 help="Optional attribute to specify on class declaration to export it " 452 "for the component build.") 453 generate_parser.add_argument( 454 "--export_header", default="", 455 help="Optional header to include in the generated headers to support the " 456 "component build.") 457 generate_parser.add_argument( 458 "--generate_non_variant_code", action="store_true", 459 help="Generate code that is shared by different variants.") 460 generate_parser.add_argument( 461 "--scrambled_message_id_salt_path", 462 dest="scrambled_message_id_salt_paths", 463 help="If non-empty, the path to a file whose contents should be used as" 464 "a salt for generating scrambled message IDs. If this switch is specified" 465 "more than once, the contents of all salt files are concatenated to form" 466 "the salt value.", default=[], action="append") 467 generate_parser.add_argument( 468 "--support_lazy_serialization", 469 help="If set, generated bindings will serialize lazily when possible.", 470 action="store_true") 471 generate_parser.add_argument( 472 "--disallow_native_types", 473 help="Disallows the [Native] attribute to be specified on structs or " 474 "enums within the mojom file.", action="store_true") 475 generate_parser.add_argument( 476 "--disallow_interfaces", 477 help="Disallows interface definitions within the mojom file. It is an " 478 "error to specify this flag when processing a mojom file which defines " 479 "any interface.", action="store_true") 480 generate_parser.add_argument( 481 "--generate_message_ids", 482 help="Generates only the message IDs header for C++ bindings. Note that " 483 "this flag only matters if --generate_non_variant_code is also " 484 "specified.", action="store_true") 485 generate_parser.add_argument( 486 "--generate_fuzzing", 487 action="store_true", 488 help="Generates additional bindings for fuzzing in JS.") 489 generate_parser.set_defaults(func=_Generate) 490 491 precompile_parser = subparsers.add_parser("precompile", 492 description="Precompile templates for the mojom bindings generator.") 493 precompile_parser.add_argument( 494 "-o", "--output_dir", dest="output_dir", default=".", 495 help="output directory for precompiled templates") 496 precompile_parser.set_defaults(func=_Precompile) 497 498 verify_parser = subparsers.add_parser("verify", description="Checks " 499 "the set of imports against the set of dependencies.") 500 verify_parser.add_argument("filename", nargs="*", 501 help="mojom input file") 502 verify_parser.add_argument("--filelist", help="mojom input file list") 503 verify_parser.add_argument("-f", "--file", dest="deps_file", 504 help="file containing paths to the sources files for " 505 "dependencies") 506 verify_parser.add_argument("-g", "--gen_dir", 507 dest="gen_dir", 508 help="directory with the syntax tree") 509 verify_parser.add_argument( 510 "-d", "--depth", dest="depth", 511 help="depth from source root") 512 513 verify_parser.set_defaults(func=_VerifyImportDeps) 514 515 args, remaining_args = parser.parse_known_args() 516 return args.func(args, remaining_args) 517 518 519if __name__ == "__main__": 520 sys.exit(main()) 521