1"""BUILD rules for generating flatbuffer files.""" 2 3load("@build_bazel_rules_android//android:rules.bzl", "android_library") 4 5flatc_path = "@flatbuffers//:flatc" 6zip_files = "//tensorflow/lite/tools:zip_files" 7 8DEFAULT_INCLUDE_PATHS = [ 9 "./", 10 "$(GENDIR)", 11 "$(BINDIR)", 12] 13 14DEFAULT_FLATC_ARGS = [ 15 "--no-union-value-namespacing", 16 "--gen-object-api", 17] 18 19def flatbuffer_library_public( 20 name, 21 srcs, 22 outs, 23 language_flag, 24 out_prefix = "", 25 includes = [], 26 include_paths = [], 27 compatible_with = [], 28 flatc_args = DEFAULT_FLATC_ARGS, 29 reflection_name = "", 30 reflection_visibility = None, 31 output_to_bindir = False): 32 """Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler. 33 34 Outs: 35 filegroup(name): all generated source files. 36 Fileset([reflection_name]): (Optional) all generated reflection binaries. 37 38 Args: 39 name: Rule name. 40 srcs: Source .fbs files. Sent in order to the compiler. 41 outs: Output files from flatc. 42 language_flag: Target language flag. One of [-c, -j, -js]. 43 out_prefix: Prepend this path to the front of all generated files except on 44 single source targets. Usually is a directory name. 45 includes: Optional, list of filegroups of schemas that the srcs depend on. 46 include_paths: Optional, list of paths the includes files can be found in. 47 compatible_with: Optional, passed to genrule for environments this rule 48 can be built for. 49 flatc_args: Optional, list of additional arguments to pass to flatc. 50 reflection_name: Optional, if set this will generate the flatbuffer 51 reflection binaries for the schemas. 52 reflection_visibility: The visibility of the generated reflection Fileset. 53 output_to_bindir: Passed to genrule for output to bin directory. 54 """ 55 include_paths_cmd = ["-I %s" % (s) for s in include_paths] 56 57 # '$(@D)' when given a single source target will give the appropriate 58 # directory. Appending 'out_prefix' is only necessary when given a build 59 # target with multiple sources. 60 output_directory = ( 61 ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)") 62 ) 63 genrule_cmd = " ".join([ 64 "for f in $(SRCS); do", 65 "$(location %s)" % (flatc_path), 66 " ".join(flatc_args), 67 " ".join(include_paths_cmd), 68 language_flag, 69 output_directory, 70 "$$f;", 71 "done", 72 ]) 73 native.genrule( 74 name = name, 75 srcs = srcs, 76 outs = outs, 77 output_to_bindir = output_to_bindir, 78 compatible_with = compatible_with, 79 tools = includes + [flatc_path], 80 cmd = genrule_cmd, 81 message = "Generating flatbuffer files for %s:" % (name), 82 ) 83 if reflection_name: 84 reflection_genrule_cmd = " ".join([ 85 "for f in $(SRCS); do", 86 "$(location %s)" % (flatc_path), 87 "-b --schema", 88 " ".join(flatc_args), 89 " ".join(include_paths_cmd), 90 language_flag, 91 output_directory, 92 "$$f;", 93 "done", 94 ]) 95 reflection_outs = [ 96 (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) 97 for s in srcs 98 ] 99 native.genrule( 100 name = "%s_srcs" % reflection_name, 101 srcs = srcs, 102 outs = reflection_outs, 103 output_to_bindir = output_to_bindir, 104 compatible_with = compatible_with, 105 tools = includes + [flatc_path], 106 cmd = reflection_genrule_cmd, 107 message = "Generating flatbuffer reflection binary for %s:" % (name), 108 ) 109 # TODO(b/114456773): Make bazel rules proper and supported by flatbuffer 110 # Have to comment this since FilesetEntry is not supported in bazel 111 # skylark. 112 # native.Fileset( 113 # name = reflection_name, 114 # out = "%s_out" % reflection_name, 115 # entries = [ 116 # native.FilesetEntry(files = reflection_outs), 117 # ], 118 # visibility = reflection_visibility, 119 # compatible_with = compatible_with, 120 # ) 121 122def flatbuffer_cc_library( 123 name, 124 srcs, 125 srcs_filegroup_name = "", 126 out_prefix = "", 127 includes = [], 128 include_paths = [], 129 compatible_with = [], 130 flatc_args = DEFAULT_FLATC_ARGS, 131 visibility = None, 132 srcs_filegroup_visibility = None, 133 gen_reflections = False): 134 '''A cc_library with the generated reader/writers for the given flatbuffer definitions. 135 136 Outs: 137 filegroup([name]_srcs): all generated .h files. 138 filegroup(srcs_filegroup_name if specified, or [name]_includes if not): 139 Other flatbuffer_cc_library's can pass this in for their `includes` 140 parameter, if they depend on the schemas in this library. 141 Fileset([name]_reflection): (Optional) all generated reflection binaries. 142 cc_library([name]): library with sources and flatbuffers deps. 143 144 Remarks: 145 ** Because the genrule used to call flatc does not have any trivial way of 146 computing the output list of files transitively generated by includes and 147 --gen-includes (the default) being defined for flatc, the --gen-includes 148 flag will not work as expected. The way around this is to add a dependency 149 to the flatbuffer_cc_library defined alongside the flatc included Fileset. 150 For example you might define: 151 152 flatbuffer_cc_library( 153 name = "my_fbs", 154 srcs = [ "schemas/foo.fbs" ], 155 includes = [ "//third_party/bazz:bazz_fbs_includes" ], 156 ) 157 158 In which foo.fbs includes a few files from the Fileset defined at 159 //third_party/bazz:bazz_fbs_includes. When compiling the library that 160 includes foo_generated.h, and therefore has my_fbs as a dependency, it 161 will fail to find any of the bazz *_generated.h files unless you also 162 add bazz's flatbuffer_cc_library to your own dependency list, e.g.: 163 164 cc_library( 165 name = "my_lib", 166 deps = [ 167 ":my_fbs", 168 "//third_party/bazz:bazz_fbs" 169 ], 170 ) 171 172 Happy dependent Flatbuffering! 173 174 Args: 175 name: Rule name. 176 srcs: Source .fbs files. Sent in order to the compiler. 177 srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this 178 filegroup into the `includes` parameter of any other 179 flatbuffer_cc_library that depends on this one's schemas. 180 out_prefix: Prepend this path to the front of all generated files. Usually 181 is a directory name. 182 includes: Optional, list of filegroups of schemas that the srcs depend on. 183 ** SEE REMARKS BELOW ** 184 include_paths: Optional, list of paths the includes files can be found in. 185 compatible_with: Optional, passed to genrule for environments this rule 186 can be built for 187 flatc_args: Optional list of additional arguments to pass to flatc 188 (e.g. --gen-mutable). 189 visibility: The visibility of the generated cc_library. By default, use the 190 default visibility of the project. 191 srcs_filegroup_visibility: The visibility of the generated srcs filegroup. 192 By default, use the value of the visibility parameter above. 193 gen_reflections: Optional, if true this will generate the flatbuffer 194 reflection binaries for the schemas. 195 ''' 196 output_headers = [ 197 (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) 198 for s in srcs 199 ] 200 reflection_name = "%s_reflection" % name if gen_reflections else "" 201 202 flatbuffer_library_public( 203 name = "%s_srcs" % (name), 204 srcs = srcs, 205 outs = output_headers, 206 language_flag = "-c", 207 out_prefix = out_prefix, 208 includes = includes, 209 include_paths = include_paths, 210 compatible_with = compatible_with, 211 flatc_args = flatc_args, 212 reflection_name = reflection_name, 213 reflection_visibility = visibility, 214 ) 215 native.cc_library( 216 name = name, 217 hdrs = output_headers, 218 srcs = output_headers, 219 features = [ 220 "-parse_headers", 221 ], 222 deps = [ 223 "@flatbuffers//:runtime_cc", 224 ], 225 includes = ["."], 226 linkstatic = 1, 227 visibility = visibility, 228 compatible_with = compatible_with, 229 ) 230 231 # A filegroup for the `srcs`. That is, all the schema files for this 232 # Flatbuffer set. 233 native.filegroup( 234 name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name), 235 srcs = srcs, 236 visibility = srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility, 237 compatible_with = compatible_with, 238 ) 239 240# Custom provider to track dependencies transitively. 241FlatbufferInfo = provider( 242 fields = { 243 "transitive_srcs": "flatbuffer schema definitions.", 244 }, 245) 246 247def _flatbuffer_schemas_aspect_impl(target, ctx): 248 _ignore = [target] 249 transitive_srcs = depset() 250 if hasattr(ctx.rule.attr, "deps"): 251 for dep in ctx.rule.attr.deps: 252 if FlatbufferInfo in dep: 253 transitive_srcs = depset(dep[FlatbufferInfo].transitive_srcs, transitive = [transitive_srcs]) 254 if hasattr(ctx.rule.attr, "srcs"): 255 for src in ctx.rule.attr.srcs: 256 if FlatbufferInfo in src: 257 transitive_srcs = depset(src[FlatbufferInfo].transitive_srcs, transitive = [transitive_srcs]) 258 for f in src.files: 259 if f.extension == "fbs": 260 transitive_srcs = depset([f], transitive = [transitive_srcs]) 261 return [FlatbufferInfo(transitive_srcs = transitive_srcs)] 262 263# An aspect that runs over all dependencies and transitively collects 264# flatbuffer schema files. 265_flatbuffer_schemas_aspect = aspect( 266 attr_aspects = [ 267 "deps", 268 "srcs", 269 ], 270 implementation = _flatbuffer_schemas_aspect_impl, 271) 272 273# Rule to invoke the flatbuffer compiler. 274def _gen_flatbuffer_srcs_impl(ctx): 275 outputs = ctx.attr.outputs 276 include_paths = ctx.attr.include_paths 277 if ctx.attr.no_includes: 278 no_includes_statement = ["--no-includes"] 279 else: 280 no_includes_statement = [] 281 282 if ctx.attr.language_flag == "--python": 283 onefile_statement = ["--gen-onefile"] 284 else: 285 onefile_statement = [] 286 287 # Need to generate all files in a directory. 288 if not outputs: 289 outputs = [ctx.actions.declare_directory("{}_all".format(ctx.attr.name))] 290 output_directory = outputs[0].path 291 else: 292 outputs = [ctx.actions.declare_file(output) for output in outputs] 293 output_directory = outputs[0].dirname 294 295 deps = depset(ctx.files.srcs + ctx.files.deps, transitive = [ 296 dep[FlatbufferInfo].transitive_srcs 297 for dep in ctx.attr.deps 298 if FlatbufferInfo in dep 299 ]) 300 301 include_paths_cmd_line = [] 302 for s in include_paths: 303 include_paths_cmd_line.extend(["-I", s]) 304 305 for src in ctx.files.srcs: 306 ctx.actions.run( 307 inputs = deps, 308 outputs = outputs, 309 executable = ctx.executable._flatc, 310 arguments = [ 311 ctx.attr.language_flag, 312 "-o", 313 output_directory, 314 # Allow for absolute imports and referencing of generated files. 315 "-I", 316 "./", 317 "-I", 318 ctx.genfiles_dir.path, 319 "-I", 320 ctx.bin_dir.path, 321 ] + no_includes_statement + 322 onefile_statement + 323 include_paths_cmd_line + [ 324 "--no-union-value-namespacing", 325 "--gen-object-api", 326 src.path, 327 ], 328 progress_message = "Generating flatbuffer files for {}:".format(src), 329 use_default_shell_env = True, 330 ) 331 return [ 332 DefaultInfo(files = depset(outputs)), 333 ] 334 335_gen_flatbuffer_srcs = rule( 336 _gen_flatbuffer_srcs_impl, 337 attrs = { 338 "srcs": attr.label_list( 339 allow_files = [".fbs"], 340 mandatory = True, 341 ), 342 "outputs": attr.string_list( 343 default = [], 344 mandatory = False, 345 ), 346 "deps": attr.label_list( 347 default = [], 348 mandatory = False, 349 aspects = [_flatbuffer_schemas_aspect], 350 ), 351 "include_paths": attr.string_list( 352 default = [], 353 mandatory = False, 354 ), 355 "language_flag": attr.string( 356 mandatory = True, 357 ), 358 "no_includes": attr.bool( 359 default = False, 360 mandatory = False, 361 ), 362 "_flatc": attr.label( 363 default = Label("@flatbuffers//:flatc"), 364 executable = True, 365 cfg = "host", 366 ), 367 }, 368 output_to_genfiles = True, 369) 370 371def flatbuffer_py_strip_prefix_srcs(name, srcs = [], strip_prefix = ""): 372 """Strips path prefix. 373 374 Args: 375 name: Rule name. (required) 376 srcs: Source .py files. (required) 377 strip_prefix: Path that needs to be stripped from the srcs filepaths. (required) 378 """ 379 for src in srcs: 380 native.genrule( 381 name = name + "_" + src.replace(".", "_").replace("/", "_"), 382 srcs = [src], 383 outs = [src.replace(strip_prefix, "")], 384 cmd = "cp $< $@", 385 ) 386 387def _concat_flatbuffer_py_srcs_impl(ctx): 388 # Merge all generated python files. The files are concatenated and import 389 # statements are removed. Finally we import the flatbuffer runtime library. 390 # IMPORTANT: Our Windows shell does not support "find ... -exec" properly. 391 # If you're changing the commandline below, please build wheels and run smoke 392 # tests on all the three operating systems. 393 command = "echo 'import flatbuffers\n' > %s; " 394 command += "for f in $(find %s -name '*.py' | sort); do cat $f | sed '/import flatbuffers/d' >> %s; done " 395 ctx.actions.run_shell( 396 inputs = ctx.attr.deps[0].files, 397 outputs = [ctx.outputs.out], 398 command = command % ( 399 ctx.outputs.out.path, 400 ctx.attr.deps[0].files.to_list()[0].path, 401 ctx.outputs.out.path, 402 ), 403 use_default_shell_env = True, 404 ) 405 406_concat_flatbuffer_py_srcs = rule( 407 _concat_flatbuffer_py_srcs_impl, 408 attrs = { 409 "deps": attr.label_list(mandatory = True), 410 }, 411 output_to_genfiles = True, 412 outputs = {"out": "%{name}.py"}, 413) 414 415def flatbuffer_py_library( 416 name, 417 srcs, 418 deps = [], 419 include_paths = []): 420 """A py_library with the generated reader/writers for the given schema. 421 422 This rule assumes that the schema files define non-conflicting names, so that 423 they can be merged in a single file. This is e.g. the case if only a single 424 namespace is used. 425 The rule call the flatbuffer compiler for all schema files and merges the 426 generated python files into a single file that is wrapped in a py_library. 427 428 Args: 429 name: Rule name. (required) 430 srcs: List of source .fbs files. (required) 431 deps: List of dependencies. 432 include_paths: Optional, list of paths the includes files can be found in. 433 """ 434 all_srcs = "{}_srcs".format(name) 435 _gen_flatbuffer_srcs( 436 name = all_srcs, 437 srcs = srcs, 438 language_flag = "--python", 439 deps = deps, 440 include_paths = include_paths, 441 ) 442 443 # TODO(b/235550563): Remove the concatnation rule with 2.0.6 update. 444 all_srcs_no_include = "{}_srcs_no_include".format(name) 445 _gen_flatbuffer_srcs( 446 name = all_srcs_no_include, 447 srcs = srcs, 448 language_flag = "--python", 449 deps = deps, 450 no_includes = True, 451 include_paths = include_paths, 452 ) 453 concat_py_srcs = "{}_generated".format(name) 454 _concat_flatbuffer_py_srcs( 455 name = concat_py_srcs, 456 deps = [ 457 ":{}".format(all_srcs_no_include), 458 ], 459 ) 460 native.py_library( 461 name = name, 462 srcs = [ 463 ":{}".format(concat_py_srcs), 464 ], 465 srcs_version = "PY3", 466 deps = deps + [ 467 "@flatbuffers//:runtime_py", 468 ], 469 ) 470 471def flatbuffer_java_library( 472 name, 473 srcs, 474 custom_package = "", 475 package_prefix = "", 476 include_paths = DEFAULT_INCLUDE_PATHS, 477 flatc_args = DEFAULT_FLATC_ARGS, 478 visibility = None): 479 """A java library with the generated reader/writers for the given flatbuffer definitions. 480 481 Args: 482 name: Rule name. (required) 483 srcs: List of source .fbs files including all includes. (required) 484 custom_package: Package name of generated Java files. If not specified 485 namespace in the schema files will be used. (optional) 486 package_prefix: like custom_package, but prefixes to the existing 487 namespace. (optional) 488 include_paths: List of paths that includes files can be found in. (optional) 489 flatc_args: List of additional arguments to pass to flatc. (optional) 490 visibility: Visibility setting for the java_library rule. (optional) 491 """ 492 out_srcjar = "java_%s_all.srcjar" % name 493 flatbuffer_java_srcjar( 494 name = "%s_srcjar" % name, 495 srcs = srcs, 496 out = out_srcjar, 497 custom_package = custom_package, 498 flatc_args = flatc_args, 499 include_paths = include_paths, 500 package_prefix = package_prefix, 501 ) 502 503 native.filegroup( 504 name = "%s.srcjar" % name, 505 srcs = [out_srcjar], 506 ) 507 508 native.java_library( 509 name = name, 510 srcs = [out_srcjar], 511 javacopts = ["-source 7 -target 7"], 512 deps = [ 513 "@flatbuffers//:runtime_java", 514 ], 515 visibility = visibility, 516 ) 517 518def flatbuffer_java_srcjar( 519 name, 520 srcs, 521 out, 522 custom_package = "", 523 package_prefix = "", 524 include_paths = DEFAULT_INCLUDE_PATHS, 525 flatc_args = DEFAULT_FLATC_ARGS): 526 """Generate flatbuffer Java source files. 527 528 Args: 529 name: Rule name. (required) 530 srcs: List of source .fbs files including all includes. (required) 531 out: Output file name. (required) 532 custom_package: Package name of generated Java files. If not specified 533 namespace in the schema files will be used. (optional) 534 package_prefix: like custom_package, but prefixes to the existing 535 namespace. (optional) 536 include_paths: List of paths that includes files can be found in. (optional) 537 flatc_args: List of additional arguments to pass to flatc. (optional) 538 """ 539 command_fmt = """set -e 540 tmpdir=$(@D) 541 schemas=$$tmpdir/schemas 542 java_root=$$tmpdir/java 543 rm -rf $$schemas 544 rm -rf $$java_root 545 mkdir -p $$schemas 546 mkdir -p $$java_root 547 548 for src in $(SRCS); do 549 dest=$$schemas/$$src 550 rm -rf $$(dirname $$dest) 551 mkdir -p $$(dirname $$dest) 552 if [ -z "{custom_package}" ] && [ -z "{package_prefix}" ]; then 553 cp -f $$src $$dest 554 else 555 if [ -z "{package_prefix}" ]; then 556 sed -e "s/namespace\\s.*/namespace {custom_package};/" $$src > $$dest 557 else 558 sed -e "s/namespace \\([^;]\\+\\);/namespace {package_prefix}.\\1;/" $$src > $$dest 559 fi 560 fi 561 done 562 563 flatc_arg_I="-I $$tmpdir/schemas" 564 for include_path in {include_paths}; do 565 flatc_arg_I="$$flatc_arg_I -I $$schemas/$$include_path" 566 done 567 568 flatc_additional_args= 569 for arg in {flatc_args}; do 570 flatc_additional_args="$$flatc_additional_args $$arg" 571 done 572 573 for src in $(SRCS); do 574 $(location {flatc_path}) $$flatc_arg_I --java $$flatc_additional_args -o $$java_root $$schemas/$$src 575 done 576 577 $(location {zip_files}) -export_zip_path=$@ -file_directory=$$java_root 578 """ 579 genrule_cmd = command_fmt.format( 580 package_name = native.package_name(), 581 custom_package = custom_package, 582 package_prefix = package_prefix, 583 flatc_path = flatc_path, 584 zip_files = zip_files, 585 include_paths = " ".join(include_paths), 586 flatc_args = " ".join(flatc_args), 587 ) 588 589 native.genrule( 590 name = name, 591 srcs = srcs, 592 outs = [out], 593 tools = [flatc_path, zip_files], 594 cmd = genrule_cmd, 595 ) 596 597def flatbuffer_android_library( 598 name, 599 srcs, 600 custom_package = "", 601 package_prefix = "", 602 include_paths = DEFAULT_INCLUDE_PATHS, 603 flatc_args = DEFAULT_FLATC_ARGS, 604 visibility = None): 605 """An android_library with the generated reader/writers for the given flatbuffer definitions. 606 607 Args: 608 name: Rule name. (required) 609 srcs: List of source .fbs files including all includes. (required) 610 custom_package: Package name of generated Java files. If not specified 611 namespace in the schema files will be used. (optional) 612 package_prefix: like custom_package, but prefixes to the existing 613 namespace. (optional) 614 include_paths: List of paths that includes files can be found in. (optional) 615 flatc_args: List of additional arguments to pass to flatc. (optional) 616 visibility: Visibility setting for the android_library rule. (optional) 617 """ 618 out_srcjar = "android_%s_all.srcjar" % name 619 flatbuffer_java_srcjar( 620 name = "%s_srcjar" % name, 621 srcs = srcs, 622 out = out_srcjar, 623 custom_package = custom_package, 624 flatc_args = flatc_args, 625 include_paths = include_paths, 626 package_prefix = package_prefix, 627 ) 628 629 native.filegroup( 630 name = "%s.srcjar" % name, 631 srcs = [out_srcjar], 632 ) 633 634 # To support org.checkerframework.dataflow.qual.Pure. 635 checkerframework_annotations = [ 636 "@org_checkerframework_qual", 637 ] if "--java-checkerframework" in flatc_args else [] 638 639 android_library( 640 name = name, 641 srcs = [out_srcjar], 642 javacopts = ["-source 7 -target 7"], 643 visibility = visibility, 644 deps = [ 645 "@flatbuffers//:runtime_android", 646 ] + checkerframework_annotations, 647 ) 648