1"""BUILD rules for generating flatbuffer files.""" 2 3load("@build_bazel_rules_android//android:rules.bzl", "android_library") 4 5flatc_path = "@flatbuffers//:flatc" 6zip_files = "//tensorflow_lite_support/tools:zip_files" 7 8DEFAULT_INCLUDE_PATHS = [ 9 "./", 10 "$(GENDIR)", 11 "$(BINDIR)", 12] 13 14DEFAULT_FLATC_ARGS = [ 15 "--no-union-value-namespacing", 16 "--gen-object-api", 17] 18 19def flatbuffer_library_public( 20 name, 21 srcs, 22 outs, 23 language_flag, 24 out_prefix = "", 25 includes = [], 26 include_paths = [], 27 compatible_with = [], 28 flatc_args = DEFAULT_FLATC_ARGS, 29 reflection_name = "", 30 reflection_visibility = None, 31 output_to_bindir = False): 32 """Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler. 33 34 Outs: 35 filegroup(name): all generated source files. 36 Fileset([reflection_name]): (Optional) all generated reflection binaries. 37 38 Args: 39 name: Rule name. 40 srcs: Source .fbs files. Sent in order to the compiler. 41 outs: Output files from flatc. 42 language_flag: Target language flag. One of [-c, -j, -js]. 43 out_prefix: Prepend this path to the front of all generated files except on 44 single source targets. Usually is a directory name. 45 includes: Optional, list of filegroups of schemas that the srcs depend on. 46 include_paths: Optional, list of paths the includes files can be found in. 47 compatible_with: Optional, passed to genrule for environments this rule 48 can be built for. 49 flatc_args: Optional, list of additional arguments to pass to flatc. 50 reflection_name: Optional, if set this will generate the flatbuffer 51 reflection binaries for the schemas. 52 reflection_visibility: The visibility of the generated reflection Fileset. 53 output_to_bindir: Passed to genrule for output to bin directory. 54 """ 55 include_paths_cmd = ["-I %s" % (s) for s in include_paths] 56 57 # '$(@D)' when given a single source target will give the appropriate 58 # directory. Appending 'out_prefix' is only necessary when given a build 59 # target with multiple sources. 60 output_directory = ( 61 ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)") 62 ) 63 genrule_cmd = " ".join([ 64 "for f in $(SRCS); do", 65 "$(location %s)" % (flatc_path), 66 " ".join(flatc_args), 67 " ".join(include_paths_cmd), 68 language_flag, 69 output_directory, 70 "$$f;", 71 "done", 72 ]) 73 native.genrule( 74 name = name, 75 srcs = srcs, 76 outs = outs, 77 output_to_bindir = output_to_bindir, 78 compatible_with = compatible_with, 79 tools = includes + [flatc_path], 80 cmd = genrule_cmd, 81 message = "Generating flatbuffer files for %s:" % (name), 82 ) 83 if reflection_name: 84 reflection_genrule_cmd = " ".join([ 85 "for f in $(SRCS); do", 86 "$(location %s)" % (flatc_path), 87 "-b --schema", 88 " ".join(flatc_args), 89 " ".join(include_paths_cmd), 90 language_flag, 91 output_directory, 92 "$$f;", 93 "done", 94 ]) 95 reflection_outs = [ 96 (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) 97 for s in srcs 98 ] 99 native.genrule( 100 name = "%s_srcs" % reflection_name, 101 srcs = srcs, 102 outs = reflection_outs, 103 output_to_bindir = output_to_bindir, 104 compatible_with = compatible_with, 105 tools = includes + [flatc_path], 106 cmd = reflection_genrule_cmd, 107 message = "Generating flatbuffer reflection binary for %s:" % (name), 108 ) 109 # TODO(b/114456773): Make bazel rules proper and supported by flatbuffer 110 # Have to comment this since FilesetEntry is not supported in bazel 111 # skylark. 112 # native.Fileset( 113 # name = reflection_name, 114 # out = "%s_out" % reflection_name, 115 # entries = [ 116 # native.FilesetEntry(files = reflection_outs), 117 # ], 118 # visibility = reflection_visibility, 119 # compatible_with = compatible_with, 120 # ) 121 122def flatbuffer_cc_library( 123 name, 124 srcs, 125 srcs_filegroup_name = "", 126 out_prefix = "", 127 includes = [], 128 include_paths = [], 129 compatible_with = [], 130 flatc_args = DEFAULT_FLATC_ARGS, 131 visibility = None, 132 srcs_filegroup_visibility = None, 133 gen_reflections = False): 134 '''A cc_library with the generated reader/writers for the given flatbuffer definitions. 135 136 Outs: 137 filegroup([name]_srcs): all generated .h files. 138 filegroup(srcs_filegroup_name if specified, or [name]_includes if not): 139 Other flatbuffer_cc_library's can pass this in for their `includes` 140 parameter, if they depend on the schemas in this library. 141 Fileset([name]_reflection): (Optional) all generated reflection binaries. 142 cc_library([name]): library with sources and flatbuffers deps. 143 144 Remarks: 145 ** Because the genrule used to call flatc does not have any trivial way of 146 computing the output list of files transitively generated by includes and 147 --gen-includes (the default) being defined for flatc, the --gen-includes 148 flag will not work as expected. The way around this is to add a dependency 149 to the flatbuffer_cc_library defined alongside the flatc included Fileset. 150 For example you might define: 151 152 flatbuffer_cc_library( 153 name = "my_fbs", 154 srcs = [ "schemas/foo.fbs" ], 155 includes = [ "//third_party/bazz:bazz_fbs_includes" ], 156 ) 157 158 In which foo.fbs includes a few files from the Fileset defined at 159 //third_party/bazz:bazz_fbs_includes. When compiling the library that 160 includes foo_generated.h, and therefore has my_fbs as a dependency, it 161 will fail to find any of the bazz *_generated.h files unless you also 162 add bazz's flatbuffer_cc_library to your own dependency list, e.g.: 163 164 cc_library( 165 name = "my_lib", 166 deps = [ 167 ":my_fbs", 168 "//third_party/bazz:bazz_fbs" 169 ], 170 ) 171 172 Happy dependent Flatbuffering! 173 174 Args: 175 name: Rule name. 176 srcs: Source .fbs files. Sent in order to the compiler. 177 srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this 178 filegroup into the `includes` parameter of any other 179 flatbuffer_cc_library that depends on this one's schemas. 180 out_prefix: Prepend this path to the front of all generated files. Usually 181 is a directory name. 182 includes: Optional, list of filegroups of schemas that the srcs depend on. 183 ** SEE REMARKS BELOW ** 184 include_paths: Optional, list of paths the includes files can be found in. 185 compatible_with: Optional, passed to genrule for environments this rule 186 can be built for 187 flatc_args: Optional list of additional arguments to pass to flatc 188 (e.g. --gen-mutable). 189 visibility: The visibility of the generated cc_library. By default, use the 190 default visibility of the project. 191 srcs_filegroup_visibility: The visibility of the generated srcs filegroup. 192 By default, use the value of the visibility parameter above. 193 gen_reflections: Optional, if true this will generate the flatbuffer 194 reflection binaries for the schemas. 195 ''' 196 output_headers = [ 197 (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) 198 for s in srcs 199 ] 200 reflection_name = "%s_reflection" % name if gen_reflections else "" 201 202 flatbuffer_library_public( 203 name = "%s_srcs" % (name), 204 srcs = srcs, 205 outs = output_headers, 206 language_flag = "-c", 207 out_prefix = out_prefix, 208 includes = includes, 209 include_paths = include_paths, 210 compatible_with = compatible_with, 211 flatc_args = flatc_args, 212 reflection_name = reflection_name, 213 reflection_visibility = visibility, 214 ) 215 native.cc_library( 216 name = name, 217 hdrs = output_headers, 218 srcs = output_headers, 219 features = [ 220 "-parse_headers", 221 ], 222 deps = [ 223 "@flatbuffers//:runtime_cc", 224 ], 225 includes = ["."], 226 linkstatic = 1, 227 visibility = visibility, 228 compatible_with = compatible_with, 229 ) 230 231 # A filegroup for the `srcs`. That is, all the schema files for this 232 # Flatbuffer set. 233 native.filegroup( 234 name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name), 235 srcs = srcs, 236 visibility = srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility, 237 compatible_with = compatible_with, 238 ) 239 240# Custom provider to track dependencies transitively. 241FlatbufferInfo = provider( 242 fields = { 243 "transitive_srcs": "flatbuffer schema definitions.", 244 }, 245) 246 247def _flatbuffer_schemas_aspect_impl(target, ctx): 248 _ignore = [target] 249 transitive_srcs = depset() 250 if hasattr(ctx.rule.attr, "deps"): 251 for dep in ctx.rule.attr.deps: 252 if FlatbufferInfo in dep: 253 transitive_srcs = depset(dep[FlatbufferInfo].transitive_srcs, transitive = [transitive_srcs]) 254 if hasattr(ctx.rule.attr, "srcs"): 255 for src in ctx.rule.attr.srcs: 256 if FlatbufferInfo in src: 257 transitive_srcs = depset(src[FlatbufferInfo].transitive_srcs, transitive = [transitive_srcs]) 258 for f in src.files: 259 if f.extension == "fbs": 260 transitive_srcs = depset([f], transitive = [transitive_srcs]) 261 return [FlatbufferInfo(transitive_srcs = transitive_srcs)] 262 263# An aspect that runs over all dependencies and transitively collects 264# flatbuffer schema files. 265_flatbuffer_schemas_aspect = aspect( 266 attr_aspects = [ 267 "deps", 268 "srcs", 269 ], 270 implementation = _flatbuffer_schemas_aspect_impl, 271) 272 273# Rule to invoke the flatbuffer compiler. 274def _gen_flatbuffer_srcs_impl(ctx): 275 outputs = ctx.attr.outputs 276 include_paths = ctx.attr.include_paths 277 if ctx.attr.no_includes: 278 no_includes_statement = ["--no-includes"] 279 else: 280 no_includes_statement = [] 281 282 # Need to generate all files in a directory. 283 if not outputs: 284 outputs = [ctx.actions.declare_directory("{}_all".format(ctx.attr.name))] 285 output_directory = outputs[0].path 286 else: 287 outputs = [ctx.actions.declare_file(output) for output in outputs] 288 output_directory = outputs[0].dirname 289 290 deps = depset(ctx.files.srcs + ctx.files.deps, transitive = [ 291 dep[FlatbufferInfo].transitive_srcs 292 for dep in ctx.attr.deps 293 if FlatbufferInfo in dep 294 ]) 295 296 include_paths_cmd_line = [] 297 for s in include_paths: 298 include_paths_cmd_line.extend(["-I", s]) 299 300 for src in ctx.files.srcs: 301 ctx.actions.run( 302 inputs = deps, 303 outputs = outputs, 304 executable = ctx.executable._flatc, 305 arguments = [ 306 ctx.attr.language_flag, 307 "-o", 308 output_directory, 309 # Allow for absolute imports and referencing of generated files. 310 "-I", 311 "./", 312 "-I", 313 ctx.genfiles_dir.path, 314 "-I", 315 ctx.bin_dir.path, 316 ] + no_includes_statement + 317 include_paths_cmd_line + [ 318 "--no-union-value-namespacing", 319 "--gen-object-api", 320 src.path, 321 ], 322 progress_message = "Generating flatbuffer files for {}:".format(src), 323 ) 324 return [ 325 DefaultInfo(files = depset(outputs)), 326 ] 327 328_gen_flatbuffer_srcs = rule( 329 _gen_flatbuffer_srcs_impl, 330 attrs = { 331 "srcs": attr.label_list( 332 allow_files = [".fbs"], 333 mandatory = True, 334 ), 335 "outputs": attr.string_list( 336 default = [], 337 mandatory = False, 338 ), 339 "deps": attr.label_list( 340 default = [], 341 mandatory = False, 342 aspects = [_flatbuffer_schemas_aspect], 343 ), 344 "include_paths": attr.string_list( 345 default = [], 346 mandatory = False, 347 ), 348 "language_flag": attr.string( 349 mandatory = True, 350 ), 351 "no_includes": attr.bool( 352 default = False, 353 mandatory = False, 354 ), 355 "_flatc": attr.label( 356 default = Label("@flatbuffers//:flatc"), 357 executable = True, 358 cfg = "host", 359 ), 360 }, 361 output_to_genfiles = True, 362) 363 364def _concat_flatbuffer_py_srcs_impl(ctx): 365 # Merge all generated python files. The files are concatenated and the 366 # import statements are removed. Finally we import the flatbuffer runtime 367 # library. 368 command = "echo 'import flatbuffers\n' > %s; " 369 command += "for f in $(find %s -name '*.py'); do cat $f | sed '/import flatbuffers/d' >> %s; done " 370 ctx.actions.run_shell( 371 inputs = ctx.attr.deps[0].files, 372 outputs = [ctx.outputs.out], 373 command = command % ( 374 ctx.outputs.out.path, 375 ctx.attr.deps[0].files.to_list()[0].path, 376 ctx.outputs.out.path, 377 ), 378 ) 379 380_concat_flatbuffer_py_srcs = rule( 381 _concat_flatbuffer_py_srcs_impl, 382 attrs = { 383 "deps": attr.label_list(mandatory = True), 384 }, 385 output_to_genfiles = True, 386 outputs = {"out": "%{name}.py"}, 387) 388 389def flatbuffer_py_library( 390 name, 391 srcs, 392 deps = [], 393 include_paths = []): 394 """A py_library with the generated reader/writers for the given schema. 395 396 This rule assumes that the schema files define non-conflicting names, so that 397 they can be merged in a single file. This is e.g. the case if only a single 398 namespace is used. 399 The rule call the flatbuffer compiler for all schema files and merges the 400 generated python files into a single file that is wrapped in a py_library. 401 402 Args: 403 name: Rule name. (required) 404 srcs: List of source .fbs files. (required) 405 deps: List of dependencies. 406 include_paths: Optional, list of paths the includes files can be found in. 407 """ 408 all_srcs = "{}_srcs".format(name) 409 _gen_flatbuffer_srcs( 410 name = all_srcs, 411 srcs = srcs, 412 language_flag = "--python", 413 deps = deps, 414 include_paths = include_paths, 415 ) 416 all_srcs_no_include = "{}_srcs_no_include".format(name) 417 _gen_flatbuffer_srcs( 418 name = all_srcs_no_include, 419 srcs = srcs, 420 language_flag = "--python", 421 deps = deps, 422 no_includes = True, 423 include_paths = include_paths, 424 ) 425 concat_py_srcs = "{}_generated".format(name) 426 _concat_flatbuffer_py_srcs( 427 name = concat_py_srcs, 428 deps = [ 429 ":{}".format(all_srcs_no_include), 430 ], 431 ) 432 native.py_library( 433 name = name, 434 srcs = [ 435 ":{}".format(concat_py_srcs), 436 ], 437 srcs_version = "PY2AND3", 438 deps = deps, 439 ) 440 441def flatbuffer_java_library( 442 name, 443 srcs, 444 custom_package = "", 445 package_prefix = "", 446 include_paths = DEFAULT_INCLUDE_PATHS, 447 flatc_args = DEFAULT_FLATC_ARGS, 448 visibility = None): 449 """A java library with the generated reader/writers for the given flatbuffer definitions. 450 451 Args: 452 name: Rule name. (required) 453 srcs: List of source .fbs files including all includes. (required) 454 custom_package: Package name of generated Java files. If not specified 455 namespace in the schema files will be used. (optional) 456 package_prefix: like custom_package, but prefixes to the existing 457 namespace. (optional) 458 include_paths: List of paths that includes files can be found in. (optional) 459 flatc_args: List of additional arguments to pass to flatc. (optional) 460 visibility: Visibility setting for the java_library rule. (optional) 461 """ 462 out_srcjar = "java_%s_all.srcjar" % name 463 flatbuffer_java_srcjar( 464 name = "%s_srcjar" % name, 465 srcs = srcs, 466 out = out_srcjar, 467 custom_package = custom_package, 468 flatc_args = flatc_args, 469 include_paths = include_paths, 470 package_prefix = package_prefix, 471 ) 472 473 native.filegroup( 474 name = "%s.srcjar" % name, 475 srcs = [out_srcjar], 476 ) 477 478 native.java_library( 479 name = name, 480 srcs = [out_srcjar], 481 javacopts = ["-source 7 -target 7"], 482 deps = [ 483 "@flatbuffers//:runtime_java", 484 ], 485 visibility = visibility, 486 ) 487 488def flatbuffer_java_srcjar( 489 name, 490 srcs, 491 out, 492 custom_package = "", 493 package_prefix = "", 494 include_paths = DEFAULT_INCLUDE_PATHS, 495 flatc_args = DEFAULT_FLATC_ARGS): 496 """Generate flatbuffer Java source files. 497 498 Args: 499 name: Rule name. (required) 500 srcs: List of source .fbs files including all includes. (required) 501 out: Output file name. (required) 502 custom_package: Package name of generated Java files. If not specified 503 namespace in the schema files will be used. (optional) 504 package_prefix: like custom_package, but prefixes to the existing 505 namespace. (optional) 506 include_paths: List of paths that includes files can be found in. (optional) 507 flatc_args: List of additional arguments to pass to flatc. (optional) 508 """ 509 command_fmt = """set -e 510 tmpdir=$(@D) 511 schemas=$$tmpdir/schemas 512 java_root=$$tmpdir/java 513 rm -rf $$schemas 514 rm -rf $$java_root 515 mkdir -p $$schemas 516 mkdir -p $$java_root 517 518 for src in $(SRCS); do 519 dest=$$schemas/$$src 520 rm -rf $$(dirname $$dest) 521 mkdir -p $$(dirname $$dest) 522 if [ -z "{custom_package}" ] && [ -z "{package_prefix}" ]; then 523 cp -f $$src $$dest 524 else 525 if [ -z "{package_prefix}" ]; then 526 sed -e "s/namespace\\s.*/namespace {custom_package};/" $$src > $$dest 527 else 528 sed -e "s/namespace \\([^;]\\+\\);/namespace {package_prefix}.\\1;/" $$src > $$dest 529 fi 530 fi 531 done 532 533 flatc_arg_I="-I $$tmpdir/schemas" 534 for include_path in {include_paths}; do 535 flatc_arg_I="$$flatc_arg_I -I $$schemas/$$include_path" 536 done 537 538 flatc_additional_args= 539 for arg in {flatc_args}; do 540 flatc_additional_args="$$flatc_additional_args $$arg" 541 done 542 543 for src in $(SRCS); do 544 $(location {flatc_path}) $$flatc_arg_I --java $$flatc_additional_args -o $$java_root $$schemas/$$src 545 done 546 547 $(location {zip_files}) -export_zip_path=$@ -file_directory=$$java_root 548 """ 549 genrule_cmd = command_fmt.format( 550 package_name = native.package_name(), 551 custom_package = custom_package, 552 package_prefix = package_prefix, 553 flatc_path = flatc_path, 554 zip_files = zip_files, 555 include_paths = " ".join(include_paths), 556 flatc_args = " ".join(flatc_args), 557 ) 558 559 native.genrule( 560 name = name, 561 srcs = srcs, 562 outs = [out], 563 tools = [flatc_path, zip_files], 564 cmd = genrule_cmd, 565 ) 566 567def flatbuffer_android_library( 568 name, 569 srcs, 570 custom_package = "", 571 package_prefix = "", 572 include_paths = DEFAULT_INCLUDE_PATHS, 573 flatc_args = DEFAULT_FLATC_ARGS, 574 visibility = None): 575 """An android_library with the generated reader/writers for the given flatbuffer definitions. 576 577 Args: 578 name: Rule name. (required) 579 srcs: List of source .fbs files including all includes. (required) 580 custom_package: Package name of generated Java files. If not specified 581 namespace in the schema files will be used. (optional) 582 package_prefix: like custom_package, but prefixes to the existing 583 namespace. (optional) 584 include_paths: List of paths that includes files can be found in. (optional) 585 flatc_args: List of additional arguments to pass to flatc. (optional) 586 visibility: Visibility setting for the android_library rule. (optional) 587 """ 588 out_srcjar = "android_%s_all.srcjar" % name 589 flatbuffer_java_srcjar( 590 name = "%s_srcjar" % name, 591 srcs = srcs, 592 out = out_srcjar, 593 custom_package = custom_package, 594 flatc_args = flatc_args, 595 include_paths = include_paths, 596 package_prefix = package_prefix, 597 ) 598 599 native.filegroup( 600 name = "%s.srcjar" % name, 601 srcs = [out_srcjar], 602 ) 603 604 # To support org.checkerframework.dataflow.qual.Pure. 605 checkerframework_annotations = [ 606 "@org_checkerframework_qual", 607 ] if "--java-checkerframework" in flatc_args else [] 608 609 android_library( 610 name = name, 611 srcs = [out_srcjar], 612 javacopts = ["-source 7 -target 7"], 613 visibility = visibility, 614 deps = [ 615 "@flatbuffers//:runtime_android", 616 ] + checkerframework_annotations, 617 ) 618