1"""Module extension for generating third-party crates for use in bazel.""" 2 3load("@bazel_features//:features.bzl", "bazel_features") 4load("@bazel_skylib//lib:structs.bzl", "structs") 5load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") 6load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 7load("//crate_universe:defs.bzl", _crate_universe_crate = "crate") 8load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest") 9load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config") 10load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS") 11load("//crate_universe/private/module_extensions:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner", "get_host_cargo_rustc") 12load("//rust/platform:triple.bzl", "get_host_triple") 13 14# A list of labels which may be relative (and if so, is within the repo the rule is generated in). 15# 16# If I were to write ":foo", with attr.label_list, it would evaluate to 17# "@@//:foo". However, for a tag such as deps, ":foo" should refer to 18# "@@rules_rust~crates~<crate>//:foo". 19_relative_label_list = attr.string_list 20 21_OPT_BOOL_VALUES = { 22 "auto": None, 23 "off": False, 24 "on": True, 25} 26 27def optional_bool(doc): 28 return attr.string( 29 doc = doc, 30 values = _OPT_BOOL_VALUES.keys(), 31 default = "auto", 32 ) 33 34def _get_or_insert(d, key, value): 35 if key not in d: 36 d[key] = value 37 return d[key] 38 39def _generate_repo_impl(repo_ctx): 40 for path, contents in repo_ctx.attr.contents.items(): 41 repo_ctx.file(path, contents) 42 43_generate_repo = repository_rule( 44 implementation = _generate_repo_impl, 45 attrs = dict( 46 contents = attr.string_dict(mandatory = True), 47 ), 48) 49 50def _annotations_for_repo(module_annotations, repo_specific_annotations): 51 """Merges the set of global annotations with the repo-specific ones 52 53 Args: 54 module_annotations (dict): The annotation tags that apply to all repos, keyed by crate. 55 repo_specific_annotations (dict): The annotation tags that apply to only this repo, keyed by crate. 56 """ 57 58 if not repo_specific_annotations: 59 return module_annotations 60 61 annotations = dict(module_annotations) 62 for crate, values in repo_specific_annotations.items(): 63 _get_or_insert(annotations, crate, []).extend(values) 64 return annotations 65 66def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = None, manifests = {}, packages = {}): 67 """Generates repositories for the transitive closure of crates defined by manifests and packages. 68 69 Args: 70 module_ctx (module_ctx): The module context object. 71 cargo_bazel (function): A function that can be called to execute cargo_bazel. 72 cfg (object): The module tag from `from_cargo` or `from_specs` 73 annotations (dict): The set of annotation tag classes that apply to this closure, keyed by crate name. 74 cargo_lockfile (path): Path to Cargo.lock, if we have one. This is optional for `from_specs` closures. 75 manifests (dict): The set of Cargo.toml manifests that apply to this closure, if any, keyed by path. 76 packages (dict): The set of extra cargo crate tags that apply to this closure, if any, keyed by package name. 77 """ 78 79 tag_path = module_ctx.path(cfg.name) 80 81 rendering_config = json.decode(render_config( 82 regen_command = "Run 'cargo update [--workspace]'", 83 )) 84 config_file = tag_path.get_child("config.json") 85 module_ctx.file( 86 config_file, 87 executable = False, 88 content = generate_config_file( 89 module_ctx, 90 mode = "remote", 91 annotations = annotations, 92 generate_build_scripts = cfg.generate_build_scripts, 93 supported_platform_triples = cfg.supported_platform_triples, 94 generate_target_compatible_with = True, 95 repository_name = cfg.name, 96 output_pkg = cfg.name, 97 workspace_name = cfg.name, 98 generate_binaries = cfg.generate_binaries, 99 render_config = rendering_config, 100 repository_ctx = module_ctx, 101 ), 102 ) 103 104 splicing_manifest = tag_path.get_child("splicing_manifest.json") 105 module_ctx.file( 106 splicing_manifest, 107 executable = False, 108 content = generate_splicing_manifest( 109 packages = packages, 110 splicing_config = "", 111 cargo_config = cfg.cargo_config, 112 manifests = manifests, 113 manifest_to_path = module_ctx.path, 114 ), 115 ) 116 117 splicing_output_dir = tag_path.get_child("splicing-output") 118 splice_args = [ 119 "splice", 120 "--output-dir", 121 splicing_output_dir, 122 "--config", 123 config_file, 124 "--splicing-manifest", 125 splicing_manifest, 126 ] 127 if cargo_lockfile: 128 splice_args.extend([ 129 "--cargo-lockfile", 130 cargo_lockfile, 131 ]) 132 cargo_bazel(splice_args) 133 134 # Create a lockfile, since we need to parse it to generate spoke 135 # repos. 136 lockfile_path = tag_path.get_child("lockfile.json") 137 module_ctx.file(lockfile_path, "") 138 139 cargo_bazel([ 140 "generate", 141 "--cargo-lockfile", 142 cargo_lockfile or splicing_output_dir.get_child("Cargo.lock"), 143 "--config", 144 config_file, 145 "--splicing-manifest", 146 splicing_manifest, 147 "--repository-dir", 148 tag_path, 149 "--metadata", 150 splicing_output_dir.get_child("metadata.json"), 151 "--repin", 152 "--lockfile", 153 lockfile_path, 154 ]) 155 156 crates_dir = tag_path.get_child(cfg.name) 157 _generate_repo( 158 name = cfg.name, 159 contents = { 160 "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")), 161 "defs.bzl": module_ctx.read(crates_dir.get_child("defs.bzl")), 162 }, 163 ) 164 165 contents = json.decode(module_ctx.read(lockfile_path)) 166 167 for crate in contents["crates"].values(): 168 repo = crate["repository"] 169 if repo == None: 170 continue 171 name = crate["name"] 172 version = crate["version"] 173 174 # "+" isn't valid in a repo name. 175 crate_repo_name = "{repo_name}__{name}-{version}".format( 176 repo_name = cfg.name, 177 name = name, 178 version = version.replace("+", "-"), 179 ) 180 181 build_file_content = module_ctx.read(crates_dir.get_child("BUILD.%s-%s.bazel" % (name, version))) 182 if "Http" in repo: 183 # Replicates functionality in repo_http.j2. 184 repo = repo["Http"] 185 http_archive( 186 name = crate_repo_name, 187 patch_args = repo.get("patch_args", None), 188 patch_tool = repo.get("patch_tool", None), 189 patches = repo.get("patches", None), 190 remote_patch_strip = 1, 191 sha256 = repo.get("sha256", None), 192 type = "tar.gz", 193 urls = [repo["url"]], 194 strip_prefix = "%s-%s" % (crate["name"], crate["version"]), 195 build_file_content = build_file_content, 196 ) 197 elif "Git" in repo: 198 # Replicates functionality in repo_git.j2 199 repo = repo["Git"] 200 kwargs = {} 201 for k, v in repo["commitish"].items(): 202 if k == "Rev": 203 kwargs["commit"] = v 204 else: 205 kwargs[k.lower()] = v 206 new_git_repository( 207 name = crate_repo_name, 208 init_submodules = True, 209 patch_args = repo.get("patch_args", None), 210 patch_tool = repo.get("patch_tool", None), 211 patches = repo.get("patches", None), 212 shallow_since = repo.get("shallow_since", None), 213 remote = repo["remote"], 214 build_file_content = build_file_content, 215 strip_prefix = repo.get("strip_prefix", None), 216 **kwargs 217 ) 218 else: 219 fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) 220 221def _package_to_json(p): 222 # Avoid adding unspecified properties. 223 # If we add them as empty strings, cargo-bazel will be unhappy. 224 return json.encode({ 225 k: v 226 for k, v in structs.to_dict(p).items() 227 if v or k == "default_features" 228 }) 229 230def _get_generator(module_ctx): 231 """Query Network Resources to local a `cargo-bazel` binary. 232 233 Based off get_generator in crates_universe/private/generate_utils.bzl 234 235 Args: 236 module_ctx (module_ctx): The rules context object 237 238 Returns: 239 tuple(path, dict) The path to a 'cargo-bazel' binary. The pairing (dict) 240 may be `None` if there is not need to update the attribute 241 """ 242 host_triple = get_host_triple(module_ctx) 243 use_environ = False 244 for var in GENERATOR_ENV_VARS: 245 if var in module_ctx.os.environ: 246 use_environ = True 247 248 if use_environ: 249 generator_sha256 = module_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_SHA256) 250 generator_url = module_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_URL) 251 elif len(CARGO_BAZEL_URLS) == 0: 252 return module_ctx.path(Label("@cargo_bazel_bootstrap//:cargo-bazel")) 253 else: 254 generator_sha256 = CARGO_BAZEL_SHA256S.get(host_triple.str) 255 generator_url = CARGO_BAZEL_URLS.get(host_triple.str) 256 257 if not generator_url: 258 fail(( 259 "No generator URL was found either in the `CARGO_BAZEL_GENERATOR_URL` " + 260 "environment variable or for the `{}` triple in the `generator_urls` attribute" 261 ).format(host_triple.str)) 262 263 output = module_ctx.path("cargo-bazel.exe" if "win" in module_ctx.os.name else "cargo-bazel") 264 265 # Download the file into place 266 download_kwargs = { 267 "executable": True, 268 "output": output, 269 "url": generator_url, 270 } 271 272 if generator_sha256: 273 download_kwargs.update({"sha256": generator_sha256}) 274 275 module_ctx.download(**download_kwargs) 276 return output 277 278def _crate_impl(module_ctx): 279 # Preload all external repositories. Calling `module_ctx.path` will cause restarts of the implementation 280 # function of the module extension, so we want to trigger all restarts before we start the actual work. 281 # Once https://github.com/bazelbuild/bazel/issues/22729 has been fixed, this code can be removed. 282 get_host_cargo_rustc(module_ctx) 283 for mod in module_ctx.modules: 284 for cfg in mod.tags.from_cargo: 285 module_ctx.path(cfg.cargo_lockfile) 286 for m in cfg.manifests: 287 module_ctx.path(m) 288 289 cargo_bazel_output = _get_generator(module_ctx) 290 cargo_bazel = get_cargo_bazel_runner(module_ctx, cargo_bazel_output) 291 292 all_repos = [] 293 reproducible = True 294 295 for mod in module_ctx.modules: 296 module_annotations = {} 297 repo_specific_annotations = {} 298 for annotation_tag in mod.tags.annotation: 299 annotation_dict = structs.to_dict(annotation_tag) 300 repositories = annotation_dict.pop("repositories") 301 crate = annotation_dict.pop("crate") 302 303 # The crate.annotation function can take in either a list or a bool. 304 # For the tag-based method, because it has type safety, we have to 305 # split it into two parameters. 306 if annotation_dict.pop("gen_all_binaries"): 307 annotation_dict["gen_binaries"] = True 308 annotation_dict["gen_build_script"] = _OPT_BOOL_VALUES[annotation_dict["gen_build_script"]] 309 310 # Process the override targets for the annotation. 311 # In the non-bzlmod approach, this is given as a dict 312 # with the possible keys "`proc_macro`, `build_script`, `lib`, `bin`". 313 # With the tag-based approach used in Bzlmod, we run into an issue 314 # where there is no dict type that takes a string as a key and a Label as the value. 315 # To work around this, we split the override option into four, and reconstruct the 316 # dictionary here during processing 317 annotation_dict["override_targets"] = dict() 318 replacement = annotation_dict.pop("override_target_lib") 319 if replacement: 320 annotation_dict["override_targets"]["lib"] = str(replacement) 321 322 replacement = annotation_dict.pop("override_target_proc_macro") 323 if replacement: 324 annotation_dict["override_targets"]["proc_macro"] = str(replacement) 325 326 replacement = annotation_dict.pop("override_target_build_script") 327 if replacement: 328 annotation_dict["override_targets"]["build_script"] = str(replacement) 329 330 replacement = annotation_dict.pop("override_target_bin") 331 if replacement: 332 annotation_dict["override_targets"]["bin"] = str(replacement) 333 334 annotation = _crate_universe_crate.annotation(**{ 335 k: v 336 for k, v in annotation_dict.items() 337 # Tag classes can't take in None, but the function requires None 338 # instead of the empty values in many cases. 339 # https://github.com/bazelbuild/bazel/issues/20744 340 if v != "" and v != [] and v != {} 341 }) 342 if not repositories: 343 _get_or_insert(module_annotations, crate, []).append(annotation) 344 for repo in repositories: 345 _get_or_insert( 346 _get_or_insert(repo_specific_annotations, repo, {}), 347 crate, 348 [], 349 ).append(annotation) 350 351 local_repos = [] 352 353 for cfg in mod.tags.from_cargo + mod.tags.from_specs: 354 if cfg.name in local_repos: 355 fail("Defined two crate universes with the same name in the same MODULE.bazel file. Use the name tag to give them different names.") 356 elif cfg.name in all_repos: 357 fail("Defined two crate universes with the same name in different MODULE.bazel files. Either give one a different name, or use use_extension(isolate=True)") 358 all_repos.append(cfg.name) 359 local_repos.append(cfg.name) 360 361 for cfg in mod.tags.from_cargo: 362 annotations = _annotations_for_repo( 363 module_annotations, 364 repo_specific_annotations.get(cfg.name), 365 ) 366 367 cargo_lockfile = module_ctx.path(cfg.cargo_lockfile) 368 manifests = {str(module_ctx.path(m)): str(m) for m in cfg.manifests} 369 _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, cargo_lockfile = cargo_lockfile, manifests = manifests) 370 371 for cfg in mod.tags.from_specs: 372 # We don't have a Cargo.lock so the resolution can change. 373 # We could maybe make this reproducible by using `-minimal-version` during resolution. 374 # See https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#minimal-versions 375 reproducible = False 376 377 annotations = _annotations_for_repo( 378 module_annotations, 379 repo_specific_annotations.get(cfg.name), 380 ) 381 382 packages = {p.package: _package_to_json(p) for p in mod.tags.spec} 383 _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations, packages = packages) 384 385 for repo in repo_specific_annotations: 386 if repo not in local_repos: 387 fail("Annotation specified for repo %s, but the module defined repositories %s" % (repo, local_repos)) 388 389 metadata_kwargs = {} 390 if bazel_features.external_deps.extension_metadata_has_reproducible: 391 metadata_kwargs["reproducible"] = reproducible 392 393 return module_ctx.extension_metadata(**metadata_kwargs) 394 395_from_cargo = tag_class( 396 doc = "Generates a repo @crates from a Cargo.toml / Cargo.lock pair", 397 attrs = dict( 398 name = attr.string( 399 doc = "The name of the repo to generate", 400 default = "crates", 401 ), 402 cargo_lockfile = CRATES_VENDOR_ATTRS["cargo_lockfile"], 403 manifests = CRATES_VENDOR_ATTRS["manifests"], 404 cargo_config = CRATES_VENDOR_ATTRS["cargo_config"], 405 generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"], 406 generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"], 407 supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"], 408 ), 409) 410 411# This should be kept in sync with crate_universe/private/crate.bzl. 412_annotation = tag_class( 413 attrs = dict( 414 repositories = attr.string_list( 415 doc = "A list of repository names specified from `crate.from_cargo(name=...)` that this annotation is applied to. Defaults to all repositories.", 416 default = [], 417 ), 418 crate = attr.string( 419 doc = "The name of the crate the annotation is applied to", 420 mandatory = True, 421 ), 422 version = attr.string( 423 doc = "The versions of the crate the annotation is applied to. Defaults to all versions.", 424 default = "*", 425 ), 426 additive_build_file_content = attr.string( 427 doc = "Extra contents to write to the bottom of generated BUILD files.", 428 ), 429 additive_build_file = attr.label( 430 doc = "A file containing extra contents to write to the bottom of generated BUILD files.", 431 ), 432 alias_rule = attr.string( 433 doc = "Alias rule to use instead of `native.alias()`. Overrides [render_config](#render_config)'s 'default_alias_rule'.", 434 ), 435 build_script_data = _relative_label_list( 436 doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute.", 437 ), 438 build_script_tools = _relative_label_list( 439 doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute.", 440 ), 441 build_script_data_glob = attr.string_list( 442 doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute", 443 ), 444 build_script_deps = _relative_label_list( 445 doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute.", 446 ), 447 build_script_env = attr.string_dict( 448 doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", 449 ), 450 build_script_proc_macro_deps = _relative_label_list( 451 doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute.", 452 ), 453 build_script_rundir = attr.string( 454 doc = "An override for the build script's rundir attribute.", 455 ), 456 build_script_rustc_env = attr.string_dict( 457 doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute.", 458 ), 459 build_script_toolchains = attr.label_list( 460 doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute.", 461 ), 462 compile_data = _relative_label_list( 463 doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute.", 464 ), 465 compile_data_glob = attr.string_list( 466 doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute.", 467 ), 468 crate_features = attr.string_list( 469 doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute.", 470 ), 471 data = _relative_label_list( 472 doc = "A list of labels to add to a crate's `rust_library::data` attribute.", 473 ), 474 data_glob = attr.string_list( 475 doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute.", 476 ), 477 deps = _relative_label_list( 478 doc = "A list of labels to add to a crate's `rust_library::deps` attribute.", 479 ), 480 extra_aliased_targets = attr.string_dict( 481 doc = "A list of targets to add to the generated aliases in the root crate_universe repository.", 482 ), 483 gen_binaries = attr.string_list( 484 doc = "As a list, the subset of the crate's bins that should get `rust_binary` targets produced.", 485 ), 486 gen_all_binaries = attr.bool( 487 doc = "If true, generates `rust_binary` targets for all of the crates bins", 488 ), 489 disable_pipelining = attr.bool( 490 doc = "If True, disables pipelining for library targets for this crate.", 491 ), 492 gen_build_script = attr.string( 493 doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate. Supported values are 'on', 'off', and 'auto'.", 494 values = _OPT_BOOL_VALUES.keys(), 495 default = "auto", 496 ), 497 patch_args = attr.string_list( 498 doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)", 499 ), 500 patch_tool = attr.string( 501 doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)", 502 ), 503 patches = attr.label_list( 504 doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)", 505 ), 506 proc_macro_deps = _relative_label_list( 507 doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute.", 508 ), 509 rustc_env = attr.string_dict( 510 doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute.", 511 ), 512 rustc_env_files = _relative_label_list( 513 doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute.", 514 ), 515 rustc_flags = attr.string_list( 516 doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute.", 517 ), 518 shallow_since = attr.string( 519 doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code.", 520 ), 521 override_target_lib = attr.label( 522 doc = "An optional alternate taget to use when something depends on this crate to allow the parent repo to provide its own version of this dependency.", 523 ), 524 override_target_proc_macro = attr.label( 525 doc = "An optional alternate taget to use when something depends on this crate to allow the parent repo to provide its own version of this dependency.", 526 ), 527 override_target_build_script = attr.label( 528 doc = "An optional alternate taget to use when something depends on this crate to allow the parent repo to provide its own version of this dependency.", 529 ), 530 override_target_bin = attr.label( 531 doc = "An optional alternate taget to use when something depends on this crate to allow the parent repo to provide its own version of this dependency.", 532 ), 533 ), 534) 535 536_from_specs = tag_class( 537 doc = "Generates a repo @crates from the defined `spec` tags", 538 attrs = dict( 539 name = attr.string(doc = "The name of the repo to generate", default = "crates"), 540 cargo_config = CRATES_VENDOR_ATTRS["cargo_config"], 541 generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"], 542 generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"], 543 supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"], 544 ), 545) 546 547# This should be kept in sync with crate_universe/private/crate.bzl. 548_spec = tag_class( 549 attrs = dict( 550 package = attr.string( 551 doc = "The explicit name of the package.", 552 mandatory = True, 553 ), 554 version = attr.string( 555 doc = "The exact version of the crate. Cannot be used with `git`.", 556 ), 557 artifact = attr.string( 558 doc = "Set to 'bin' to pull in a binary crate as an artifact dependency. Requires a nightly Cargo.", 559 ), 560 lib = attr.bool( 561 doc = "If using `artifact = 'bin'`, additionally setting `lib = True` declares a dependency on both the package's library and binary, as opposed to just the binary.", 562 ), 563 default_features = attr.bool( 564 doc = "Maps to the `default-features` flag.", 565 default = True, 566 ), 567 features = attr.string_list( 568 doc = "A list of features to use for the crate.", 569 ), 570 git = attr.string( 571 doc = "The Git url to use for the crate. Cannot be used with `version`.", 572 ), 573 branch = attr.string( 574 doc = "The git branch of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.", 575 ), 576 tag = attr.string( 577 doc = "The git tag of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified. Specifying `rev` is recommended for fully-reproducible builds.", 578 ), 579 rev = attr.string( 580 doc = "The git revision of the remote crate. Tied with the `git` param. Only one of branch, tag or rev may be specified.", 581 ), 582 ), 583) 584 585crate = module_extension( 586 implementation = _crate_impl, 587 tag_classes = dict( 588 from_cargo = _from_cargo, 589 annotation = _annotation, 590 from_specs = _from_specs, 591 spec = _spec, 592 ), 593) 594