1# Copyright 2023 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"pip module extension for use with bzlmod" 16 17load("@bazel_features//:features.bzl", "bazel_features") 18load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") 19load("//python/private:auth.bzl", "AUTH_ATTRS") 20load("//python/private:normalize_name.bzl", "normalize_name") 21load("//python/private:repo_utils.bzl", "repo_utils") 22load("//python/private:semver.bzl", "semver") 23load("//python/private:version_label.bzl", "version_label") 24load(":attrs.bzl", "use_isolated") 25load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") 26load(":hub_repository.bzl", "hub_repository") 27load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") 28load(":parse_whl_name.bzl", "parse_whl_name") 29load(":pip_repository_attrs.bzl", "ATTRS") 30load(":render_pkg_aliases.bzl", "whl_alias") 31load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") 32load(":simpleapi_download.bzl", "simpleapi_download") 33load(":whl_library.bzl", "whl_library") 34load(":whl_repo_name.bzl", "whl_repo_name") 35 36def _major_minor_version(version): 37 version = semver(version) 38 return "{}.{}".format(version.major, version.minor) 39 40def _whl_mods_impl(mctx): 41 """Implementation of the pip.whl_mods tag class. 42 43 This creates the JSON files used to modify the creation of different wheels. 44""" 45 whl_mods_dict = {} 46 for mod in mctx.modules: 47 for whl_mod_attr in mod.tags.whl_mods: 48 if whl_mod_attr.hub_name not in whl_mods_dict.keys(): 49 whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr} 50 elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys(): 51 # We cannot have the same wheel name in the same hub, as we 52 # will create the same JSON file name. 53 fail("""\ 54Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( 55 whl_mod_attr.whl_name, 56 whl_mod_attr.hub_name, 57 )) 58 else: 59 whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr 60 61 for hub_name, whl_maps in whl_mods_dict.items(): 62 whl_mods = {} 63 64 # create a struct that we can pass to the _whl_mods_repo rule 65 # to create the different JSON files. 66 for whl_name, mods in whl_maps.items(): 67 build_content = mods.additive_build_content 68 if mods.additive_build_content_file != None and mods.additive_build_content != "": 69 fail("""\ 70You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. 71""") 72 elif mods.additive_build_content_file != None: 73 build_content = mctx.read(mods.additive_build_content_file) 74 75 whl_mods[whl_name] = json.encode(struct( 76 additive_build_content = build_content, 77 copy_files = mods.copy_files, 78 copy_executables = mods.copy_executables, 79 data = mods.data, 80 data_exclude_glob = mods.data_exclude_glob, 81 srcs_exclude_glob = mods.srcs_exclude_glob, 82 )) 83 84 _whl_mods_repo( 85 name = hub_name, 86 whl_mods = whl_mods, 87 ) 88 89def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache, exposed_packages): 90 logger = repo_utils.logger(module_ctx, "pypi:create_whl_repos") 91 python_interpreter_target = pip_attr.python_interpreter_target 92 is_hub_reproducible = True 93 94 # if we do not have the python_interpreter set in the attributes 95 # we programmatically find it. 96 hub_name = pip_attr.hub_name 97 if python_interpreter_target == None and not pip_attr.python_interpreter: 98 python_name = "python_{}_host".format( 99 pip_attr.python_version.replace(".", "_"), 100 ) 101 if python_name not in INTERPRETER_LABELS: 102 fail(( 103 "Unable to find interpreter for pip hub '{hub_name}' for " + 104 "python_version={version}: Make sure a corresponding " + 105 '`python.toolchain(python_version="{version}")` call exists.' + 106 "Expected to find {python_name} among registered versions:\n {labels}" 107 ).format( 108 hub_name = hub_name, 109 version = pip_attr.python_version, 110 python_name = python_name, 111 labels = " \n".join(INTERPRETER_LABELS), 112 )) 113 python_interpreter_target = INTERPRETER_LABELS[python_name] 114 115 pip_name = "{}_{}".format( 116 hub_name, 117 version_label(pip_attr.python_version), 118 ) 119 major_minor = _major_minor_version(pip_attr.python_version) 120 121 if hub_name not in whl_map: 122 whl_map[hub_name] = {} 123 124 whl_modifications = {} 125 if pip_attr.whl_modifications != None: 126 for mod, whl_name in pip_attr.whl_modifications.items(): 127 whl_modifications[whl_name] = mod 128 129 if pip_attr.experimental_requirement_cycles: 130 requirement_cycles = { 131 name: [normalize_name(whl_name) for whl_name in whls] 132 for name, whls in pip_attr.experimental_requirement_cycles.items() 133 } 134 135 whl_group_mapping = { 136 whl_name: group_name 137 for group_name, group_whls in requirement_cycles.items() 138 for whl_name in group_whls 139 } 140 141 # TODO @aignas 2024-04-05: how do we support different requirement 142 # cycles for different abis/oses? For now we will need the users to 143 # assume the same groups across all versions/platforms until we start 144 # using an alternative cycle resolution strategy. 145 group_map[hub_name] = pip_attr.experimental_requirement_cycles 146 else: 147 whl_group_mapping = {} 148 requirement_cycles = {} 149 150 # Create a new wheel library for each of the different whls 151 152 get_index_urls = None 153 if pip_attr.experimental_index_url: 154 if pip_attr.download_only: 155 fail("Currently unsupported to use `download_only` and `experimental_index_url`") 156 157 get_index_urls = lambda ctx, distributions: simpleapi_download( 158 ctx, 159 attr = struct( 160 index_url = pip_attr.experimental_index_url, 161 extra_index_urls = pip_attr.experimental_extra_index_urls or [], 162 index_url_overrides = pip_attr.experimental_index_url_overrides or {}, 163 sources = distributions, 164 envsubst = pip_attr.envsubst, 165 # Auth related info 166 netrc = pip_attr.netrc, 167 auth_patterns = pip_attr.auth_patterns, 168 ), 169 cache = simpleapi_cache, 170 parallel_download = pip_attr.parallel_download, 171 ) 172 173 requirements_by_platform = parse_requirements( 174 module_ctx, 175 requirements_by_platform = requirements_files_by_platform( 176 requirements_by_platform = pip_attr.requirements_by_platform, 177 requirements_linux = pip_attr.requirements_linux, 178 requirements_lock = pip_attr.requirements_lock, 179 requirements_osx = pip_attr.requirements_darwin, 180 requirements_windows = pip_attr.requirements_windows, 181 extra_pip_args = pip_attr.extra_pip_args, 182 python_version = major_minor, 183 logger = logger, 184 ), 185 get_index_urls = get_index_urls, 186 # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either 187 # in the PATH or if specified as a label. We will configure the env 188 # markers when evaluating the requirement lines based on the output 189 # from the `requirements_files_by_platform` which should have something 190 # similar to: 191 # { 192 # "//:requirements.txt": ["cp311_linux_x86_64", ...] 193 # } 194 # 195 # We know the target python versions that we need to evaluate the 196 # markers for and thus we don't need to use multiple python interpreter 197 # instances to perform this manipulation. This function should be executed 198 # only once by the underlying code to minimize the overhead needed to 199 # spin up a Python interpreter. 200 evaluate_markers = lambda module_ctx, requirements: evaluate_markers( 201 module_ctx, 202 requirements = requirements, 203 python_interpreter = pip_attr.python_interpreter, 204 python_interpreter_target = python_interpreter_target, 205 srcs = pip_attr._evaluate_markers_srcs, 206 logger = logger, 207 ), 208 logger = logger, 209 ) 210 211 repository_platform = host_platform(module_ctx) 212 for whl_name, requirements in requirements_by_platform.items(): 213 # We are not using the "sanitized name" because the user 214 # would need to guess what name we modified the whl name 215 # to. 216 annotation = whl_modifications.get(whl_name) 217 whl_name = normalize_name(whl_name) 218 219 group_name = whl_group_mapping.get(whl_name) 220 group_deps = requirement_cycles.get(group_name, []) 221 222 # Construct args separately so that the lock file can be smaller and does not include unused 223 # attrs. 224 whl_library_args = dict( 225 repo = pip_name, 226 dep_template = "@{}//{{name}}:{{target}}".format(hub_name), 227 ) 228 maybe_args = dict( 229 # The following values are safe to omit if they have false like values 230 annotation = annotation, 231 download_only = pip_attr.download_only, 232 enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, 233 environment = pip_attr.environment, 234 envsubst = pip_attr.envsubst, 235 experimental_target_platforms = pip_attr.experimental_target_platforms, 236 group_deps = group_deps, 237 group_name = group_name, 238 pip_data_exclude = pip_attr.pip_data_exclude, 239 python_interpreter = pip_attr.python_interpreter, 240 python_interpreter_target = python_interpreter_target, 241 whl_patches = { 242 p: json.encode(args) 243 for p, args in whl_overrides.get(whl_name, {}).items() 244 }, 245 ) 246 whl_library_args.update({k: v for k, v in maybe_args.items() if v}) 247 maybe_args_with_default = dict( 248 # The following values have defaults next to them 249 isolated = (use_isolated(module_ctx, pip_attr), True), 250 quiet = (pip_attr.quiet, True), 251 timeout = (pip_attr.timeout, 600), 252 ) 253 whl_library_args.update({ 254 k: v 255 for k, (v, default) in maybe_args_with_default.items() 256 if v != default 257 }) 258 259 if get_index_urls: 260 # TODO @aignas 2024-05-26: move to a separate function 261 found_something = False 262 is_exposed = False 263 for requirement in requirements: 264 is_exposed = is_exposed or requirement.is_exposed 265 for distribution in requirement.whls + [requirement.sdist]: 266 if not distribution: 267 # sdist may be None 268 continue 269 270 found_something = True 271 is_hub_reproducible = False 272 273 if pip_attr.netrc: 274 whl_library_args["netrc"] = pip_attr.netrc 275 if pip_attr.auth_patterns: 276 whl_library_args["auth_patterns"] = pip_attr.auth_patterns 277 278 # pip is not used to download wheels and the python `whl_library` helpers are only extracting things 279 whl_library_args.pop("extra_pip_args", None) 280 281 # This is no-op because pip is not used to download the wheel. 282 whl_library_args.pop("download_only", None) 283 284 repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) 285 whl_library_args["requirement"] = requirement.srcs.requirement 286 whl_library_args["urls"] = [distribution.url] 287 whl_library_args["sha256"] = distribution.sha256 288 whl_library_args["filename"] = distribution.filename 289 whl_library_args["experimental_target_platforms"] = requirement.target_platforms 290 291 # Pure python wheels or sdists may need to have a platform here 292 target_platforms = None 293 if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): 294 if len(requirements) > 1: 295 target_platforms = requirement.target_platforms 296 297 whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) 298 299 whl_map[hub_name].setdefault(whl_name, []).append( 300 whl_alias( 301 repo = repo_name, 302 version = major_minor, 303 filename = distribution.filename, 304 target_platforms = target_platforms, 305 ), 306 ) 307 308 if found_something: 309 if is_exposed: 310 exposed_packages.setdefault(hub_name, {})[whl_name] = None 311 continue 312 313 requirement = select_requirement( 314 requirements, 315 platform = None if pip_attr.download_only else repository_platform, 316 ) 317 if not requirement: 318 # Sometimes the package is not present for host platform if there 319 # are whls specified only in particular requirements files, in that 320 # case just continue, however, if the download_only flag is set up, 321 # then the user can also specify the target platform of the wheel 322 # packages they want to download, in that case there will be always 323 # a requirement here, so we will not be in this code branch. 324 continue 325 elif get_index_urls: 326 logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) 327 328 whl_library_args["requirement"] = requirement.requirement_line 329 if requirement.extra_pip_args: 330 whl_library_args["extra_pip_args"] = requirement.extra_pip_args 331 332 # We sort so that the lock-file remains the same no matter the order of how the 333 # args are manipulated in the code going before. 334 repo_name = "{}_{}".format(pip_name, whl_name) 335 whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) 336 whl_map[hub_name].setdefault(whl_name, []).append( 337 whl_alias( 338 repo = repo_name, 339 version = major_minor, 340 ), 341 ) 342 343 return is_hub_reproducible 344 345def _pip_impl(module_ctx): 346 """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. 347 348 This implementation iterates through all of the `pip.parse` calls and creates 349 different pip hub repositories based on the "hub_name". Each of the 350 pip calls create spoke repos that uses a specific Python interpreter. 351 352 In a MODULES.bazel file we have: 353 354 pip.parse( 355 hub_name = "pip", 356 python_version = 3.9, 357 requirements_lock = "//:requirements_lock_3_9.txt", 358 requirements_windows = "//:requirements_windows_3_9.txt", 359 ) 360 pip.parse( 361 hub_name = "pip", 362 python_version = 3.10, 363 requirements_lock = "//:requirements_lock_3_10.txt", 364 requirements_windows = "//:requirements_windows_3_10.txt", 365 ) 366 367 For instance, we have a hub with the name of "pip". 368 A repository named the following is created. It is actually called last when 369 all of the pip spokes are collected. 370 371 - @@rules_python~override~pip~pip 372 373 As shown in the example code above we have the following. 374 Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". 375 These definitions create two different pip spoke repositories that are 376 related to the hub "pip". 377 One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically 378 determines the Python version and the interpreter. 379 Both of these pip spokes contain requirements files that includes websocket 380 and its dependencies. 381 382 We also need repositories for the wheels that the different pip spokes contain. 383 For each Python version a different wheel repository is created. In our example 384 each pip spoke had a requirements file that contained websockets. We 385 then create two different wheel repositories that are named the following. 386 387 - @@rules_python~override~pip~pip_39_websockets 388 - @@rules_python~override~pip~pip_310_websockets 389 390 And if the wheel has any other dependencies subsequent wheels are created in the same fashion. 391 392 The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to 393 a spoke repository depending on the Python version. 394 395 Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple 396 hubs pointing to various different pip spokes. 397 398 Some other business rules notes. A hub can only have one spoke per Python version. We cannot 399 have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second 400 we cannot have the same hub name used in sub-modules. The hub name has to be globally 401 unique. 402 403 This implementation also handles the creation of whl_modification JSON files that are used 404 during the creation of wheel libraries. These JSON files used via the annotations argument 405 when calling wheel_installer.py. 406 407 Args: 408 module_ctx: module contents 409 """ 410 411 # Build all of the wheel modifications if the tag class is called. 412 _whl_mods_impl(module_ctx) 413 414 _overriden_whl_set = {} 415 whl_overrides = {} 416 417 for module in module_ctx.modules: 418 for attr in module.tags.override: 419 if not module.is_root: 420 fail("overrides are only supported in root modules") 421 422 if not attr.file.endswith(".whl"): 423 fail("Only whl overrides are supported at this time") 424 425 whl_name = normalize_name(parse_whl_name(attr.file).distribution) 426 427 if attr.file in _overriden_whl_set: 428 fail("Duplicate module overrides for '{}'".format(attr.file)) 429 _overriden_whl_set[attr.file] = None 430 431 for patch in attr.patches: 432 if whl_name not in whl_overrides: 433 whl_overrides[whl_name] = {} 434 435 if patch not in whl_overrides[whl_name]: 436 whl_overrides[whl_name][patch] = struct( 437 patch_strip = attr.patch_strip, 438 whls = [], 439 ) 440 441 whl_overrides[whl_name][patch].whls.append(attr.file) 442 443 # Used to track all the different pip hubs and the spoke pip Python 444 # versions. 445 pip_hub_map = {} 446 447 # Keeps track of all the hub's whl repos across the different versions. 448 # dict[hub, dict[whl, dict[version, str pip]]] 449 # Where hub, whl, and pip are the repo names 450 hub_whl_map = {} 451 hub_group_map = {} 452 exposed_packages = {} 453 454 simpleapi_cache = {} 455 is_extension_reproducible = True 456 457 for mod in module_ctx.modules: 458 for pip_attr in mod.tags.parse: 459 hub_name = pip_attr.hub_name 460 if hub_name not in pip_hub_map: 461 pip_hub_map[pip_attr.hub_name] = struct( 462 module_name = mod.name, 463 python_versions = [pip_attr.python_version], 464 ) 465 elif pip_hub_map[hub_name].module_name != mod.name: 466 # We cannot have two hubs with the same name in different 467 # modules. 468 fail(( 469 "Duplicate cross-module pip hub named '{hub}': pip hub " + 470 "names must be unique across modules. First defined " + 471 "by module '{first_module}', second attempted by " + 472 "module '{second_module}'" 473 ).format( 474 hub = hub_name, 475 first_module = pip_hub_map[hub_name].module_name, 476 second_module = mod.name, 477 )) 478 479 elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: 480 fail(( 481 "Duplicate pip python version '{version}' for hub " + 482 "'{hub}' in module '{module}': the Python versions " + 483 "used for a hub must be unique" 484 ).format( 485 hub = hub_name, 486 module = mod.name, 487 version = pip_attr.python_version, 488 )) 489 else: 490 pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) 491 492 is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache, exposed_packages) 493 is_extension_reproducible = is_extension_reproducible and is_hub_reproducible 494 495 for hub_name, whl_map in hub_whl_map.items(): 496 hub_repository( 497 name = hub_name, 498 repo_name = hub_name, 499 whl_map = { 500 key: json.encode(value) 501 for key, value in whl_map.items() 502 }, 503 default_version = _major_minor_version(DEFAULT_PYTHON_VERSION), 504 packages = sorted(exposed_packages.get(hub_name, {})), 505 groups = hub_group_map.get(hub_name), 506 ) 507 508 if bazel_features.external_deps.extension_metadata_has_reproducible: 509 # If we are not using the `experimental_index_url feature, the extension is fully 510 # deterministic and we don't need to create a lock entry for it. 511 # 512 # In order to be able to dogfood the `experimental_index_url` feature before it gets 513 # stabilized, we have created the `_pip_non_reproducible` function, that will result 514 # in extra entries in the lock file. 515 return module_ctx.extension_metadata(reproducible = is_extension_reproducible) 516 else: 517 return None 518 519def _pip_non_reproducible(module_ctx): 520 _pip_impl(module_ctx) 521 522 # We default to calling the PyPI index and that will go into the 523 # MODULE.bazel.lock file, hence return nothing here. 524 return None 525 526def _pip_parse_ext_attrs(**kwargs): 527 """Get the attributes for the pip extension. 528 529 Args: 530 **kwargs: A kwarg for setting defaults for the specific attributes. The 531 key is expected to be the same as the attribute key. 532 533 Returns: 534 A dict of attributes. 535 """ 536 attrs = dict({ 537 "experimental_extra_index_urls": attr.string_list( 538 doc = """\ 539The extra index URLs to use for downloading wheels using bazel downloader. 540Each value is going to be subject to `envsubst` substitutions if necessary. 541 542The indexes must support Simple API as described here: 543https://packaging.python.org/en/latest/specifications/simple-repository-api/ 544 545This is equivalent to `--extra-index-urls` `pip` option. 546""", 547 default = [], 548 ), 549 "experimental_index_url": attr.string( 550 default = kwargs.get("experimental_index_url", ""), 551 doc = """\ 552The index URL to use for downloading wheels using bazel downloader. This value is going 553to be subject to `envsubst` substitutions if necessary. 554 555The indexes must support Simple API as described here: 556https://packaging.python.org/en/latest/specifications/simple-repository-api/ 557 558In the future this could be defaulted to `https://pypi.org` when this feature becomes 559stable. 560 561This is equivalent to `--index-url` `pip` option. 562""", 563 ), 564 "experimental_index_url_overrides": attr.string_dict( 565 doc = """\ 566The index URL overrides for each package to use for downloading wheels using 567bazel downloader. This value is going to be subject to `envsubst` substitutions 568if necessary. 569 570The key is the package name (will be normalized before usage) and the value is the 571index URL. 572 573This design pattern has been chosen in order to be fully deterministic about which 574packages come from which source. We want to avoid issues similar to what happened in 575https://pytorch.org/blog/compromised-nightly-dependency/. 576 577The indexes must support Simple API as described here: 578https://packaging.python.org/en/latest/specifications/simple-repository-api/ 579""", 580 ), 581 "hub_name": attr.string( 582 mandatory = True, 583 doc = """ 584The name of the repo pip dependencies will be accessible from. 585 586This name must be unique between modules; unless your module is guaranteed to 587always be the root module, it's highly recommended to include your module name 588in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can 589be used for shorter local names within your module. 590 591Within a module, the same `hub_name` can be specified to group different Python 592versions of pip dependencies under one repository name. This allows using a 593Python version-agnostic name when referring to pip dependencies; the 594correct version will be automatically selected. 595 596Typically, a module will only have a single hub of pip dependencies, but this 597is not required. Each hub is a separate resolution of pip dependencies. This 598means if different programs need different versions of some library, separate 599hubs can be created, and each program can use its respective hub's targets. 600Targets from different hubs should not be used together. 601""", 602 ), 603 "parallel_download": attr.bool( 604 doc = """\ 605The flag allows to make use of parallel downloading feature in bazel 7.1 and above 606when the bazel downloader is used. This is by default enabled as it improves the 607performance by a lot, but in case the queries to the simple API are very expensive 608or when debugging authentication issues one may want to disable this feature. 609 610NOTE, This will download (potentially duplicate) data for multiple packages if 611there is more than one index available, but in general this should be negligible 612because the simple API calls are very cheap and the user should not notice any 613extra overhead. 614 615If we are in synchronous mode, then we will use the first result that we 616find in case extra indexes are specified. 617""", 618 default = True, 619 ), 620 "python_version": attr.string( 621 mandatory = True, 622 doc = """ 623The Python version the dependencies are targetting, in Major.Minor format 624(e.g., "3.11") or patch level granularity (e.g. "3.11.1"). 625 626If an interpreter isn't explicitly provided (using `python_interpreter` or 627`python_interpreter_target`), then the version specified here must have 628a corresponding `python.toolchain()` configured. 629""", 630 ), 631 "whl_modifications": attr.label_keyed_string_dict( 632 mandatory = False, 633 doc = """\ 634A dict of labels to wheel names that is typically generated by the whl_modifications. 635The labels are JSON config files describing the modifications. 636""", 637 ), 638 "_evaluate_markers_srcs": attr.label_list( 639 default = EVALUATE_MARKERS_SRCS, 640 doc = """\ 641The list of labels to use as SRCS for the marker evaluation code. This ensures that the 642code will be re-evaluated when any of files in the default changes. 643""", 644 ), 645 }, **ATTRS) 646 attrs.update(AUTH_ATTRS) 647 648 return attrs 649 650def _whl_mod_attrs(): 651 attrs = { 652 "additive_build_content": attr.string( 653 doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", 654 ), 655 "additive_build_content_file": attr.label( 656 doc = """\ 657(label, optional): path to a BUILD file to add to the generated 658`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file 659arguments at the same time.""", 660 ), 661 "copy_executables": attr.string_dict( 662 doc = """\ 663(dict, optional): A mapping of `src` and `out` files for 664[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as 665executable.""", 666 ), 667 "copy_files": attr.string_dict( 668 doc = """\ 669(dict, optional): A mapping of `src` and `out` files for 670[@bazel_skylib//rules:copy_file.bzl][cf]""", 671 ), 672 "data": attr.string_list( 673 doc = """\ 674(list, optional): A list of labels to add as `data` dependencies to 675the generated `py_library` target.""", 676 ), 677 "data_exclude_glob": attr.string_list( 678 doc = """\ 679(list, optional): A list of exclude glob patterns to add as `data` to 680the generated `py_library` target.""", 681 ), 682 "hub_name": attr.string( 683 doc = """\ 684Name of the whl modification, hub we use this name to set the modifications for 685pip.parse. If you have different pip hubs you can use a different name, 686otherwise it is best practice to just use one. 687 688You cannot have the same `hub_name` in different modules. You can reuse the same 689name in the same module for different wheels that you put in the same hub, but you 690cannot have a child module that uses the same `hub_name`. 691""", 692 mandatory = True, 693 ), 694 "srcs_exclude_glob": attr.string_list( 695 doc = """\ 696(list, optional): A list of labels to add as `srcs` to the generated 697`py_library` target.""", 698 ), 699 "whl_name": attr.string( 700 doc = "The whl name that the modifications are used for.", 701 mandatory = True, 702 ), 703 } 704 return attrs 705 706# NOTE: the naming of 'override' is taken from the bzlmod native 707# 'archive_override', 'git_override' bzlmod functions. 708_override_tag = tag_class( 709 attrs = { 710 "file": attr.string( 711 doc = """\ 712The Python distribution file name which needs to be patched. This will be 713applied to all repositories that setup this distribution via the pip.parse tag 714class.""", 715 mandatory = True, 716 ), 717 "patch_strip": attr.int( 718 default = 0, 719 doc = """\ 720The number of leading path segments to be stripped from the file name in the 721patches.""", 722 ), 723 "patches": attr.label_list( 724 doc = """\ 725A list of patches to apply to the repository *after* 'whl_library' is extracted 726and BUILD.bazel file is generated.""", 727 mandatory = True, 728 ), 729 }, 730 doc = """\ 731Apply any overrides (e.g. patches) to a given Python distribution defined by 732other tags in this extension.""", 733) 734 735pypi = module_extension( 736 doc = """\ 737This extension is used to make dependencies from pip available. 738 739pip.parse: 740To use, call `pip.parse()` and specify `hub_name` and your requirements file. 741Dependencies will be downloaded and made available in a repo named after the 742`hub_name` argument. 743 744Each `pip.parse()` call configures a particular Python version. Multiple calls 745can be made to configure different Python versions, and will be grouped by 746the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` 747to automatically resolve to different, Python version-specific, libraries. 748 749pip.whl_mods: 750This tag class is used to help create JSON files to describe modifications to 751the BUILD files for wheels. 752""", 753 implementation = _pip_impl, 754 tag_classes = { 755 "override": _override_tag, 756 "parse": tag_class( 757 attrs = _pip_parse_ext_attrs(), 758 doc = """\ 759This tag class is used to create a pip hub and all of the spokes that are part of that hub. 760This tag class reuses most of the pip attributes that are found in 761@rules_python//python/pip_install:pip_repository.bzl. 762The exception is it does not use the arg 'repo_prefix'. We set the repository 763prefix for the user and the alias arg is always True in bzlmod. 764""", 765 ), 766 "whl_mods": tag_class( 767 attrs = _whl_mod_attrs(), 768 doc = """\ 769This tag class is used to create JSON file that are used when calling wheel_builder.py. These 770JSON files contain instructions on how to modify a wheel's project. Each of the attributes 771create different modifications based on the type of attribute. Previously to bzlmod these 772JSON files where referred to as annotations, and were renamed to whl_modifications in this 773extension. 774""", 775 ), 776 }, 777) 778 779pypi_internal = module_extension( 780 doc = """\ 781This extension is used to make dependencies from pypi available. 782 783For now this is intended to be used internally so that usage of the `pip` 784extension in `rules_python` does not affect the evaluations of the extension 785for the consumers. 786 787pip.parse: 788To use, call `pip.parse()` and specify `hub_name` and your requirements file. 789Dependencies will be downloaded and made available in a repo named after the 790`hub_name` argument. 791 792Each `pip.parse()` call configures a particular Python version. Multiple calls 793can be made to configure different Python versions, and will be grouped by 794the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` 795to automatically resolve to different, Python version-specific, libraries. 796 797pip.whl_mods: 798This tag class is used to help create JSON files to describe modifications to 799the BUILD files for wheels. 800""", 801 implementation = _pip_non_reproducible, 802 tag_classes = { 803 "override": _override_tag, 804 "parse": tag_class( 805 attrs = _pip_parse_ext_attrs( 806 experimental_index_url = "https://pypi.org/simple", 807 ), 808 doc = """\ 809This tag class is used to create a pypi hub and all of the spokes that are part of that hub. 810This tag class reuses most of the pypi attributes that are found in 811@rules_python//python/pip_install:pip_repository.bzl. 812The exception is it does not use the arg 'repo_prefix'. We set the repository 813prefix for the user and the alias arg is always True in bzlmod. 814""", 815 ), 816 "whl_mods": tag_class( 817 attrs = _whl_mod_attrs(), 818 doc = """\ 819This tag class is used to create JSON file that are used when calling wheel_builder.py. These 820JSON files contain instructions on how to modify a wheel's project. Each of the attributes 821create different modifications based on the type of attribute. Previously to bzlmod these 822JSON files where referred to as annotations, and were renamed to whl_modifications in this 823extension. 824""", 825 ), 826 }, 827) 828 829def _whl_mods_repo_impl(rctx): 830 rctx.file("BUILD.bazel", "") 831 for whl_name, mods in rctx.attr.whl_mods.items(): 832 rctx.file("{}.json".format(whl_name), mods) 833 834_whl_mods_repo = repository_rule( 835 doc = """\ 836This rule creates json files based on the whl_mods attribute. 837""", 838 implementation = _whl_mods_repo_impl, 839 attrs = { 840 "whl_mods": attr.string_dict( 841 mandatory = True, 842 doc = "JSON endcoded string that is provided to wheel_builder.py", 843 ), 844 }, 845) 846