1# Copyright 2023 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""render_pkg_aliases is a function to generate BUILD.bazel contents used to create user-friendly aliases. 16 17This is used in bzlmod and non-bzlmod setups.""" 18 19load("//python/private:normalize_name.bzl", "normalize_name") 20load("//python/private:text_util.bzl", "render") 21load( 22 ":generate_group_library_build_bazel.bzl", 23 "generate_group_library_build_bazel", 24) # buildifier: disable=bzl-visibility 25load( 26 ":labels.bzl", 27 "DATA_LABEL", 28 "DIST_INFO_LABEL", 29 "PY_LIBRARY_IMPL_LABEL", 30 "PY_LIBRARY_PUBLIC_LABEL", 31 "WHEEL_FILE_IMPL_LABEL", 32 "WHEEL_FILE_PUBLIC_LABEL", 33) 34load(":parse_whl_name.bzl", "parse_whl_name") 35load(":whl_target_platforms.bzl", "whl_target_platforms") 36 37NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ 38No matching wheel for current configuration's Python version. 39 40The current build configuration's Python version doesn't match any of the Python 41versions available for this wheel. This wheel supports the following Python versions: 42 {supported_versions} 43 44As matched by the `@{rules_python}//python/config_settings:is_python_<version>` 45configuration settings. 46 47To determine the current configuration's Python version, run: 48 `bazel config <config id>` (shown further below) 49and look for 50 {rules_python}//python/config_settings:python_version 51 52If the value is missing, then the "default" Python version is being used, 53which has a "null" version value and will not match version constraints. 54""" 55 56NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2 = """\ 57No matching wheel for current configuration's Python version. 58 59The current build configuration's Python version doesn't match any of the Python 60wheels available for this wheel. This wheel supports the following Python 61configuration settings: 62 {config_settings} 63 64To determine the current configuration's Python version, run: 65 `bazel config <config id>` (shown further below) 66and look for 67 {rules_python}//python/config_settings:python_version 68 69If the value is missing, then the "default" Python version is being used, 70which has a "null" version value and will not match version constraints. 71""" 72 73def _render_whl_library_alias( 74 *, 75 name, 76 default_config_setting, 77 aliases, 78 target_name, 79 **kwargs): 80 """Render an alias for common targets.""" 81 if len(aliases) == 1 and not aliases[0].version: 82 alias = aliases[0] 83 return render.alias( 84 name = name, 85 actual = repr("@{repo}//:{name}".format( 86 repo = alias.repo, 87 name = target_name, 88 )), 89 **kwargs 90 ) 91 92 # Create the alias repositories which contains different select 93 # statements These select statements point to the different pip 94 # whls that are based on a specific version of Python. 95 selects = {} 96 no_match_error = "_NO_MATCH_ERROR" 97 for alias in sorted(aliases, key = lambda x: x.version): 98 actual = "@{repo}//:{name}".format(repo = alias.repo, name = target_name) 99 selects.setdefault(actual, []).append(alias.config_setting) 100 if alias.config_setting == default_config_setting: 101 selects[actual].append("//conditions:default") 102 no_match_error = None 103 104 return render.alias( 105 name = name, 106 actual = render.select( 107 { 108 tuple(sorted( 109 conditions, 110 # Group `is_python` and other conditions for easier reading 111 # when looking at the generated files. 112 key = lambda condition: ("is_python" not in condition, condition), 113 )): target 114 for target, conditions in sorted(selects.items()) 115 }, 116 no_match_error = no_match_error, 117 # This key_repr is used to render selects.with_or keys 118 key_repr = lambda x: repr(x[0]) if len(x) == 1 else render.tuple(x), 119 name = "selects.with_or", 120 ), 121 **kwargs 122 ) 123 124def _render_common_aliases(*, name, aliases, default_config_setting = None, group_name = None): 125 lines = [ 126 """load("@bazel_skylib//lib:selects.bzl", "selects")""", 127 """package(default_visibility = ["//visibility:public"])""", 128 ] 129 130 config_settings = None 131 if aliases: 132 config_settings = sorted([v.config_setting for v in aliases if v.config_setting]) 133 134 if not config_settings or default_config_setting in config_settings: 135 pass 136 else: 137 error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2.format( 138 config_settings = render.indent( 139 "\n".join(config_settings), 140 ).lstrip(), 141 rules_python = "rules_python", 142 ) 143 144 lines.append("_NO_MATCH_ERROR = \"\"\"\\\n{error_msg}\"\"\"".format( 145 error_msg = error_msg, 146 )) 147 148 # This is to simplify the code in _render_whl_library_alias and to ensure 149 # that we don't pass a 'default_version' that is not in 'versions'. 150 default_config_setting = None 151 152 lines.append( 153 render.alias( 154 name = name, 155 actual = repr(":pkg"), 156 ), 157 ) 158 lines.extend( 159 [ 160 _render_whl_library_alias( 161 name = name, 162 default_config_setting = default_config_setting, 163 aliases = aliases, 164 target_name = target_name, 165 visibility = ["//_groups:__subpackages__"] if name.startswith("_") else None, 166 ) 167 for target_name, name in { 168 PY_LIBRARY_PUBLIC_LABEL: PY_LIBRARY_IMPL_LABEL if group_name else PY_LIBRARY_PUBLIC_LABEL, 169 WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if group_name else WHEEL_FILE_PUBLIC_LABEL, 170 DATA_LABEL: DATA_LABEL, 171 DIST_INFO_LABEL: DIST_INFO_LABEL, 172 }.items() 173 ], 174 ) 175 if group_name: 176 lines.extend( 177 [ 178 render.alias( 179 name = "pkg", 180 actual = repr("//_groups:{}_pkg".format(group_name)), 181 ), 182 render.alias( 183 name = "whl", 184 actual = repr("//_groups:{}_whl".format(group_name)), 185 ), 186 ], 187 ) 188 189 return "\n\n".join(lines) 190 191def render_pkg_aliases(*, aliases, default_config_setting = None, requirement_cycles = None): 192 """Create alias declarations for each PyPI package. 193 194 The aliases should be appended to the pip_repository BUILD.bazel file. These aliases 195 allow users to use requirement() without needed a corresponding `use_repo()` for each dep 196 when using bzlmod. 197 198 Args: 199 aliases: dict, the keys are normalized distribution names and values are the 200 whl_alias instances. 201 default_config_setting: the default to be used for the aliases. 202 requirement_cycles: any package groups to also add. 203 204 Returns: 205 A dict of file paths and their contents. 206 """ 207 contents = {} 208 if not aliases: 209 return contents 210 elif type(aliases) != type({}): 211 fail("The aliases need to be provided as a dict, got: {}".format(type(aliases))) 212 213 whl_group_mapping = {} 214 if requirement_cycles: 215 requirement_cycles = { 216 name: [normalize_name(whl_name) for whl_name in whls] 217 for name, whls in requirement_cycles.items() 218 } 219 220 whl_group_mapping = { 221 whl_name: group_name 222 for group_name, group_whls in requirement_cycles.items() 223 for whl_name in group_whls 224 } 225 226 files = { 227 "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( 228 name = normalize_name(name), 229 aliases = pkg_aliases, 230 default_config_setting = default_config_setting, 231 group_name = whl_group_mapping.get(normalize_name(name)), 232 ).strip() 233 for name, pkg_aliases in aliases.items() 234 } 235 236 if requirement_cycles: 237 files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) 238 return files 239 240def whl_alias(*, repo, version = None, config_setting = None, filename = None, target_platforms = None): 241 """The bzl_packages value used by by the render_pkg_aliases function. 242 243 This contains the minimum amount of information required to generate correct 244 aliases in a hub repository. 245 246 Args: 247 repo: str, the repo of where to find the things to be aliased. 248 version: optional(str), the version of the python toolchain that this 249 whl alias is for. If not set, then non-version aware aliases will be 250 constructed. This is mainly used for better error messages when there 251 is no match found during a select. 252 config_setting: optional(Label or str), the config setting that we should use. Defaults 253 to "//_config:is_python_{version}". 254 filename: optional(str), the distribution filename to derive the config_setting. 255 target_platforms: optional(list[str]), the list of target_platforms for this 256 distribution. 257 258 Returns: 259 a struct with the validated and parsed values. 260 """ 261 if not repo: 262 fail("'repo' must be specified") 263 264 if version: 265 config_setting = config_setting or ("//_config:is_python_" + version) 266 config_setting = str(config_setting) 267 268 if target_platforms: 269 for p in target_platforms: 270 if not p.startswith("cp"): 271 fail("target_platform should start with 'cp' denoting the python version, got: " + p) 272 273 return struct( 274 repo = repo, 275 version = version, 276 config_setting = config_setting, 277 filename = filename, 278 target_platforms = target_platforms, 279 ) 280 281def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwargs): 282 """Render the multi-platform pkg aliases. 283 284 Args: 285 aliases: dict[str, list(whl_alias)] A list of aliases that will be 286 transformed from ones having `filename` to ones having `config_setting`. 287 default_version: str, the default python version. Defaults to None. 288 **kwargs: extra arguments passed to render_pkg_aliases. 289 290 Returns: 291 A dict of file paths and their contents. 292 """ 293 294 flag_versions = get_whl_flag_versions( 295 aliases = [ 296 a 297 for bunch in aliases.values() 298 for a in bunch 299 ], 300 ) 301 302 config_setting_aliases = { 303 pkg: multiplatform_whl_aliases( 304 aliases = pkg_aliases, 305 default_version = default_version, 306 glibc_versions = flag_versions.get("glibc_versions", []), 307 muslc_versions = flag_versions.get("muslc_versions", []), 308 osx_versions = flag_versions.get("osx_versions", []), 309 ) 310 for pkg, pkg_aliases in aliases.items() 311 } 312 313 contents = render_pkg_aliases( 314 aliases = config_setting_aliases, 315 **kwargs 316 ) 317 contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions) 318 return contents 319 320def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): 321 """convert a list of aliases from filename to config_setting ones. 322 323 Args: 324 aliases: list(whl_alias): The aliases to process. Any aliases that have 325 the filename set will be converted to a list of aliases, each with 326 an appropriate config_setting value. 327 default_version: string | None, the default python version to use. 328 **kwargs: Extra parameters passed to get_filename_config_settings. 329 330 Returns: 331 A dict with aliases to be used in the hub repo. 332 """ 333 334 ret = [] 335 versioned_additions = {} 336 for alias in aliases: 337 if not alias.filename: 338 ret.append(alias) 339 continue 340 341 config_settings, all_versioned_settings = get_filename_config_settings( 342 # TODO @aignas 2024-05-27: pass the parsed whl to reduce the 343 # number of duplicate operations. 344 filename = alias.filename, 345 target_platforms = alias.target_platforms, 346 python_version = alias.version, 347 python_default = default_version == alias.version, 348 **kwargs 349 ) 350 351 for setting in config_settings: 352 ret.append(whl_alias( 353 repo = alias.repo, 354 version = alias.version, 355 config_setting = "//_config" + setting, 356 )) 357 358 # Now for the versioned platform config settings, we need to select one 359 # that best fits the bill and if there are multiple wheels, e.g. 360 # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select 361 # the former when the glibc is in the range of [2.17, 2.28) and then chose 362 # the later if it is [2.28, ...). If the 2.28 wheel was not present in 363 # the hub, then we would need to use 2.17 for all the glibc version 364 # configurations. 365 # 366 # Here we add the version settings to a dict where we key the range of 367 # versions that the whl spans. If the wheel supports musl and glibc at 368 # the same time, we do this for each supported platform, hence the 369 # double dict. 370 for default_setting, versioned in all_versioned_settings.items(): 371 versions = sorted(versioned) 372 min_version = versions[0] 373 max_version = versions[-1] 374 375 versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( 376 repo = alias.repo, 377 python_version = alias.version, 378 settings = versioned, 379 ) 380 381 versioned = {} 382 for default_setting, candidates in versioned_additions.items(): 383 # Sort the candidates by the range of versions the span, so that we 384 # start with the lowest version. 385 for _, candidate in sorted(candidates.items()): 386 # Set the default with the first candidate, which gives us the highest 387 # compatibility. If the users want to use a higher-version than the default 388 # they can configure the glibc_version flag. 389 versioned.setdefault(default_setting, whl_alias( 390 version = candidate.python_version, 391 config_setting = "//_config" + default_setting, 392 repo = candidate.repo, 393 )) 394 395 # We will be overwriting previously added entries, but that is intended. 396 for _, setting in sorted(candidate.settings.items()): 397 versioned[setting] = whl_alias( 398 version = candidate.python_version, 399 config_setting = "//_config" + setting, 400 repo = candidate.repo, 401 ) 402 403 ret.extend(versioned.values()) 404 return ret 405 406def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): 407 return """\ 408load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") 409 410config_settings( 411 name = "config_settings", 412 glibc_versions = {glibc_versions}, 413 muslc_versions = {muslc_versions}, 414 osx_versions = {osx_versions}, 415 python_versions = {python_versions}, 416 target_platforms = {target_platforms}, 417 visibility = ["//:__subpackages__"], 418)""".format( 419 glibc_versions = render.indent(render.list(glibc_versions)).lstrip(), 420 muslc_versions = render.indent(render.list(muslc_versions)).lstrip(), 421 osx_versions = render.indent(render.list(osx_versions)).lstrip(), 422 python_versions = render.indent(render.list(python_versions)).lstrip(), 423 target_platforms = render.indent(render.list(target_platforms)).lstrip(), 424 ) 425 426def get_whl_flag_versions(aliases): 427 """Return all of the flag versions that is used by the aliases 428 429 Args: 430 aliases: list[whl_alias] 431 432 Returns: 433 dict, which may have keys: 434 * python_versions 435 """ 436 python_versions = {} 437 glibc_versions = {} 438 target_platforms = {} 439 muslc_versions = {} 440 osx_versions = {} 441 442 for a in aliases: 443 if not a.version and not a.filename: 444 continue 445 446 if a.version: 447 python_versions[a.version] = None 448 449 if not a.filename: 450 continue 451 452 if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): 453 parsed = parse_whl_name(a.filename) 454 else: 455 for plat in a.target_platforms or []: 456 target_platforms[_non_versioned_platform(plat)] = None 457 continue 458 459 for platform_tag in parsed.platform_tag.split("."): 460 parsed = whl_target_platforms(platform_tag) 461 462 for p in parsed: 463 target_platforms[p.target_platform] = None 464 465 if platform_tag.startswith("win") or platform_tag.startswith("linux"): 466 continue 467 468 head, _, tail = platform_tag.partition("_") 469 major, _, tail = tail.partition("_") 470 minor, _, tail = tail.partition("_") 471 if tail: 472 version = (int(major), int(minor)) 473 if "many" in head: 474 glibc_versions[version] = None 475 elif "musl" in head: 476 muslc_versions[version] = None 477 elif "mac" in head: 478 osx_versions[version] = None 479 else: 480 fail(platform_tag) 481 482 return { 483 k: sorted(v) 484 for k, v in { 485 "glibc_versions": glibc_versions, 486 "muslc_versions": muslc_versions, 487 "osx_versions": osx_versions, 488 "python_versions": python_versions, 489 "target_platforms": target_platforms, 490 }.items() 491 if v 492 } 493 494def _non_versioned_platform(p, *, strict = False): 495 """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. 496 497 This is so that we can tighten the code structure later by using strict = True. 498 """ 499 has_abi = p.startswith("cp") 500 if has_abi: 501 return p.partition("_")[-1] 502 elif not strict: 503 return p 504 else: 505 fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) 506 507def get_filename_config_settings( 508 *, 509 filename, 510 target_platforms, 511 glibc_versions, 512 muslc_versions, 513 osx_versions, 514 python_version = "", 515 python_default = True): 516 """Get the filename config settings. 517 518 Args: 519 filename: the distribution filename (can be a whl or an sdist). 520 target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. 521 glibc_versions: list[tuple[int, int]], list of versions. 522 muslc_versions: list[tuple[int, int]], list of versions. 523 osx_versions: list[tuple[int, int]], list of versions. 524 python_version: the python version to generate the config_settings for. 525 python_default: if we should include the setting when python_version is not set. 526 527 Returns: 528 A tuple: 529 * A list of config settings that are generated by ./pip_config_settings.bzl 530 * The list of default version settings. 531 """ 532 prefixes = [] 533 suffixes = [] 534 if (0, 0) in glibc_versions: 535 fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") 536 if (0, 0) in muslc_versions: 537 fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") 538 if (0, 0) in osx_versions: 539 fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") 540 541 glibc_versions = sorted(glibc_versions) 542 muslc_versions = sorted(muslc_versions) 543 osx_versions = sorted(osx_versions) 544 setting_supported_versions = {} 545 546 if filename.endswith(".whl"): 547 parsed = parse_whl_name(filename) 548 if parsed.python_tag == "py2.py3": 549 py = "py" 550 elif parsed.python_tag.startswith("cp"): 551 py = "cp3x" 552 else: 553 py = "py3" 554 555 if parsed.abi_tag.startswith("cp"): 556 abi = "cp" 557 else: 558 abi = parsed.abi_tag 559 560 if parsed.platform_tag == "any": 561 prefixes = ["{}_{}_any".format(py, abi)] 562 suffixes = [_non_versioned_platform(p) for p in target_platforms or []] 563 else: 564 prefixes = ["{}_{}".format(py, abi)] 565 suffixes = _whl_config_setting_suffixes( 566 platform_tag = parsed.platform_tag, 567 glibc_versions = glibc_versions, 568 muslc_versions = muslc_versions, 569 osx_versions = osx_versions, 570 setting_supported_versions = setting_supported_versions, 571 ) 572 else: 573 prefixes = ["sdist"] 574 suffixes = [_non_versioned_platform(p) for p in target_platforms or []] 575 576 if python_default and python_version: 577 prefixes += ["cp{}_{}".format(python_version, p) for p in prefixes] 578 elif python_version: 579 prefixes = ["cp{}_{}".format(python_version, p) for p in prefixes] 580 elif python_default: 581 pass 582 else: 583 fail("BUG: got no python_version and it is not default") 584 585 versioned = { 586 ":is_{}_{}".format(p, suffix): { 587 version: ":is_{}_{}".format(p, setting) 588 for version, setting in versions.items() 589 } 590 for p in prefixes 591 for suffix, versions in setting_supported_versions.items() 592 } 593 594 if suffixes or versioned: 595 return [":is_{}_{}".format(p, s) for p in prefixes for s in suffixes], versioned 596 else: 597 return [":is_{}".format(p) for p in prefixes], setting_supported_versions 598 599def _whl_config_setting_suffixes( 600 platform_tag, 601 glibc_versions, 602 muslc_versions, 603 osx_versions, 604 setting_supported_versions): 605 suffixes = [] 606 for platform_tag in platform_tag.split("."): 607 for p in whl_target_platforms(platform_tag): 608 prefix = p.os 609 suffix = p.cpu 610 if "manylinux" in platform_tag: 611 prefix = "manylinux" 612 versions = glibc_versions 613 elif "musllinux" in platform_tag: 614 prefix = "musllinux" 615 versions = muslc_versions 616 elif p.os in ["linux", "windows"]: 617 versions = [(0, 0)] 618 elif p.os == "osx": 619 versions = osx_versions 620 if "universal2" in platform_tag: 621 suffix += "_universal2" 622 else: 623 fail("Unsupported whl os: {}".format(p.os)) 624 625 default_version_setting = "{}_{}".format(prefix, suffix) 626 supported_versions = {} 627 for v in versions: 628 if v == (0, 0): 629 suffixes.append(default_version_setting) 630 elif v >= p.version: 631 supported_versions[v] = "{}_{}_{}_{}".format( 632 prefix, 633 v[0], 634 v[1], 635 suffix, 636 ) 637 if supported_versions: 638 setting_supported_versions[default_version_setting] = supported_versions 639 640 return suffixes 641