1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/error.gni") 18import("$dir_pw_build/python.gni") 19import("$dir_pw_build/python_action.gni") 20import("$dir_pw_build/python_gn_args.gni") 21import("$dir_pw_build/zip.gni") 22 23# Builds a directory containing a collection of Python wheels. 24# 25# Given one or more pw_python_package targets, this target will build their 26# .wheel sub-targets along with the .wheel sub-targets of all dependencies, 27# direct and indirect, as understood by GN. The resulting .whl files will be 28# collected into a single directory called 'python_wheels'. 29# 30# Args: 31# packages: A list of pw_python_package targets whose wheels should be 32# included; their dependencies will be pulled in as wheels also. 33# directory: output directory for the wheels; defaults to 34# $target_out_dir/$target_name 35# deps: additional dependencies 36# 37template("pw_python_wheels") { 38 _wheel_paths_path = "${target_gen_dir}/${target_name}_wheel_paths.txt" 39 40 _deps = [] 41 if (defined(invoker.deps)) { 42 _deps = invoker.deps 43 } 44 45 if (defined(invoker.directory)) { 46 _directory = invoker.directory 47 } else { 48 _directory = "$target_out_dir/$target_name" 49 } 50 51 _packages = [] 52 foreach(_pkg, invoker.packages) { 53 _pkg_name = get_label_info(_pkg, "label_no_toolchain") 54 _pkg_toolchain = get_label_info(_pkg, "toolchain") 55 _packages += [ "${_pkg_name}.wheel(${_pkg_toolchain})" ] 56 } 57 58 if (defined(invoker.venv)) { 59 _venv_target_label = pw_build_PYTHON_BUILD_VENV 60 _venv_target_label = invoker.venv 61 _venv_target_label = 62 get_label_info(_venv_target_label, "label_no_toolchain") 63 _packages += 64 [ "${_venv_target_label}.vendor_wheels($pw_build_PYTHON_TOOLCHAIN)" ] 65 } 66 67 # Build a list of relative paths containing all the wheels we depend on. 68 generated_file("${target_name}._wheel_paths") { 69 data_keys = [ "pw_python_package_wheels" ] 70 rebase = root_build_dir 71 deps = _packages 72 outputs = [ _wheel_paths_path ] 73 } 74 75 pw_python_action(target_name) { 76 forward_variables_from(invoker, [ "public_deps" ]) 77 deps = _deps + [ ":$target_name._wheel_paths" ] 78 module = "pw_build.collect_wheels" 79 python_deps = [ "$dir_pw_build/py" ] 80 81 args = [ 82 "--prefix", 83 rebase_path(root_build_dir, root_build_dir), 84 "--suffix", 85 rebase_path(_wheel_paths_path, root_build_dir), 86 "--out-dir", 87 rebase_path(_directory, root_build_dir), 88 ] 89 90 stamp = true 91 } 92} 93 94# Builds a .zip containing Python wheels and setup scripts. 95# 96# The resulting .zip archive will contain a directory with Python wheels for 97# all pw_python_package targets listed in 'packages', plus wheels for any 98# pw_python_package targets those packages depend on, directly or indirectly, 99# as understood by GN. 100# 101# In addition to Python wheels, the resulting .zip will also contain simple 102# setup scripts for Linux, MacOS, and Windows that take care of creating a 103# Python venv and installing all the included wheels into it, and a README.md 104# file with setup and usage instructions. 105# 106# Args: 107# packages: A list of pw_python_package targets whose wheels should be 108# included; their dependencies will be pulled in as wheels also. 109# inputs: An optional list of extra files to include in the generated .zip, 110# formatted the same was as the 'inputs' argument to pw_zip targets. 111# dirs: An optional list of directories to include in the generated .zip, 112# formatted the same way as the 'dirs' argument to pw_zip targets. 113template("pw_python_zip_with_setup") { 114 _outer_name = target_name 115 _zip_path = "${target_out_dir}/${target_name}.zip" 116 117 _inputs = [] 118 if (defined(invoker.inputs)) { 119 _inputs = invoker.inputs 120 } 121 _dirs = [] 122 if (defined(invoker.dirs)) { 123 _dirs = invoker.dirs 124 } 125 _public_deps = [] 126 if (defined(invoker.public_deps)) { 127 _public_deps = invoker.public_deps 128 } 129 130 pw_python_wheels("$target_name.wheels") { 131 packages = invoker.packages 132 forward_variables_from(invoker, 133 [ 134 "deps", 135 "venv", 136 ]) 137 } 138 139 pw_zip(target_name) { 140 forward_variables_from(invoker, [ "deps" ]) 141 inputs = _inputs + [ 142 "$dir_pw_build/python_dist/setup.bat > /${target_name}/", 143 "$dir_pw_build/python_dist/setup.sh > /${target_name}/", 144 ] 145 146 dirs = _dirs + [ "$target_out_dir/$target_name.wheels/ > /$target_name/python_wheels/" ] 147 148 output = _zip_path 149 150 # TODO: b/235245034 - Remove the plumbing-through of invoker's public_deps. 151 public_deps = _public_deps + [ ":${_outer_name}.wheels" ] 152 153 if (defined(invoker.venv)) { 154 _venv_target_label = get_label_info(invoker.venv, "label_no_toolchain") 155 _requirements_target_name = 156 get_label_info("${_venv_target_label}($pw_build_PYTHON_TOOLCHAIN)", 157 "name") 158 _requirements_gen_dir = 159 get_label_info("${_venv_target_label}($pw_build_PYTHON_TOOLCHAIN)", 160 "target_gen_dir") 161 162 inputs += [ "$_requirements_gen_dir/$_requirements_target_name/compiled_requirements.txt > /${target_name}/requirements.txt" ] 163 164 public_deps += [ "${_venv_target_label}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)" ] 165 } 166 } 167} 168 169# Generates a directory of Python packages from source files suitable for 170# deployment outside of the project developer environment. 171# 172# The resulting directory contains only files mentioned in each package's 173# setup.cfg file. This is useful for bundling multiple Python packages up 174# into a single package for distribution to other locations like 175# http://pypi.org. 176# 177# Args: 178# packages: A list of pw_python_package targets to be installed into the build 179# directory. Their dependencies will be pulled in as wheels also. 180# 181# include_tests: If true, copy Python package tests to a `tests` subdir. 182# 183# extra_files: A list of extra files that should be included in the output. The 184# format of each item in this list follows this convention: 185# //some/nested/source_file > nested/destination_file 186# 187# generate_setup_cfg: A scope containing either common_config_file or 'name' 188# and 'version' If included this creates a merged setup.cfg for all python 189# Packages using either a common_config_file as a base or name and version 190# strings. This scope can optionally include: 191# 192# include_default_pyproject_file: Include a standard pyproject.toml file 193# that uses setuptools. 194# 195# append_git_sha_to_version: Append the current git SHA to the package 196# version string after a + sign. 197# 198# append_date_to_version: Append the current date to the package version 199# string after a + sign. 200# 201# include_extra_files_in_package_data: Add any extra_files to the setup.cfg 202# file under the [options.package_data] section. 203# 204# auto_create_package_data_init_py_files: Default: true 205# Create __init__.py files as needed in all subdirs of extra_files when 206# including in [options.package_data]. 207# 208template("pw_python_distribution") { 209 _metadata_path_list_suffix = "_pw_python_distribution_metadata_path_list.txt" 210 _output_dir = "${target_out_dir}/${target_name}/" 211 _metadata_json_file_list = 212 "${target_gen_dir}/${target_name}${_metadata_path_list_suffix}" 213 214 # If generating a setup.cfg file a common base file must be provided. 215 if (defined(invoker.generate_setup_cfg)) { 216 generate_setup_cfg = invoker.generate_setup_cfg 217 assert( 218 defined(generate_setup_cfg.common_config_file) || 219 (defined(generate_setup_cfg.name) && 220 defined(generate_setup_cfg.version)), 221 "Either 'common_config_file' or ('name' + 'version') are required in generate_setup_cfg") 222 } 223 224 _inputs = [] 225 if (defined(invoker.generate_setup_cfg)) { 226 if (defined(generate_setup_cfg.common_config_file)) { 227 _inputs += [ generate_setup_cfg.common_config_file ] 228 } 229 } 230 _extra_file_inputs = [] 231 _extra_file_args = [] 232 233 # Convert extra_file strings to input, outputs and create_python_tree.py args. 234 if (defined(invoker.extra_files)) { 235 _delimiter = ">" 236 _extra_file_outputs = [] 237 foreach(input, invoker.extra_files) { 238 # Remove spaces before and after the delimiter 239 input = string_replace(input, " $_delimiter", _delimiter) 240 input = string_replace(input, "$_delimiter ", _delimiter) 241 242 input_list = [] 243 input_list = string_split(input, _delimiter) 244 245 # Save the input file 246 _extra_file_inputs += [ input_list[0] ] 247 248 # Save the output file 249 _this_output = _output_dir + "/" + input_list[1] 250 _extra_file_outputs += [ _this_output ] 251 252 # Compose an arg for passing to create_python_tree.py with properly 253 # rebased paths. 254 _extra_file_args += 255 [ string_join(" $_delimiter ", 256 [ 257 rebase_path(input_list[0], root_build_dir), 258 rebase_path(_this_output, root_build_dir), 259 ]) ] 260 } 261 } 262 263 _include_tests = defined(invoker.include_tests) && invoker.include_tests 264 265 _public_deps = [] 266 if (defined(invoker.public_deps)) { 267 _public_deps += invoker.public_deps 268 } 269 270 # Set source files for the Python package metadata json file. 271 _sources = [] 272 _setup_sources = [ 273 "$_output_dir/pyproject.toml", 274 "$_output_dir/setup.cfg", 275 ] 276 _test_sources = [] 277 278 # Create the Python package_metadata.json file so this can be used as a 279 # Python dependency. 280 _package_metadata_json_file = 281 "$target_gen_dir/$target_name/package_metadata.json" 282 283 # Get Python package metadata and write to disk as JSON. 284 _package_metadata = { 285 gn_target_name = 286 get_label_info(":${invoker.target_name}", "label_no_toolchain") 287 288 # Get package source files 289 sources = rebase_path(_sources, root_build_dir) 290 291 # Get setup.cfg, pyproject.toml, or setup.py file 292 setup_sources = rebase_path(_setup_sources, root_build_dir) 293 294 # Get test source files 295 tests = rebase_path(_test_sources, root_build_dir) 296 297 # Get package input files (package data) 298 inputs = [] 299 if (defined(invoker.inputs)) { 300 inputs = rebase_path(invoker.inputs, root_build_dir) 301 } 302 inputs += rebase_path(_extra_file_inputs, root_build_dir) 303 } 304 305 # Finally, write out the json 306 write_file(_package_metadata_json_file, _package_metadata, "json") 307 308 group("$target_name._package_metadata") { 309 metadata = { 310 pw_python_package_metadata_json = [ _package_metadata_json_file ] 311 } 312 313 # Forward the package_metadata subtarget for all packages bundled in this 314 # distribution. 315 public_deps = [] 316 foreach(dep, invoker.packages) { 317 public_deps += [ get_label_info(dep, "label_no_toolchain") + 318 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 319 } 320 } 321 322 _package_metadata_targets = [] 323 foreach(pkg, invoker.packages) { 324 _package_metadata_targets += 325 [ get_label_info(pkg, "label_no_toolchain") + 326 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 327 } 328 329 # Build a list of relative paths containing all the python 330 # package_metadata.json files we depend on. 331 generated_file("${target_name}.${_metadata_path_list_suffix}") { 332 data_keys = [ "pw_python_package_metadata_json" ] 333 rebase = root_build_dir 334 deps = _package_metadata_targets 335 outputs = [ _metadata_json_file_list ] 336 } 337 338 # Run the python action on the metadata_path_list.txt file 339 pw_python_action(target_name) { 340 # Save the Python package metadata so this can be installed using 341 # pw_internal_pip_install. 342 metadata = { 343 pw_python_package_metadata_json = [ _package_metadata_json_file ] 344 } 345 346 deps = invoker.packages + 347 [ ":${invoker.target_name}.${_metadata_path_list_suffix}" ] 348 349 script = "$dir_pw_build/py/pw_build/create_python_tree.py" 350 inputs = _inputs + _extra_file_inputs 351 public_deps = _public_deps 352 _pw_internal_run_in_venv = false 353 354 args = [ 355 "--repo-root", 356 rebase_path("//", root_build_dir), 357 "--tree-destination-dir", 358 rebase_path(_output_dir, root_build_dir), 359 "--input-list-files", 360 rebase_path(_metadata_json_file_list, root_build_dir), 361 ] 362 363 # Add required setup.cfg args if we are generating a merged config. 364 if (defined(generate_setup_cfg)) { 365 if (defined(generate_setup_cfg.common_config_file)) { 366 args += [ 367 "--setupcfg-common-file", 368 rebase_path(generate_setup_cfg.common_config_file, root_build_dir), 369 ] 370 } 371 if (defined(generate_setup_cfg.append_git_sha_to_version)) { 372 args += [ "--setupcfg-version-append-git-sha" ] 373 } 374 if (defined(generate_setup_cfg.append_date_to_version)) { 375 args += [ "--setupcfg-version-append-date" ] 376 } 377 if (defined(generate_setup_cfg.name)) { 378 args += [ 379 "--setupcfg-override-name", 380 generate_setup_cfg.name, 381 ] 382 } 383 if (defined(generate_setup_cfg.version)) { 384 args += [ 385 "--setupcfg-override-version", 386 generate_setup_cfg.version, 387 ] 388 } 389 if (defined(generate_setup_cfg.include_default_pyproject_file) && 390 generate_setup_cfg.include_default_pyproject_file == true) { 391 args += [ "--create-default-pyproject-toml" ] 392 } 393 if (defined(generate_setup_cfg.include_extra_files_in_package_data)) { 394 args += [ "--setupcfg-extra-files-in-package-data" ] 395 } 396 _auto_create_package_data_init_py_files = true 397 if (defined(generate_setup_cfg.auto_create_package_data_init_py_files)) { 398 _auto_create_package_data_init_py_files = 399 generate_setup_cfg.auto_create_package_data_init_py_files 400 } 401 if (_auto_create_package_data_init_py_files) { 402 args += [ "--auto-create-package-data-init-py-files" ] 403 } 404 } 405 406 if (_extra_file_args == []) { 407 # No known output files - stamp instead. 408 stamp = true 409 } else { 410 args += [ "--extra-files" ] 411 args += _extra_file_args 412 413 # Include extra_files as outputs 414 outputs = _extra_file_outputs 415 } 416 417 if (_include_tests) { 418 args += [ "--include-tests" ] 419 } 420 } 421 422 # Template to build a bundled Python package wheel. 423 pw_python_action("$target_name._build_wheel") { 424 _wheel_out_dir = "$target_out_dir/$target_name" 425 _wheel_requirement = "$_wheel_out_dir/requirements.txt" 426 metadata = { 427 pw_python_package_wheels = [ _wheel_out_dir ] 428 } 429 430 script = "$dir_pw_build/py/pw_build/generate_python_wheel.py" 431 432 args = [ 433 "--package-dir", 434 rebase_path(_output_dir, root_build_dir), 435 "--out-dir", 436 rebase_path(_wheel_out_dir, root_build_dir), 437 ] 438 439 # Add hashes to the _wheel_requirement output. 440 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 441 args += [ "--generate-hashes" ] 442 } 443 444 public_deps = [] 445 if (defined(invoker.public_deps)) { 446 public_deps += invoker.public_deps 447 } 448 public_deps += [ ":${invoker.target_name}" ] 449 450 outputs = [ _wheel_requirement ] 451 } 452 group("$target_name.wheel") { 453 public_deps = [ ":${invoker.target_name}._build_wheel" ] 454 } 455 456 # Allow using pw_python_distribution targets as a python_dep in 457 # pw_python_group. To do this, create a pw_python_group with the relevant 458 # packages and create wrappers for each subtarget, except those that are 459 # actually implemented by this template. 460 # 461 # This is an ugly workaround that will be removed when the Python build is 462 # refactored (b/235278298). 463 pw_python_group("$target_name._pw_python_group") { 464 python_deps = invoker.packages 465 } 466 467 wrapped_subtargets = pw_python_package_subtargets - [ 468 "wheel", 469 "_build_wheel", 470 ] 471 472 foreach(subtarget, wrapped_subtargets) { 473 group("$target_name.$subtarget") { 474 public_deps = [ ":${invoker.target_name}._pw_python_group.$subtarget" ] 475 } 476 } 477} 478 479# TODO: b/232800695 - Remove this template when all projects no longer use it. 480template("pw_create_python_source_tree") { 481 pw_python_distribution("$target_name") { 482 forward_variables_from(invoker, "*") 483 } 484} 485 486# Runs pip install on a set of pw_python_packages. This will install 487# pw_python_packages into the user's developer environment. 488# 489# Args: 490# packages: A list of pw_python_package targets to be pip installed. 491# These will be installed one at a time. 492# 493# editable: If true, --editable is passed to the pip install command. 494# 495# force_reinstall: If true, --force-reinstall is passed to the pip install 496# command. 497template("pw_python_pip_install") { 498 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 499 # Create a target group for the Python package metadata only. 500 group("$target_name._package_metadata") { 501 # Forward the package_metadata subtarget for all python_deps. 502 public_deps = [] 503 if (defined(invoker.packages)) { 504 foreach(dep, invoker.packages) { 505 public_deps += [ get_label_info(dep, "label_no_toolchain") + 506 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 507 } 508 } 509 } 510 511 pw_python_action("$target_name") { 512 script = "$dir_pw_build/py/pw_build/pip_install_python_deps.py" 513 514 assert( 515 defined(invoker.packages), 516 "packages = [ 'python_package' ] is required by pw_internal_pip_install") 517 518 public_deps = [] 519 if (defined(invoker.public_deps)) { 520 public_deps += invoker.public_deps 521 } 522 523 python_deps = [] 524 python_metadata_deps = [] 525 if (defined(invoker.packages)) { 526 public_deps += invoker.packages 527 python_deps += invoker.packages 528 python_metadata_deps += invoker.packages 529 } 530 531 python_deps = [] 532 if (defined(invoker.python_deps)) { 533 python_deps += invoker.python_deps 534 } 535 536 _pw_internal_run_in_venv = false 537 _forward_python_metadata_deps = true 538 539 _editable_install = false 540 if (defined(invoker.editable)) { 541 _editable_install = invoker.editable 542 } 543 544 _pkg_gn_labels = [] 545 foreach(pkg, invoker.packages) { 546 _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") ] 547 } 548 549 _pip_install_log_file = "$target_gen_dir/$target_name/pip_install_log.txt" 550 551 args = [ 552 "--gn-packages", 553 string_join(",", _pkg_gn_labels), 554 ] 555 556 if (_editable_install) { 557 args += [ "--editable-pip-install" ] 558 } 559 560 args += [ 561 "--log", 562 rebase_path(_pip_install_log_file, root_build_dir), 563 ] 564 args += pw_build_PYTHON_PIP_DEFAULT_OPTIONS 565 args += [ 566 "install", 567 "--no-build-isolation", 568 ] 569 570 if (!_editable_install) { 571 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 572 args += [ "--require-hashes" ] 573 574 # The --require-hashes option can only install wheels via 575 # requirement.txt files that contain hashes. Depend on this package's 576 # _build_wheel target. 577 foreach(pkg, _pkg_gn_labels) { 578 public_deps += [ "${pkg}._build_wheel" ] 579 } 580 } 581 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 582 args += [ "--no-index" ] 583 } 584 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 585 args += [ "--no-cache-dir" ] 586 } 587 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 588 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 589 args += [ 590 "--find-links", 591 rebase_path(link_dir, root_build_dir), 592 ] 593 } 594 } 595 } 596 597 _force_reinstall = false 598 if (defined(invoker.force_reinstall)) { 599 _force_reinstall = true 600 } 601 if (_force_reinstall) { 602 args += [ "--force-reinstall" ] 603 } 604 605 inputs = pw_build_PIP_CONSTRAINTS 606 607 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) { 608 args += [ 609 "--constraint", 610 rebase_path(_constraints_file, root_build_dir), 611 ] 612 } 613 614 stamp = true 615 616 # Parallel pip installations don't work, so serialize pip invocations. 617 pool = "$dir_pw_build/pool:pip($default_toolchain)" 618 } 619 } else { 620 group("$target_name") { 621 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ] 622 } 623 not_needed("*") 624 not_needed(invoker, "*") 625 } 626 627 group("$target_name.install") { 628 public_deps = [ ":${invoker.target_name}" ] 629 } 630 631 # Allow using pw_internal_pip_install targets as a python_dep in 632 # pw_python_group. To do this, create a pw_python_group with the relevant 633 # packages and create wrappers for each subtarget, except those that are 634 # actually implemented by this template. 635 # 636 # This is an ugly workaround that will be removed when the Python build is 637 # refactored (b/235278298). 638 pw_python_group("$target_name._pw_python_group") { 639 python_deps = invoker.packages 640 } 641 642 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) { 643 group("$target_name.$subtarget") { 644 _test_and_lint_subtargets = [ 645 "tests", 646 "lint", 647 "lint.mypy", 648 "lint.pylint", 649 "lint.ruff", 650 ] 651 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS || 652 filter_exclude([ subtarget ], _test_and_lint_subtargets) != []) { 653 public_deps = [ ":${invoker.target_name}._pw_python_group.$subtarget" ] 654 } 655 not_needed([ "_test_and_lint_subtargets" ]) 656 } 657 } 658} 659 660# TODO: b/232800695 - Remove this template when all projects no longer use it. 661template("pw_internal_pip_install") { 662 pw_python_pip_install("$target_name") { 663 forward_variables_from(invoker, "*") 664 } 665} 666