1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/python.gni") 18import("$dir_pw_build/python_action.gni") 19 20# Defines and creates a Python virtualenv. This template is used by Pigweed in 21# https://cs.pigweed.dev/pigweed/+/main:pw_env_setup/BUILD.gn to 22# create a virtualenv for use within the GN build that all Python actions will 23# run in. 24# 25# Example: 26# 27# pw_python_venv("test_venv") { 28# path = "test-venv" 29# constraints = [ "//tools/constraints.list" ] 30# requirements = [ "//tools/requirements.txt" ] 31# source_packages = [ 32# "$dir_pw_cli/py", 33# "$dir_pw_console/py", 34# "//tools:another_pw_python_package", 35# ] 36# } 37# 38# Args: 39# path: The directory where the virtualenv will be created. This is relative 40# to the GN root and must begin with "$root_build_dir/" if it lives in the 41# output directory or "//" if it lives in elsewhere. 42# 43# constraints: A list of constraint files used when performing pip install 44# into this virtualenv. By default this is set to pw_build_PIP_CONSTRAINTS 45# 46# requirements: A list of requirements files to install into this virtualenv 47# on creation. By default this is set to pw_build_PIP_REQUIREMENTS 48# 49# pip_generate_hashes: (Default: false) Use --generate-hashes When 50# running pip-compile to compute the final requirements.txt 51# 52# source_packages: A list of in-tree pw_python_package targets that will be 53# checked for external third_party pip dependencies to install into this 54# virtualenv. Note this list of targets isn't actually installed into the 55# virtualenv. Only packages defined inside the [options] install_requires 56# section of each pw_python_package's setup.cfg will be pip installed. See 57# this page for a setup.cfg example: 58# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html 59# 60# output_logs: (Default: true) Commands will output logs. 61# 62template("pw_python_venv") { 63 assert(defined(invoker.path), "pw_python_venv requires a 'path'") 64 65 _path = invoker.path 66 67 _generated_requirements_file = 68 "$target_gen_dir/$target_name/generated_requirements.txt" 69 70 _compiled_requirements_file = 71 "$target_gen_dir/$target_name/compiled_requirements.txt" 72 73 _source_packages = [] 74 if (defined(invoker.source_packages)) { 75 _source_packages += invoker.source_packages 76 } else { 77 not_needed([ 78 "_source_packages", 79 "_generated_requirements_file", 80 ]) 81 } 82 _output_logs = true 83 if (defined(invoker.output_logs)) { 84 _output_logs = invoker.output_logs 85 } 86 if (!defined(invoker.output_logs) || 87 current_toolchain != pw_build_PYTHON_TOOLCHAIN) { 88 not_needed([ "_output_logs" ]) 89 } 90 91 _source_package_labels = [] 92 foreach(pkg, _source_packages) { 93 _source_package_labels += [ get_label_info(pkg, "label_no_toolchain") ] 94 } 95 96 if (defined(invoker.requirements)) { 97 _requirements = invoker.requirements 98 } else { 99 _requirements = pw_build_PIP_REQUIREMENTS 100 } 101 102 if (defined(invoker.constraints)) { 103 _constraints = invoker.constraints 104 } else { 105 _constraints = pw_build_PIP_CONSTRAINTS 106 } 107 108 _python_interpreter = _path + "/bin/python" 109 if (host_os == "win") { 110 _python_interpreter = _path + "/Scripts/python.exe" 111 } 112 113 _venv_metadata_json_file = "$target_gen_dir/$target_name/venv_metadata.json" 114 _venv_metadata = { 115 gn_target_name = 116 get_label_info(":${invoker.target_name}", "label_no_toolchain") 117 path = rebase_path(_path, root_build_dir) 118 generated_requirements = 119 rebase_path(_generated_requirements_file, root_build_dir) 120 compiled_requirements = 121 rebase_path(_compiled_requirements_file, root_build_dir) 122 requirements = rebase_path(_requirements, root_build_dir) 123 constraints = rebase_path(_constraints, root_build_dir) 124 interpreter = rebase_path(_python_interpreter, root_build_dir) 125 source_packages = _source_package_labels 126 } 127 write_file(_venv_metadata_json_file, _venv_metadata, "json") 128 129 pw_python_action("${target_name}._create_virtualenv") { 130 _pw_internal_run_in_venv = false 131 132 # Note: The if the venv isn't in the out dir then we can't declare 133 # outputs and must stamp instead. 134 stamp = true 135 136 # The virtualenv should depend on the version of Python currently in use. 137 stampfile = "$target_gen_dir/$target_name.pw_pystamp" 138 depfile = "$target_gen_dir/$target_name.d" 139 script = "$dir_pw_build/py/pw_build/create_gn_venv.py" 140 args = [ 141 "--depfile", 142 rebase_path(depfile, root_build_dir), 143 "--destination-dir", 144 rebase_path(_path, root_build_dir), 145 "--stampfile", 146 rebase_path(stampfile, root_build_dir), 147 ] 148 } 149 150 if (defined(invoker.source_packages) && 151 current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 152 pw_python_action("${target_name}._generate_3p_requirements") { 153 inputs = _requirements + _constraints 154 155 _pw_internal_run_in_venv = false 156 _forward_python_metadata_deps = true 157 158 script = "$dir_pw_build/py/pw_build/generate_python_requirements.py" 159 160 _pkg_gn_labels = [] 161 foreach(pkg, _source_packages) { 162 _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") + 163 "($pw_build_PYTHON_TOOLCHAIN)" ] 164 } 165 166 # Add target packages to the python_metadata_deps. This will let 167 # GN expand the transitive pw_python_package deps which are read 168 # by generate_python_requirements.py 169 python_metadata_deps = _pkg_gn_labels 170 171 args = [ 172 "--gn-root-build-dir", 173 rebase_path(root_build_dir, root_build_dir), 174 "--output-requirement-file", 175 rebase_path(_generated_requirements_file, root_build_dir), 176 ] 177 178 if (_constraints != []) { 179 args += [ "--constraint-files" ] 180 } 181 foreach(_constraints_file, _constraints) { 182 args += [ rebase_path(_constraints_file, root_build_dir) ] 183 } 184 185 args += [ 186 "--gn-packages", 187 string_join(",", _pkg_gn_labels), 188 ] 189 190 outputs = [ _generated_requirements_file ] 191 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 192 } 193 } else { 194 group("${target_name}._generate_3p_requirements") { 195 } 196 } 197 198 _pip_generate_hashes = false 199 if (defined(invoker.pip_generate_hashes)) { 200 _pip_generate_hashes = invoker.pip_generate_hashes 201 } else { 202 not_needed([ "_pip_generate_hashes" ]) 203 } 204 205 if (defined(invoker.source_packages) || defined(invoker.requirements)) { 206 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 207 # Compile requirements with hashes 208 pw_python_action("${target_name}._compile_requirements") { 209 module = "piptools" 210 211 # Set the venv to run this pip install in. 212 _pw_internal_run_in_venv = true 213 _skip_installing_external_python_deps = true 214 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 215 216 _pip_args = [] 217 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 218 _pip_args += [ "--no-index" ] 219 } 220 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 221 _pip_args += [ "--no-cache-dir" ] 222 } 223 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 224 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 225 _pip_args += 226 [ "--find-links=" + rebase_path(link_dir, root_build_dir) ] 227 } 228 } 229 230 args = [ "compile" ] 231 232 if (_pip_generate_hashes) { 233 args += [ 234 "--generate-hashes", 235 "--reuse-hashes", 236 ] 237 } 238 239 args += [ 240 "--resolver=backtracking", 241 242 # --allow-unsafe will force pinning pip and setuptools. 243 "--allow-unsafe", 244 "--output-file", 245 rebase_path(_compiled_requirements_file, root_build_dir), 246 247 # Input requirements file: 248 rebase_path(_generated_requirements_file, root_build_dir), 249 ] 250 251 # Pass offline related pip args through the pip-compile command. 252 if (_pip_args != []) { 253 args += [ 254 "--pip-args", 255 string_join(" ", _pip_args), 256 ] 257 } 258 259 # Extra requirements files 260 foreach(_requirements_file, _requirements) { 261 args += [ rebase_path(_requirements_file, root_build_dir) ] 262 } 263 264 inputs = [] 265 266 # NOTE: constraint files are included in the content of the 267 # _generated_requirements_file. This occurs in the 268 # ._generate_3p_requirements target. 269 inputs += _constraints 270 inputs += _requirements 271 inputs += [ _generated_requirements_file ] 272 273 deps = [ 274 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 275 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 276 ] 277 outputs = [ _compiled_requirements_file ] 278 } 279 280 # This target will run 'pip install' in the venv to provide base 281 # dependencies needed to run piptools commands. That is required for the 282 # _compile_requirements sub target. 283 pw_python_action("${target_name}._install_base_3p_deps") { 284 module = "pip" 285 286 # Set the venv to run this pip install in. 287 _pw_internal_run_in_venv = true 288 _skip_installing_external_python_deps = true 289 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 290 291 _base_requirement_file = "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/python_base_requirements.txt" 292 293 args = [ 294 "install", 295 "--requirement", 296 rebase_path(_base_requirement_file, root_build_dir), 297 ] 298 if (_output_logs) { 299 _pip_install_log_file = 300 "$target_gen_dir/$target_name/pip_install_log.txt" 301 args += [ 302 "--log", 303 rebase_path(_pip_install_log_file, root_build_dir), 304 ] 305 outputs = [ _pip_install_log_file ] 306 } 307 308 # NOTE: Constraints should be ignored for this step. 309 310 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 311 args += [ "--no-index" ] 312 } 313 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 314 args += [ "--no-cache-dir" ] 315 } 316 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 317 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 318 args += [ 319 "--find-links", 320 rebase_path(link_dir, root_build_dir), 321 ] 322 } 323 } 324 325 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 326 stamp = true 327 pool = "$dir_pw_build/pool:pip($default_toolchain)" 328 } 329 330 # Install all 3rd party Python dependencies. 331 pw_python_action("${target_name}._install_3p_deps") { 332 module = "pip" 333 334 # Set the venv to run this pip install in. 335 _pw_internal_run_in_venv = true 336 _skip_installing_external_python_deps = true 337 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 338 339 args = pw_build_PYTHON_PIP_DEFAULT_OPTIONS 340 args += [ 341 "install", 342 "--upgrade", 343 ] 344 345 if (_output_logs) { 346 _pip_install_log_file = 347 "$target_gen_dir/$target_name/pip_install_log.txt" 348 args += [ 349 "--log", 350 rebase_path(_pip_install_log_file, root_build_dir), 351 ] 352 } 353 354 if (_pip_generate_hashes) { 355 args += [ "--require-hashes" ] 356 } 357 358 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 359 args += [ "--no-index" ] 360 } 361 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 362 args += [ "--no-cache-dir" ] 363 } 364 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 365 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 366 args += [ 367 "--find-links", 368 rebase_path(link_dir, root_build_dir), 369 ] 370 } 371 } 372 373 # Note: --no-build-isolation should be avoided for installing 3rd party 374 # Python packages that use C/C++ extension modules. 375 # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html 376 inputs = _constraints + _requirements + [ _compiled_requirements_file ] 377 378 # Use the pip-tools compiled requiremets file. This contains the fully 379 # expanded list of deps with constraints applied. 380 if (defined(invoker.source_packages)) { 381 inputs += [ _compiled_requirements_file ] 382 args += [ 383 "--requirement", 384 rebase_path(_compiled_requirements_file, root_build_dir), 385 ] 386 } 387 388 deps = [ 389 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 390 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 391 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 392 ] 393 stamp = true 394 pool = "$dir_pw_build/pool:pip($default_toolchain)" 395 } 396 397 # Target to create a Python package cache for this pw_python_venv. 398 pw_python_action("${target_name}.vendor_wheels") { 399 _wheel_output_dir = "$target_gen_dir/$target_name/wheels" 400 _pip_download_logfile = 401 "$target_gen_dir/$target_name/pip_download_log.txt" 402 _pip_wheel_logfile = "$target_gen_dir/$target_name/pip_wheel_log.txt" 403 metadata = { 404 pw_python_package_wheels = [ _wheel_output_dir ] 405 } 406 407 script = "$dir_pw_build/py/pw_build/generate_python_wheel_cache.py" 408 409 # Set the venv to run this pip install in. 410 _pw_internal_run_in_venv = true 411 _skip_installing_external_python_deps = true 412 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 413 414 args = [ 415 "--pip-download-log", 416 rebase_path(_pip_download_logfile, root_build_dir), 417 "--wheel-dir", 418 rebase_path(_wheel_output_dir, root_build_dir), 419 "-r", 420 rebase_path(_compiled_requirements_file, root_build_dir), 421 ] 422 423 if (pw_build_PYTHON_PIP_DOWNLOAD_ALL_PLATFORMS) { 424 args += [ "--download-all-platforms" ] 425 } 426 427 deps = [ 428 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 429 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 430 ] 431 432 outputs = [ 433 _wheel_output_dir, 434 _pip_wheel_logfile, 435 _pip_download_logfile, 436 ] 437 pool = "$dir_pw_build/pool:pip($default_toolchain)" 438 } 439 440 # End pw_build_PYTHON_TOOLCHAIN check 441 } else { 442 group("${target_name}._compile_requirements") { 443 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 444 } 445 group("${target_name}._install_3p_deps") { 446 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 447 } 448 group("${target_name}.vendor_wheels") { 449 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 450 } 451 } 452 } else { 453 group("${target_name}._compile_requirements") { 454 } 455 group("${target_name}._install_3p_deps") { 456 } 457 group("${target_name}.vendor_wheels") { 458 } 459 } 460 461 # Have this target directly depend on _install_3p_deps 462 group("$target_name") { 463 public_deps = 464 [ ":${target_name}._install_3p_deps($pw_build_PYTHON_TOOLCHAIN)" ] 465 } 466} 467