1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Build a Python Source tree.""" 15 16import argparse 17import configparser 18from datetime import datetime 19import io 20from pathlib import Path 21import re 22import shutil 23import subprocess 24import tempfile 25from typing import Iterable 26 27import setuptools # type: ignore 28 29try: 30 from pw_build.python_package import ( 31 PythonPackage, 32 load_packages, 33 change_working_dir, 34 ) 35 from pw_build.generate_python_package import PYPROJECT_FILE 36 37except ImportError: 38 # Load from python_package from this directory if pw_build is not available. 39 from python_package import ( # type: ignore 40 PythonPackage, 41 load_packages, 42 change_working_dir, 43 ) 44 from generate_python_package import PYPROJECT_FILE # type: ignore 45 46 47def _parse_args(): 48 parser = argparse.ArgumentParser(description=__doc__) 49 parser.add_argument( 50 '--repo-root', 51 type=Path, 52 help='Path to the root git repo.', 53 ) 54 parser.add_argument( 55 '--tree-destination-dir', type=Path, help='Path to output directory.' 56 ) 57 parser.add_argument( 58 '--include-tests', 59 action='store_true', 60 help='Include tests in the tests dir.', 61 ) 62 63 parser.add_argument( 64 '--setupcfg-common-file', 65 type=Path, 66 help='A file containing the common set of options for' 67 'incluing in the merged setup.cfg provided version.', 68 ) 69 parser.add_argument( 70 '--setupcfg-version-append-git-sha', 71 action='store_true', 72 help='Append the current git SHA to the setup.cfg ' 'version.', 73 ) 74 parser.add_argument( 75 '--setupcfg-version-append-date', 76 action='store_true', 77 help='Append the current date to the setup.cfg ' 'version.', 78 ) 79 parser.add_argument( 80 '--setupcfg-override-name', help='Override metadata.name in setup.cfg' 81 ) 82 parser.add_argument( 83 '--setupcfg-override-version', 84 help='Override metadata.version in setup.cfg', 85 ) 86 parser.add_argument( 87 '--create-default-pyproject-toml', 88 action='store_true', 89 help='Generate a default pyproject.toml file', 90 ) 91 92 parser.add_argument( 93 '--extra-files', 94 nargs='+', 95 help='Paths to extra files that should be included in the output dir.', 96 ) 97 98 parser.add_argument( 99 '--setupcfg-extra-files-in-package-data', 100 action='store_true', 101 help='List --extra-files in [options.package_data]', 102 ) 103 104 parser.add_argument( 105 '--auto-create-package-data-init-py-files', 106 action='store_true', 107 help=( 108 'Create __init__.py files as needed in subdirs of extra_files ' 109 'when including in [options.package_data].' 110 ), 111 ) 112 113 parser.add_argument( 114 '--input-list-files', 115 nargs='+', 116 type=Path, 117 help='Paths to text files containing lists of Python package metadata ' 118 'json files.', 119 ) 120 121 return parser.parse_args() 122 123 124class UnknownGitSha(Exception): 125 """Exception thrown when the current git SHA cannot be found.""" 126 127 128def get_current_git_sha(repo_root: Path | None = None) -> str: 129 if not repo_root: 130 repo_root = Path.cwd() 131 git_command = [ 132 'git', 133 '-C', 134 str(repo_root), 135 'log', 136 '-1', 137 '--pretty=format:%h', 138 ] 139 140 process = subprocess.run( 141 git_command, 142 stdout=subprocess.PIPE, 143 stderr=subprocess.STDOUT, 144 ) 145 gitsha = process.stdout.decode() 146 if process.returncode != 0 or not gitsha: 147 error_output = f'\n"{git_command}" failed with:' f'\n{gitsha}' 148 if process.stderr: 149 error_output += f'\n{process.stderr.decode()}' 150 raise UnknownGitSha( 151 'Could not determine the current git SHA.' + error_output 152 ) 153 return gitsha.strip() 154 155 156def get_current_date() -> str: 157 return datetime.now().strftime('%Y%m%d%H%M%S') 158 159 160class UnexpectedConfigSection(Exception): 161 """Exception thrown when the common config contains unexpected values.""" 162 163 164def load_common_config( 165 common_config: Path | None = None, 166 package_name_override: str | None = None, 167 package_version_override: str | None = None, 168 append_git_sha: bool = False, 169 append_date: bool = False, 170 repo_root: Path | None = None, 171) -> configparser.ConfigParser: 172 """Load an existing ConfigParser file and update metadata.version.""" 173 config = configparser.ConfigParser() 174 if common_config: 175 config.read(common_config) 176 177 # Metadata and option sections need to exist. 178 if not config.has_section('metadata'): 179 config['metadata'] = {} 180 if not config.has_section('options'): 181 config['options'] = {} 182 183 if package_name_override: 184 config['metadata']['name'] = package_name_override 185 if package_version_override: 186 config['metadata']['version'] = package_version_override 187 188 # Check for existing values that should not be present 189 if config.has_option('options', 'packages'): 190 value = str(config['options']['packages']) 191 raise UnexpectedConfigSection( 192 f'[options] packages already defined as: {value}' 193 ) 194 195 # Append build metadata if applicable. 196 build_metadata = [] 197 if append_date: 198 build_metadata.append(get_current_date()) 199 if append_git_sha: 200 build_metadata.append(get_current_git_sha(repo_root)) 201 if build_metadata: 202 version_prefix = config['metadata']['version'] 203 build_metadata_text = '.'.join(build_metadata) 204 config['metadata'][ 205 'version' 206 ] = f'{version_prefix}+{build_metadata_text}' 207 return config 208 209 210def _update_package_data_value( 211 config: configparser.ConfigParser, key: str, value: str 212) -> None: 213 existing_values = config['options.package_data'].get(key, '').splitlines() 214 new_value = '\n'.join(sorted(set(existing_values + value.splitlines()))) 215 # Remove any empty lines 216 new_value = new_value.replace('\n\n', '\n') 217 config['options.package_data'][key] = new_value 218 219 220def update_config_with_packages( 221 config: configparser.ConfigParser, 222 python_packages: Iterable[PythonPackage], 223) -> None: 224 """Merge setup.cfg files from a set of python packages.""" 225 config['options']['packages'] = 'find:' 226 if not config.has_section('options.package_data'): 227 config['options.package_data'] = {} 228 if not config.has_section('options.entry_points'): 229 config['options.entry_points'] = {} 230 231 # Save a list of packages being bundled. 232 included_packages = [pkg.package_name for pkg in python_packages] 233 234 for pkg in python_packages: 235 # Skip this package if no setup.cfg is defined. 236 if not pkg.config: 237 continue 238 239 # Collect install_requires 240 if pkg.config.has_option('options', 'install_requires'): 241 existing_requires = config['options'].get('install_requires', '\n') 242 243 new_requires = existing_requires.splitlines() 244 new_requires += pkg.install_requires_entries() 245 # Remove requires already included in this merged config. 246 new_requires = [ 247 line 248 for line in new_requires 249 if line and line not in included_packages 250 ] 251 # Remove duplictes and sort require list. 252 new_requires_text = '\n' + '\n'.join(sorted(set(new_requires))) 253 config['options']['install_requires'] = new_requires_text 254 255 # Collect package_data 256 if pkg.config.has_section('options.package_data'): 257 for key, value in pkg.config['options.package_data'].items(): 258 _update_package_data_value(config, key, value) 259 260 # Collect entry_points 261 if pkg.config.has_section('options.entry_points'): 262 for key, value in pkg.config['options.entry_points'].items(): 263 existing_entry_points = config['options.entry_points'].get( 264 key, '' 265 ) 266 new_entry_points = '\n'.join([existing_entry_points, value]) 267 # Remove any empty lines 268 new_entry_points = new_entry_points.replace('\n\n', '\n') 269 config['options.entry_points'][key] = new_entry_points 270 271 272def update_config_with_package_data( 273 config: configparser.ConfigParser, 274 extra_files_list: list[Path], 275 auto_create_init_py_files: bool, 276 tree_destination_dir: Path, 277) -> None: 278 """Create options.package_data entries from a list of paths.""" 279 for path in extra_files_list: 280 relative_file = path.relative_to(tree_destination_dir) 281 282 # Update options.package_data config section 283 root_package_dir = list(relative_file.parents)[-2] 284 _update_package_data_value( 285 config, 286 str(root_package_dir), 287 str(relative_file.relative_to(root_package_dir)), 288 ) 289 290 # Add an __init__.py file to subdirectories 291 if ( 292 auto_create_init_py_files 293 and relative_file.parent != tree_destination_dir 294 ): 295 init_py = ( 296 tree_destination_dir / relative_file.parent / '__init__.py' 297 ) 298 if not init_py.exists(): 299 init_py.touch() 300 301 302def write_config( 303 final_config: configparser.ConfigParser, 304 tree_destination_dir: Path, 305 common_config: Path | None = None, 306) -> None: 307 """Write a the final setup.cfg file with license comment block.""" 308 comment_block_text = '' 309 if common_config: 310 # Get the license comment block from the common_config. 311 comment_block_match = re.search( 312 r'((^#.*?[\r\n])*)([^#])', common_config.read_text(), re.MULTILINE 313 ) 314 if comment_block_match: 315 comment_block_text = comment_block_match.group(1) 316 317 setup_cfg_file = tree_destination_dir.resolve() / 'setup.cfg' 318 setup_cfg_text = io.StringIO() 319 final_config.write(setup_cfg_text) 320 setup_cfg_file.write_text(comment_block_text + setup_cfg_text.getvalue()) 321 322 323def setuptools_build_with_base( 324 pkg: PythonPackage, build_base: Path, include_tests: bool = False 325) -> Path: 326 """Run setuptools build for this package.""" 327 328 # If there is no setup_dir or setup_sources, just copy this packages 329 # source files. 330 if not pkg.setup_dir: 331 pkg.copy_sources_to(build_base) 332 return build_base 333 # Create the lib install dir in case it doesn't exist. 334 lib_dir_path = build_base / 'lib' 335 lib_dir_path.mkdir(parents=True, exist_ok=True) 336 337 starting_directory = Path.cwd() 338 # cd to the location of setup.py 339 with change_working_dir(pkg.setup_dir): 340 # Run build with temp build-base location 341 # Note: New files will be placed inside lib_dir_path 342 setuptools.setup( 343 script_args=[ 344 'build', 345 '--force', 346 '--build-base', 347 str(build_base), 348 ] 349 ) 350 351 new_pkg_dir = lib_dir_path / pkg.package_name 352 # If tests should be included, copy them to the tests dir 353 if include_tests and pkg.tests: 354 test_dir_path = new_pkg_dir / 'tests' 355 test_dir_path.mkdir(parents=True, exist_ok=True) 356 357 for test_source_path in pkg.tests: 358 shutil.copy( 359 starting_directory / test_source_path, test_dir_path 360 ) 361 362 return lib_dir_path 363 364 365def build_python_tree( 366 python_packages: Iterable[PythonPackage], 367 tree_destination_dir: Path, 368 include_tests: bool = False, 369) -> None: 370 """Install PythonPackages to a destination directory.""" 371 372 # Create the root destination directory. 373 destination_path = tree_destination_dir.resolve() 374 # Delete any existing files 375 shutil.rmtree(destination_path, ignore_errors=True) 376 destination_path.mkdir(parents=True, exist_ok=True) 377 378 for pkg in python_packages: 379 # Define a temporary location to run setup.py build in. 380 with tempfile.TemporaryDirectory() as build_base_name: 381 build_base = Path(build_base_name) 382 383 lib_dir_path = setuptools_build_with_base( 384 pkg, build_base, include_tests=include_tests 385 ) 386 387 # Move installed files from the temp build-base into 388 # destination_path. 389 shutil.copytree(lib_dir_path, destination_path, dirs_exist_ok=True) 390 391 # Clean build base lib folder for next install 392 shutil.rmtree(lib_dir_path, ignore_errors=True) 393 394 395def copy_extra_files(extra_file_strings: Iterable[str]) -> list[Path]: 396 """Copy extra files to their destinations.""" 397 output_files: list[Path] = [] 398 399 if not extra_file_strings: 400 return output_files 401 402 for extra_file_string in extra_file_strings: 403 # Convert 'source > destination' strings to Paths. 404 input_output = re.split(r' *> *', extra_file_string) 405 source_file = Path(input_output[0]) 406 dest_file = Path(input_output[1]) 407 408 if not source_file.exists(): 409 raise FileNotFoundError( 410 f'extra_file "{source_file}" not found.\n' 411 f' Defined by: "{extra_file_string}"' 412 ) 413 414 # Copy files and make parent directories. 415 dest_file.parent.mkdir(parents=True, exist_ok=True) 416 # Raise an error if the destination file already exists. 417 if dest_file.exists(): 418 raise FileExistsError( 419 f'Copying "{source_file}" would overwrite "{dest_file}"' 420 ) 421 422 shutil.copy(source_file, dest_file) 423 output_files.append(dest_file) 424 425 return output_files 426 427 428def _main(): 429 args = _parse_args() 430 431 # Check the common_config file exists if provided. 432 if args.setupcfg_common_file: 433 assert args.setupcfg_common_file.is_file() 434 435 py_packages = load_packages(args.input_list_files) 436 437 build_python_tree( 438 python_packages=py_packages, 439 tree_destination_dir=args.tree_destination_dir, 440 include_tests=args.include_tests, 441 ) 442 extra_files_list = copy_extra_files(args.extra_files) 443 444 if args.create_default_pyproject_toml: 445 pyproject_path = args.tree_destination_dir / 'pyproject.toml' 446 pyproject_path.write_text(PYPROJECT_FILE) 447 448 if args.setupcfg_common_file or ( 449 args.setupcfg_override_name and args.setupcfg_override_version 450 ): 451 config = load_common_config( 452 common_config=args.setupcfg_common_file, 453 package_name_override=args.setupcfg_override_name, 454 package_version_override=args.setupcfg_override_version, 455 append_git_sha=args.setupcfg_version_append_git_sha, 456 append_date=args.setupcfg_version_append_date, 457 repo_root=args.repo_root, 458 ) 459 460 update_config_with_packages(config=config, python_packages=py_packages) 461 462 if args.setupcfg_extra_files_in_package_data: 463 update_config_with_package_data( 464 config, 465 extra_files_list, 466 args.auto_create_package_data_init_py_files, 467 args.tree_destination_dir, 468 ) 469 470 write_config( 471 common_config=args.setupcfg_common_file, 472 final_config=config, 473 tree_destination_dir=args.tree_destination_dir, 474 ) 475 476 477if __name__ == '__main__': 478 _main() 479