1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Dataclass for a Python package.""" 15 16from __future__ import annotations 17 18import configparser 19from contextlib import contextmanager 20import copy 21from dataclasses import dataclass, asdict 22import io 23import json 24import os 25from pathlib import Path 26import pprint 27import re 28import shutil 29from typing import Any, Iterable 30 31_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 32 33# List of known environment markers supported by pip. 34# https://peps.python.org/pep-0508/#environment-markers 35_PY_REQUIRE_ENVIRONMENT_MARKER_NAMES = [ 36 'os_name', 37 'sys_platform', 38 'platform_machine', 39 'platform_python_implementation', 40 'platform_release', 41 'platform_system', 42 'platform_version', 43 'python_version', 44 'python_full_version', 45 'implementation_name', 46 'implementation_version', 47 'extra', 48] 49 50 51@contextmanager 52def change_working_dir(directory: Path): 53 original_dir = Path.cwd() 54 try: 55 os.chdir(directory) 56 yield directory 57 finally: 58 os.chdir(original_dir) 59 60 61class UnknownPythonPackageName(Exception): 62 """Exception thrown when a Python package_name cannot be determined.""" 63 64 65class UnknownPythonPackageVersion(Exception): 66 """Exception thrown when a Python package version cannot be determined.""" 67 68 69class MissingSetupSources(Exception): 70 """Exception thrown when a Python package is missing setup source files. 71 72 For example: setup.cfg and pyproject.toml.i 73 """ 74 75 76def _sanitize_install_requires(metadata_dict: dict) -> dict: 77 """Convert install_requires lists into strings joined with line breaks.""" 78 try: 79 install_requires = metadata_dict['options']['install_requires'] 80 if isinstance(install_requires, list): 81 metadata_dict['options']['install_requires'] = '\n'.join( 82 install_requires 83 ) 84 except KeyError: 85 pass 86 return metadata_dict 87 88 89@dataclass 90class PythonPackage: 91 """Class to hold a single Python package's metadata.""" 92 93 sources: list[Path] 94 setup_sources: list[Path] 95 tests: list[Path] 96 inputs: list[Path] 97 gn_target_name: str = '' 98 generate_setup: dict | None = None 99 config: configparser.ConfigParser | None = None 100 101 @staticmethod 102 def from_dict(**kwargs) -> PythonPackage: 103 """Build a PythonPackage instance from a dictionary.""" 104 transformed_kwargs = copy.copy(kwargs) 105 106 # Transform string filenames to Paths 107 for attribute in ['sources', 'tests', 'inputs', 'setup_sources']: 108 transformed_kwargs[attribute] = [Path(s) for s in kwargs[attribute]] 109 110 return PythonPackage(**transformed_kwargs) 111 112 def __post_init__(self): 113 # Read the setup.cfg file if possible 114 if not self.config: 115 self.config = self._load_config() 116 117 @property 118 def setup_dir(self) -> Path | None: 119 if not self.setup_sources: 120 return None 121 # Assuming all setup_source files live in the same parent directory. 122 return self.setup_sources[0].parent 123 124 @property 125 def setup_py(self) -> Path: 126 setup_py = [ 127 setup_file 128 for setup_file in self.setup_sources 129 if str(setup_file).endswith('setup.py') 130 ] 131 # setup.py will not exist for GN generated Python packages 132 assert len(setup_py) == 1 133 return setup_py[0] 134 135 @property 136 def setup_cfg(self) -> Path | None: 137 setup_cfg = [ 138 setup_file 139 for setup_file in self.setup_sources 140 if str(setup_file).endswith('setup.cfg') 141 ] 142 if len(setup_cfg) < 1: 143 return None 144 return setup_cfg[0] 145 146 def as_dict(self) -> dict[Any, Any]: 147 """Return a dict representation of this class.""" 148 self_dict = asdict(self) 149 if self.config: 150 # Expand self.config into text. 151 setup_cfg_text = io.StringIO() 152 self.config.write(setup_cfg_text) 153 self_dict['config'] = setup_cfg_text.getvalue() 154 return self_dict 155 156 @property 157 def package_name(self) -> str: 158 unknown_package_message = ( 159 'Cannot determine the package_name for the Python ' 160 f'library/package: {self.gn_target_name}\n\n' 161 'This could be due to a missing python dependency in GN for:\n' 162 f'{self.gn_target_name}\n\n' 163 ) 164 165 if self.config: 166 try: 167 name = self.config['metadata']['name'] 168 except KeyError: 169 raise UnknownPythonPackageName( 170 unknown_package_message + _pretty_format(self.as_dict()) 171 ) 172 return name 173 top_level_source_dir = self.top_level_source_dir 174 if top_level_source_dir: 175 return top_level_source_dir.name 176 177 actual_gn_target_name = self.gn_target_name.split(':') 178 if len(actual_gn_target_name) < 2: 179 raise UnknownPythonPackageName(unknown_package_message) 180 181 return actual_gn_target_name[-1] 182 183 @property 184 def package_version(self) -> str: 185 version = '' 186 if self.config: 187 try: 188 version = self.config['metadata']['version'] 189 except KeyError: 190 raise UnknownPythonPackageVersion( 191 'Unknown Python package version for: ' 192 + _pretty_format(self.as_dict()) 193 ) 194 return version 195 196 @property 197 def package_dir(self) -> Path: 198 if self.setup_cfg and self.setup_cfg.is_file(): 199 return self.setup_cfg.parent / self.package_name 200 root_source_dir = self.top_level_source_dir 201 if root_source_dir: 202 return root_source_dir 203 if self.sources: 204 return self.sources[0].parent 205 # If no sources available, assume the setup file root is the 206 # package_dir. This may be the case in a package with data files only. 207 return self.setup_sources[0].parent 208 209 @property 210 def top_level_source_dir(self) -> Path | None: 211 source_dir_paths = sorted( 212 set((len(sfile.parts), sfile.parent) for sfile in self.sources), 213 key=lambda s: s[1], 214 ) 215 if not source_dir_paths: 216 return None 217 218 top_level_source_dir = source_dir_paths[0][1] 219 if not top_level_source_dir.is_dir(): 220 return None 221 222 return top_level_source_dir 223 224 def _load_config(self) -> configparser.ConfigParser | None: 225 config = configparser.ConfigParser() 226 227 # Check for a setup.cfg and load that config. 228 if self.setup_cfg: 229 if self.setup_cfg.is_file(): 230 with self.setup_cfg.open() as config_file: 231 config.read_file(config_file) 232 return config 233 if self.setup_cfg.with_suffix('.json').is_file(): 234 return self._load_setup_json_config() 235 236 # Fallback on the generate_setup scope from GN 237 if self.generate_setup: 238 config.read_dict(_sanitize_install_requires(self.generate_setup)) 239 return config 240 return None 241 242 def _load_setup_json_config(self) -> configparser.ConfigParser: 243 assert self.setup_cfg 244 setup_json = self.setup_cfg.with_suffix('.json') 245 config = configparser.ConfigParser() 246 with setup_json.open() as json_fp: 247 json_dict = _sanitize_install_requires(json.load(json_fp)) 248 249 config.read_dict(json_dict) 250 return config 251 252 def copy_sources_to(self, destination: Path) -> None: 253 """Copy this PythonPackage source files to another path.""" 254 new_destination = destination / self.package_dir.name 255 new_destination.mkdir(parents=True, exist_ok=True) 256 shutil.copytree(self.package_dir, new_destination, dirs_exist_ok=True) 257 258 def install_requires_entries(self) -> list[str]: 259 """Convert the install_requires entry into a list of strings.""" 260 this_requires: list[str] = [] 261 # If there's no setup.cfg, do nothing. 262 if not self.config: 263 return this_requires 264 265 # Requires are delimited by newlines or semicolons. 266 # Split existing list on either one. 267 for req in re.split( 268 r' *[\n;] *', self.config['options']['install_requires'] 269 ): 270 # Skip empty lines. 271 if not req: 272 continue 273 # Get the name part part of the dep, ignoring any spaces or 274 # other characters. 275 req_name_match = re.match(r'^(?P<name_part>[A-Za-z0-9_-]+)', req) 276 if not req_name_match: 277 continue 278 req_name = req_name_match.groupdict().get('name_part', '') 279 # Check if this is an environment marker. 280 if req_name in _PY_REQUIRE_ENVIRONMENT_MARKER_NAMES: 281 # Append this req as an environment marker for the previous 282 # requirement. 283 this_requires[-1] += f';{req}' 284 continue 285 # Normal pip requirement, save to this_requires. 286 this_requires.append(req) 287 return this_requires 288 289 290def load_packages( 291 input_list_files: Iterable[Path], ignore_missing=False 292) -> list[PythonPackage]: 293 """Load Python package metadata and configs.""" 294 295 packages = [] 296 for input_path in input_list_files: 297 if ignore_missing and not input_path.is_file(): 298 continue 299 with input_path.open() as input_file: 300 # Each line contains the path to a json file. 301 for json_file in input_file.readlines(): 302 # Load the json as a dict. 303 json_file_path = Path(json_file.strip()).resolve() 304 with json_file_path.open() as json_fp: 305 json_dict = json.load(json_fp) 306 307 packages.append(PythonPackage.from_dict(**json_dict)) 308 return packages 309