1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Evaluates target expressions within a GN build context.""" 15 16from __future__ import annotations 17 18import argparse 19from dataclasses import dataclass 20import enum 21import logging 22import os 23import re 24import sys 25from pathlib import Path 26from typing import ( 27 Callable, 28 Iterable, 29 Iterator, 30 NamedTuple, 31) 32 33_LOG = logging.getLogger(__name__) 34 35 36def abspath(path: Path) -> Path: 37 """Turns a path into an absolute path, not resolving symlinks.""" 38 return Path(os.path.abspath(path)) 39 40 41class GnPaths(NamedTuple): 42 """The set of paths needed to resolve GN paths to filesystem paths.""" 43 44 root: Path 45 build: Path 46 cwd: Path 47 48 # Toolchain label or '' if using the default toolchain 49 toolchain: str 50 51 def resolve(self, gn_path: str) -> Path: 52 """Resolves a GN path to a filesystem path.""" 53 if gn_path.startswith('//'): 54 return abspath(self.root.joinpath(gn_path.lstrip('/'))) 55 56 return abspath(self.cwd.joinpath(gn_path)) 57 58 def resolve_paths(self, gn_paths: str, sep: str = ';') -> str: 59 """Resolves GN paths to filesystem paths in a delimited string.""" 60 return sep.join(str(self.resolve(path)) for path in gn_paths.split(sep)) 61 62 63@dataclass(frozen=True) 64class Label: 65 """Represents a GN label.""" 66 67 name: str 68 dir: Path 69 relative_dir: Path 70 toolchain: Label | None 71 out_dir: Path 72 gen_dir: Path 73 74 def __init__(self, paths: GnPaths, label: str): 75 # Use this lambda to set attributes on this frozen dataclass. 76 def set_attr(attr, val): 77 return object.__setattr__(self, attr, val) 78 79 # Handle explicitly-specified toolchains 80 if label.endswith(')'): 81 label, toolchain = label[:-1].rsplit('(', 1) 82 else: 83 # Prevent infinite recursion for toolchains 84 toolchain = paths.toolchain if paths.toolchain != label else '' 85 86 set_attr('toolchain', Label(paths, toolchain) if toolchain else None) 87 88 # Split off the :target, if provided, or use the last part of the path. 89 try: 90 directory, name = label.rsplit(':', 1) 91 except ValueError: 92 directory, name = label, label.rsplit('/', 1)[-1] 93 94 set_attr('name', name) 95 96 # Resolve the directory to an absolute path 97 set_attr('dir', paths.resolve(directory)) 98 set_attr('relative_dir', self.dir.relative_to(abspath(paths.root))) 99 100 set_attr( 101 'out_dir', 102 paths.build / self.toolchain_name() / 'obj' / self.relative_dir, 103 ) 104 set_attr( 105 'gen_dir', 106 paths.build / self.toolchain_name() / 'gen' / self.relative_dir, 107 ) 108 109 def gn_label(self) -> str: 110 label = f'//{self.relative_dir.as_posix()}:{self.name}' 111 return f'{label}({self.toolchain!r})' if self.toolchain else label 112 113 def toolchain_name(self) -> str: 114 return self.toolchain.name if self.toolchain else '' 115 116 def __repr__(self) -> str: 117 return self.gn_label() 118 119 120class _Artifact(NamedTuple): 121 path: Path 122 variables: dict[str, str] 123 124 125# Matches a non-phony build statement. 126_GN_NINJA_BUILD_STATEMENT = re.compile(r'^build (.+):[ \n](?!phony\b)') 127 128_OBJECTS_EXTENSIONS = ('.o',) 129 130# Extensions used for compilation artifacts. 131_MAIN_ARTIFACTS = '', '.elf', '.a', '.so', '.dylib', '.exe', '.lib', '.dll' 132 133 134def _get_artifact(entries: list[str]) -> _Artifact: 135 """Attempts to resolve which artifact to use if there are multiple. 136 137 Selects artifacts based on extension. This will not work if a toolchain 138 creates multiple compilation artifacts from one command (e.g. .a and .elf). 139 """ 140 assert entries, "There should be at least one entry here!" 141 142 if len(entries) == 1: 143 return _Artifact(Path(entries[0]), {}) 144 145 filtered = [p for p in entries if Path(p).suffix in _MAIN_ARTIFACTS] 146 147 if len(filtered) == 1: 148 return _Artifact(Path(filtered[0]), {}) 149 150 raise ExpressionError( 151 f'Expected 1, but found {len(filtered)} artifacts, after filtering for ' 152 f'extensions {", ".join(repr(e) for e in _MAIN_ARTIFACTS)}: {entries}' 153 ) 154 155 156def _parse_build_artifacts(fd) -> Iterator[_Artifact]: 157 """Partially parses the build statements in a Ninja file.""" 158 lines = iter(fd) 159 160 def next_line(): 161 try: 162 return next(lines) 163 except StopIteration: 164 return None 165 166 # Serves as the parse state (only two states) 167 artifact: _Artifact | None = None 168 169 line = next_line() 170 171 while line is not None: 172 if artifact: 173 if line.startswith(' '): # build variable statements are indented 174 key, value = (a.strip() for a in line.split('=', 1)) 175 artifact.variables[key] = value 176 line = next_line() 177 else: 178 yield artifact 179 artifact = None 180 else: 181 match = _GN_NINJA_BUILD_STATEMENT.match(line) 182 if match: 183 artifact = _get_artifact(match.group(1).split()) 184 185 line = next_line() 186 187 if artifact: 188 yield artifact 189 190 191def _search_target_ninja( 192 ninja_file: Path, target: Label 193) -> tuple[Path | None, list[Path]]: 194 """Parses the main output file and object files from <target>.ninja.""" 195 196 artifact: Path | None = None 197 objects: list[Path] = [] 198 199 _LOG.debug('Parsing target Ninja file %s for %s', ninja_file, target) 200 201 with ninja_file.open() as fd: 202 for path, _ in _parse_build_artifacts(fd): 203 # Older GN used .stamp files when there is no build artifact. 204 if path.suffix == '.stamp': 205 continue 206 207 if str(path).endswith(_OBJECTS_EXTENSIONS): 208 objects.append(Path(path)) 209 else: 210 assert not artifact, f'Multiple artifacts for {target}!' 211 artifact = Path(path) 212 213 return artifact, objects 214 215 216def _search_toolchain_ninja( 217 ninja_file: Path, paths: GnPaths, target: Label 218) -> Path | None: 219 """Searches the toolchain.ninja file for outputs from the provided target. 220 221 Files created by an action appear in toolchain.ninja instead of in their own 222 <target>.ninja. If the specified target has a single output file in 223 toolchain.ninja, this function returns its path. 224 """ 225 226 _LOG.debug('Searching toolchain Ninja file %s for %s', ninja_file, target) 227 228 # Older versions of GN used a .stamp file to signal completion of a target. 229 stamp_dir = target.out_dir.relative_to(paths.build).as_posix() 230 stamp_tool = 'stamp' 231 if target.toolchain_name() != '': 232 stamp_tool = f'{target.toolchain_name()}_stamp' 233 stamp_statement = f'build {stamp_dir}/{target.name}.stamp: {stamp_tool} ' 234 235 # Newer GN uses a phony Ninja target to signal completion of a target. 236 phony_dir = Path( 237 target.toolchain_name(), 'phony', target.relative_dir 238 ).as_posix() 239 phony_statement = f'build {phony_dir}/{target.name}: phony ' 240 241 with ninja_file.open() as fd: 242 for line in fd: 243 for statement in (phony_statement, stamp_statement): 244 if line.startswith(statement): 245 output_files = line[len(statement) :].strip().split() 246 if len(output_files) == 1: 247 return Path(output_files[0]) 248 249 break 250 251 return None 252 253 254def _search_ninja_files( 255 paths: GnPaths, target: Label 256) -> tuple[bool, Path | None, list[Path]]: 257 ninja_file = target.out_dir / f'{target.name}.ninja' 258 if ninja_file.exists(): 259 return (True, *_search_target_ninja(ninja_file, target)) 260 261 ninja_file = paths.build / target.toolchain_name() / 'toolchain.ninja' 262 if ninja_file.exists(): 263 return True, _search_toolchain_ninja(ninja_file, paths, target), [] 264 265 return False, None, [] 266 267 268@dataclass(frozen=True) 269class TargetInfo: 270 """Provides information about a target parsed from a .ninja file.""" 271 272 label: Label 273 generated: bool # True if the Ninja files for this target were generated. 274 artifact: Path | None 275 object_files: tuple[Path] 276 277 def __init__(self, paths: GnPaths, target: str): 278 object.__setattr__(self, 'label', Label(paths, target)) 279 280 generated, artifact, objects = _search_ninja_files(paths, self.label) 281 282 object.__setattr__(self, 'generated', generated) 283 object.__setattr__(self, 'artifact', artifact) 284 object.__setattr__(self, 'object_files', tuple(objects)) 285 286 def __repr__(self) -> str: 287 return repr(self.label) 288 289 290class ExpressionError(Exception): 291 """An error occurred while parsing an expression.""" 292 293 294class _ArgAction(enum.Enum): 295 APPEND = 0 296 OMIT = 1 297 EMIT_NEW = 2 298 299 300class _Expression: 301 def __init__(self, match: re.Match, ending: int): 302 self._match = match 303 self._ending = ending 304 305 @property 306 def string(self): 307 return self._match.string 308 309 @property 310 def end(self) -> int: 311 return self._ending + len(_ENDING) 312 313 def contents(self) -> str: 314 return self.string[self._match.end() : self._ending] 315 316 def expression(self) -> str: 317 return self.string[self._match.start() : self.end] 318 319 320_Actions = Iterator[tuple[_ArgAction, str]] 321 322 323def _target_file(paths: GnPaths, expr: _Expression) -> _Actions: 324 target = TargetInfo(paths, expr.contents()) 325 326 if not target.generated: 327 raise ExpressionError(f'Target {target} has not been generated by GN!') 328 329 if target.artifact is None: 330 raise ExpressionError(f'Target {target} has no output file!') 331 332 yield _ArgAction.APPEND, str(target.artifact) 333 334 335def _target_file_if_exists(paths: GnPaths, expr: _Expression) -> _Actions: 336 target = TargetInfo(paths, expr.contents()) 337 338 if target.generated: 339 if target.artifact is None: 340 raise ExpressionError(f'Target {target} has no output file!') 341 342 if paths.build.joinpath(target.artifact).exists(): 343 yield _ArgAction.APPEND, str(target.artifact) 344 return 345 346 yield _ArgAction.OMIT, '' 347 348 349def _target_objects(paths: GnPaths, expr: _Expression) -> _Actions: 350 if expr.expression() != expr.string: 351 raise ExpressionError( 352 f'The expression "{expr.expression()}" in "{expr.string}" may ' 353 'expand to multiple arguments, so it cannot be used alongside ' 354 'other text or expressions' 355 ) 356 357 target = TargetInfo(paths, expr.contents()) 358 if not target.generated: 359 raise ExpressionError(f'Target {target} has not been generated by GN!') 360 361 for obj in target.object_files: 362 yield _ArgAction.EMIT_NEW, str(obj) 363 364 365# TODO: b/234886742 - Replace expressions with native GN features when possible. 366_FUNCTIONS: dict[str, Callable[[GnPaths, _Expression], _Actions]] = { 367 'TARGET_FILE': _target_file, 368 'TARGET_FILE_IF_EXISTS': _target_file_if_exists, 369 'TARGET_OBJECTS': _target_objects, 370} 371 372_START_EXPRESSION = re.compile(fr'<({"|".join(_FUNCTIONS)})\(') 373_ENDING = ')>' 374 375 376def _expand_arguments(paths: GnPaths, string: str) -> _Actions: 377 pos = 0 378 379 for match in _START_EXPRESSION.finditer(string): 380 if pos != match.start(): 381 yield _ArgAction.APPEND, string[pos : match.start()] 382 383 ending = string.find(_ENDING, match.end()) 384 if ending == -1: 385 raise ExpressionError( 386 f'Parse error: no terminating "{_ENDING}" ' 387 f'was found for "{string[match.start():]}"' 388 ) 389 390 expression = _Expression(match, ending) 391 yield from _FUNCTIONS[match.group(1)](paths, expression) 392 393 pos = expression.end 394 395 if pos < len(string): 396 yield _ArgAction.APPEND, string[pos:] 397 398 399def expand_expressions(paths: GnPaths, arg: str) -> Iterable[str]: 400 """Expands <FUNCTION(...)> expressions; yields zero or more arguments.""" 401 if arg == '': 402 return [''] 403 404 expanded_args: list[list[str]] = [[]] 405 406 for action, piece in _expand_arguments(paths, arg): 407 if action is _ArgAction.OMIT: 408 return [] 409 410 expanded_args[-1].append(piece) 411 if action is _ArgAction.EMIT_NEW: 412 expanded_args.append([]) 413 414 return (''.join(arg) for arg in expanded_args if arg) 415 416 417def _parse_args() -> argparse.Namespace: 418 def file_pair(s): 419 return tuple(Path(p) for p in s.split(':')) 420 421 parser = argparse.ArgumentParser(description=__doc__) 422 parser.add_argument( 423 '--gn-root', 424 type=Path, 425 required=True, 426 help=( 427 'Path to the root of the GN tree; ' 428 'value of rebase_path("//", root_build_dir)' 429 ), 430 ) 431 parser.add_argument( 432 '--current-path', 433 type=Path, 434 required=True, 435 help='Value of rebase_path(".", root_build_dir)', 436 ) 437 parser.add_argument( 438 '--default-toolchain', required=True, help='Value of default_toolchain' 439 ) 440 parser.add_argument( 441 '--current-toolchain', required=True, help='Value of current_toolchain' 442 ) 443 parser.add_argument( 444 'files', 445 metavar='FILE', 446 nargs='+', 447 type=file_pair, 448 help='Pairs of src:dest files to scan for expressions to evaluate', 449 ) 450 return parser.parse_args() 451 452 453def _resolve_expressions_in_file(src: Path, dst: Path, paths: GnPaths): 454 dst.write_text(''.join(expand_expressions(paths, src.read_text()))) 455 456 457def main( 458 gn_root: Path, 459 current_path: Path, 460 default_toolchain: str, 461 current_toolchain: str, 462 files: Iterable[tuple[Path, Path]], 463) -> int: 464 """Evaluates GN target expressions within a list of files. 465 466 Modifies the files in-place with their resolved contents. 467 """ 468 tool = current_toolchain if current_toolchain != default_toolchain else '' 469 paths = GnPaths( 470 root=abspath(gn_root), 471 build=Path.cwd(), 472 cwd=abspath(current_path), 473 toolchain=tool, 474 ) 475 476 for src, dst in files: 477 try: 478 _resolve_expressions_in_file(src, dst, paths) 479 except ExpressionError as err: 480 _LOG.error('Error evaluating expressions in %s:', src) 481 _LOG.error(' %s', err) 482 return 1 483 484 return 0 485 486 487if __name__ == '__main__': 488 sys.exit(main(**vars(_parse_args()))) 489