1# Copyright 2018 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import json 16import os 17import re 18import shutil 19import fnmatch 20from copy import deepcopy 21from pathlib import Path 22from typing import Dict, List, Optional 23import jinja2 24from datetime import date 25 26from synthtool import shell, _tracked_paths 27from synthtool.gcp import partials 28from synthtool.languages import node, node_mono_repo 29from synthtool.log import logger 30from synthtool.sources import git, templates 31 32PathOrStr = templates.PathOrStr 33TEMPLATES_URL: str = git.make_repo_clone_url("googleapis/synthtool") 34DEFAULT_TEMPLATES_PATH = "synthtool/gcp/templates" 35LOCAL_TEMPLATES: Optional[str] = os.environ.get("SYNTHTOOL_TEMPLATES") 36 37 38class CommonTemplates: 39 def __init__(self, template_path: Optional[Path] = None): 40 if template_path: 41 self._template_root = template_path 42 elif LOCAL_TEMPLATES: 43 logger.debug(f"Using local templates at {LOCAL_TEMPLATES}") 44 self._template_root = Path(LOCAL_TEMPLATES) 45 else: 46 templates_git = git.clone(TEMPLATES_URL) 47 self._template_root = templates_git / DEFAULT_TEMPLATES_PATH 48 49 self._templates = templates.Templates(self._template_root) 50 self.excludes = [] # type: List[str] 51 52 def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path: 53 # load common repo meta information (metadata that's not language specific). 54 if "metadata" in kwargs: 55 self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir) 56 # if no samples were found, don't attempt to render a 57 # samples/README.md. 58 if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]: 59 self.excludes.append("samples/README.md") 60 61 t = templates.TemplateGroup(self._template_root / directory, self.excludes) 62 63 if "repository" in kwargs["metadata"] and "repo" in kwargs["metadata"]: 64 kwargs["metadata"]["repo"]["default_branch"] = _get_default_branch_name( 65 kwargs["metadata"]["repository"] 66 ) 67 68 # TODO: migrate to python.py once old sample gen is deprecated 69 if directory == "python_samples": 70 t.env.globals["get_help"] = lambda filename: shell.run( 71 ["python", filename, "--help"] 72 ).stdout 73 74 result = t.render(**kwargs) 75 _tracked_paths.add(result) 76 77 return result 78 79 def py_samples(self, **kwargs) -> List[Path]: 80 """ 81 Handles generation of README.md templates for Python samples 82 - Determines whether generation is being done in a client library or in a samples 83 folder automatically 84 - Otherwise accepts manually set sample_project_dir through kwargs metadata 85 - Delegates generation of additional sample documents alternate/overridden folders 86 through py_samples_override() 87 """ 88 # kwargs["metadata"] is required to load values from .repo-metadata.json 89 if "metadata" not in kwargs: 90 kwargs["metadata"] = {} 91 92 # load common repo meta information (metadata that's not language specific). 93 self._load_generic_metadata(kwargs["metadata"]) 94 95 # temporary exclusion prior to old templates being migrated out 96 self.excludes.extend( 97 [ 98 "README.rst", 99 "auth_api_key.tmpl.rst", 100 "auth.tmpl.rst", 101 "install_deps.tmpl.rst", 102 "install_portaudio.tmpl.rst", 103 "noxfile.py.j2", 104 ] 105 ) 106 107 # ensure samples will generate 108 kwargs["metadata"]["samples"] = True 109 110 # determine if in client lib and set custom root sample dir if specified, else None 111 in_client_library = Path("samples").exists() 112 sample_project_dir = kwargs["metadata"]["repo"].get("sample_project_dir") 113 114 if sample_project_dir is None: # Not found in metadata 115 if in_client_library: 116 sample_project_dir = "samples" 117 else: 118 sample_project_dir = "." 119 elif not Path(sample_project_dir).exists(): 120 raise Exception(f"'{sample_project_dir}' does not exist") 121 122 override_paths_to_samples: Dict[ 123 str, List[str] 124 ] = {} # Dict of format { override_path : sample(s) } 125 samples_dict = deepcopy(kwargs["metadata"]["repo"].get("samples")) 126 default_samples_dict = [] # Dict which will generate in sample_project_dir 127 128 # Iterate through samples to store override_paths_to_samples for all existing 129 # override paths 130 for sample_idx, sample in enumerate(samples_dict): 131 override_path = samples_dict[sample_idx].get("override_path") 132 133 if override_path is not None: 134 # add absolute path to metadata so `python foo.py --help` succeeds 135 if sample.get("file") is not None: 136 path = os.path.join( 137 sample_project_dir, override_path, sample.get("file") 138 ) 139 sample["abs_path"] = Path(path).resolve() 140 141 cur_override_sample = override_paths_to_samples.get(override_path) 142 # Base case: No samples are yet planned to gen in this override dir 143 if cur_override_sample is None: 144 override_paths_to_samples[override_path] = [sample] 145 # Else: Sample docs will be generated in README merged with other 146 # sample doc(s) already planned to generate in this dir 147 else: 148 cur_override_sample.append(sample) 149 override_paths_to_samples[override_path] = cur_override_sample 150 # If override path none, will be generated in the default 151 # folder: sample_project_dir 152 else: 153 if sample.get("file") is not None: 154 path = os.path.join(sample_project_dir, sample.get("file")) 155 sample["abs_path"] = Path(path).resolve() 156 default_samples_dict.append(sample) 157 158 # List of paths to tempdirs which will be copied into sample folders 159 result = [] 160 161 # deep copy is req. here to avoid kwargs being affected 162 overridden_samples_kwargs = deepcopy(kwargs) 163 for override_path in override_paths_to_samples: 164 # Generate override sample docs 165 result.append( 166 self.py_samples_override( 167 root=sample_project_dir, 168 override_path=override_path, 169 override_samples=override_paths_to_samples[override_path], 170 **overridden_samples_kwargs, 171 ) 172 ) 173 kwargs["metadata"]["repo"]["samples"] = default_samples_dict 174 175 logger.debug( 176 f"Generating templates for samples directory '{sample_project_dir}'" 177 ) 178 kwargs["subdir"] = sample_project_dir 179 # Generate default sample docs 180 result.append(self._generic_library("python_samples", **kwargs)) 181 182 for path in result: 183 # .add() records the root of the paths and needs to be applied to each 184 _tracked_paths.add(path) 185 186 return result 187 188 def py_samples_override( 189 self, root, override_path, override_samples, **overridden_samples_kwargs 190 ) -> Path: 191 """ 192 Handles additional generation of READMEs where "override_path"s 193 are set in one or more samples' metadata 194 """ 195 overridden_samples_kwargs["metadata"]["repo"][ 196 "sample_project_dir" 197 ] = override_path 198 # Set samples metadata to ONLY samples intended to generate 199 # under this directory (override_path) 200 overridden_samples_kwargs["metadata"]["repo"]["samples"] = override_samples 201 if root != ".": 202 override_path = Path(root) / override_path 203 204 logger.debug(f"Generating templates for override path '{override_path}'") 205 206 overridden_samples_kwargs["subdir"] = override_path 207 return self._generic_library("python_samples", **overridden_samples_kwargs) 208 209 def python_notebooks(self, **kwargs) -> Path: 210 # kwargs["metadata"] is required to load values from .repo-metadata.json 211 if "metadata" not in kwargs: 212 kwargs["metadata"] = {} 213 return self._generic_library("python_notebooks", **kwargs) 214 215 def py_mono_repo_library(self, relative_dir, **kwargs) -> Path: 216 # kwargs["metadata"] is required to load values from .repo-metadata.json 217 if "metadata" not in kwargs: 218 kwargs["metadata"] = {} 219 220 # load common repo meta information (metadata that's not language specific). 221 self._load_generic_metadata(kwargs["metadata"], relative_dir) 222 223 # initialize default_version if it doesn't exist in kwargs["metadata"]['repo'] 224 if "default_version" not in kwargs["metadata"]["repo"]: 225 kwargs["metadata"]["repo"]["default_version"] = "" 226 227 # Don't add `docs/index.rst` if `versions` is not provided or `default_version` is empty 228 if ( 229 "versions" not in kwargs 230 or not kwargs["metadata"]["repo"]["default_version"] 231 or kwargs["metadata"]["repo"]["default_version"] == "apiVersion" 232 ): 233 self.excludes += ["docs/index.rst"] 234 235 # If the directory `google/cloud` exists, add kwargs to signal that the client library is for a Cloud API 236 if Path("google/cloud").exists(): 237 kwargs["is_google_cloud_api"] = True 238 239 return self._generic_library("python_mono_repo_library", relative_dir, **kwargs) 240 241 def py_library(self, **kwargs) -> Path: 242 # kwargs["metadata"] is required to load values from .repo-metadata.json 243 if "metadata" not in kwargs: 244 kwargs["metadata"] = {} 245 246 # load common repo meta information (metadata that's not language specific). 247 self._load_generic_metadata(kwargs["metadata"]) 248 249 # initialize default_version if it doesn't exist in kwargs["metadata"]['repo'] 250 if "default_version" not in kwargs["metadata"]["repo"]: 251 kwargs["metadata"]["repo"]["default_version"] = "" 252 253 # rename variable to accommodate existing owlbot.py files 254 if "system_test_dependencies" in kwargs: 255 kwargs["system_test_local_dependencies"] = kwargs[ 256 "system_test_dependencies" 257 ] 258 logger.warning( 259 "Template argument 'system_test_dependencies' is deprecated." 260 "Use 'system_test_local_dependencies' or 'system_test_external_dependencies'" 261 "instead." 262 ) 263 264 # Set default Python versions for noxfile.py 265 if "default_python_version" not in kwargs: 266 kwargs["default_python_version"] = "3.8" 267 if "unit_test_python_versions" not in kwargs: 268 kwargs["unit_test_python_versions"] = ["3.7", "3.8", "3.9", "3.10"] 269 270 if "system_test_python_versions" not in kwargs: 271 kwargs["system_test_python_versions"] = ["3.8"] 272 273 # If cov_level is not given, set it to None. 274 if "cov_level" not in kwargs: 275 kwargs["cov_level"] = None 276 277 # Don't add samples templates if there are no samples 278 if "samples" not in kwargs: 279 self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"] 280 281 # Don't add `docs/index.rst` if `versions` is not provided or `default_version` is empty 282 if ( 283 "versions" not in kwargs 284 or not kwargs["metadata"]["repo"]["default_version"] 285 ): 286 self.excludes += ["docs/index.rst"] 287 288 # Add kwargs to signal that UPGRADING.md should be included in docs/index.rst if it exists 289 if Path("docs/UPGRADING.md").exists() or Path("docs/UPGRADING.rst").exists(): 290 kwargs["include_uprading_doc"] = True 291 292 # If the directory `google/cloud` exists, add kwargs to signal that the client library is for a Cloud API 293 if Path("google/cloud").exists(): 294 kwargs["is_google_cloud_api"] = True 295 296 # If Dockerfile exists in .kokoro/docker/samples, add kwargs to 297 # signal that a custom docker image should be used when testing samples. 298 kwargs["custom_samples_dockerfile"] = Path( 299 ".kokoro/docker/samples/Dockerfile" 300 ).exists() 301 302 ret = self._generic_library("python_library", **kwargs) 303 304 # If split_system_tests is set to True, we disable the system 305 # test in the main presubmit build and create individual build 306 # configs for each python versions. 307 if kwargs.get("split_system_tests", False): 308 template_root = self._template_root / "py_library_split_systests" 309 # copy the main presubmit config 310 shutil.copy2( 311 template_root / ".kokoro/presubmit/presubmit.cfg", 312 ret / ".kokoro/presubmit/presubmit.cfg", 313 ) 314 env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_root))) 315 tmpl = env.get_template(".kokoro/presubmit/system.cfg") 316 for v in kwargs["system_test_python_versions"]: 317 nox_session = f"system-{v}" 318 dest = ret / f".kokoro/presubmit/system-{v}.cfg" 319 content = tmpl.render(nox_session=nox_session) 320 with open(dest, "w") as f: 321 f.write(content) 322 return ret 323 324 def java_library(self, **kwargs) -> Path: 325 # kwargs["metadata"] is required to load values from .repo-metadata.json 326 if "metadata" not in kwargs: 327 kwargs["metadata"] = {} 328 return self._generic_library("java_library", **kwargs) 329 330 def node_library(self, **kwargs) -> Path: 331 # TODO: once we've migrated all Node.js repos to either having 332 # .repo-metadata.json, or excluding README.md, we can remove this. 333 if not os.path.exists("./.repo-metadata.json"): 334 self.excludes.append("README.md") 335 if "samples/README.md" not in self.excludes: 336 self.excludes.append("samples/README.md") 337 338 kwargs["metadata"] = node.template_metadata() 339 kwargs["publish_token"] = node.get_publish_token(kwargs["metadata"]["name"]) 340 341 ignore_src_index = [ 342 "yes" for f in self.excludes if fnmatch.fnmatch("src/index.ts", f) 343 ] 344 # generate root-level `src/index.ts` to export multiple versions and its default clients 345 if ( 346 "versions" in kwargs 347 and "default_version" in kwargs 348 and not ignore_src_index 349 ): 350 node.generate_index_ts( 351 versions=kwargs["versions"], default_version=kwargs["default_version"] 352 ) 353 354 return self._generic_library("node_library", **kwargs) 355 356 def node_mono_repo_library(self, relative_dir, **kwargs) -> Path: 357 # TODO: once we've migrated all Node.js repos to either having 358 # .repo-metadata.json, or excluding README.md, we can remove this. 359 if not os.path.exists(Path(relative_dir, ".repo-metadata.json").resolve()): 360 self.excludes.append("README.md") 361 if "samples/README.md" not in self.excludes: 362 self.excludes.append("samples/README.md") 363 364 kwargs["metadata"] = node_mono_repo.template_metadata(relative_dir) 365 366 ignore_src_index = [ 367 "yes" for f in self.excludes if fnmatch.fnmatch("src/index.ts", f) 368 ] 369 # generate root-level `src/index.ts` to export multiple versions and its default clients 370 if ( 371 "versions" in kwargs 372 and "default_version" in kwargs 373 and not ignore_src_index 374 ): 375 node_mono_repo.generate_index_ts( 376 versions=kwargs["versions"], 377 default_version=kwargs["default_version"], 378 relative_dir=relative_dir, 379 year=str(date.today().year), 380 ) 381 382 return self._generic_library( 383 "node_mono_repo_library", relative_dir=relative_dir, **kwargs 384 ) 385 386 def php_library(self, **kwargs) -> Path: 387 return self._generic_library("php_library", **kwargs) 388 389 def ruby_library(self, **kwargs) -> Path: 390 # kwargs["metadata"] is required to load values from .repo-metadata.json 391 if "metadata" not in kwargs: 392 kwargs["metadata"] = {} 393 return self._generic_library("ruby_library", **kwargs) 394 395 def render(self, template_name: str, **kwargs) -> Path: 396 template = self._templates.render(template_name, **kwargs) 397 _tracked_paths.add(template) 398 return template 399 400 def _load_generic_metadata(self, metadata: Dict, relative_dir=None): 401 """ 402 loads additional meta information from .repo-metadata.json. 403 """ 404 metadata["partials"] = partials.load_partials() 405 406 # Loads repo metadata information from the default location if it 407 # hasn't already been set. Some callers may have already loaded repo 408 # metadata, so we don't need to do it again or overwrite it. Also, only 409 # set the "repo" key. 410 if "repo" not in metadata: 411 metadata["repo"] = _load_repo_metadata(relative_dir=relative_dir) 412 413 414def detect_versions( 415 path: str = "./src", 416 default_version: Optional[str] = None, 417 default_first: Optional[bool] = None, 418) -> List[str]: 419 """ 420 Detects the versions a library has, based on distinct folders 421 within path. This is based on the fact that our GAPIC libraries are 422 structured as follows: 423 424 src/v1 425 src/v1beta 426 src/v1alpha 427 428 With folder names mapping directly to versions. 429 430 Returns: a list of the sorted subdirectories; for the example above: 431 ['v1', 'v1alpha', 'v1beta'] 432 If the `default_version` argument is not provided, the `default_version` 433 will be read from `.repo-metadata.json`, if it exists. 434 If `default_version` is available, the `default_version` is moved to 435 at the front or the end of the sorted list depending on the value of `default_first`. 436 The `default_version` will be first in the list when `default_first` is `True`. 437 """ 438 439 versions = [] 440 441 if not default_version: 442 try: 443 # Get the `default_version` from ``.repo-metadata.json`. 444 default_version = json.load(open(".repo-metadata.json", "rt")).get( 445 "default_version" 446 ) 447 except FileNotFoundError: 448 pass 449 450 # Detect versions up to a depth of 4 in directory hierarchy 451 for level in ("*v[1-9]*", "*/*v[1-9]*", "*/*/*v[1-9]*", "*/*/*/*v[1-9]*"): 452 # Sort the sub directories alphabetically. 453 sub_dirs = sorted([p.name for p in Path(path).glob(level) if p.is_dir()]) 454 # Don't proceed to the next level if we've detected versions in this depth level 455 if sub_dirs: 456 break 457 458 if sub_dirs: 459 # if `default_version` is not specified, return the sorted directories. 460 if not default_version: 461 versions = sub_dirs 462 else: 463 # The subdirectory with the same suffix as the default_version 464 # will be the default client. 465 default_client = next( 466 iter([d for d in sub_dirs if d.endswith(default_version)]), None 467 ) 468 469 # start with all the versions except for the default client 470 versions = [d for d in sub_dirs if not d.endswith(default_version)] 471 472 if default_client: 473 # If `default_first` is true, the default_client will be first 474 # in the list. 475 if default_first: 476 versions = [default_client] + versions 477 else: 478 versions += [default_client] 479 return versions 480 481 482def decamelize(value: str): 483 """Parser to convert fooBar.js to Foo Bar.""" 484 if not value: 485 return "" 486 str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. 487 str_decamelize = re.sub( 488 "([A-Z]+)([A-Z])([a-z0-9])", r"\1 \2\3", str_decamelize 489 ) # ACLBatman -> ACL Batman. 490 return re.sub("([a-z0-9])([A-Z])", r"\1 \2", str_decamelize) # FooBar -> Foo Bar. 491 492 493def _load_repo_metadata( 494 relative_dir=None, metadata_file: str = "./.repo-metadata.json" 495) -> Dict: 496 """Parse a metadata JSON file into a Dict. 497 498 Currently, the defined fields are: 499 * `name` - The service's API name 500 * `name_pretty` - The service's API title. This will be used for generating titles on READMEs 501 * `product_documentation` - The product documentation on cloud.google.com 502 * `client_documentation` - The client library reference documentation 503 * `issue_tracker` - The public issue tracker for the product 504 * `release_level` - The release level of the client library. One of: alpha, beta, 505 ga, deprecated, preview, stable 506 * `language` - The repo language. One of dotnet, go, java, nodejs, php, python, ruby 507 * `repo` - The GitHub repo in the format {owner}/{repo} 508 * `distribution_name` - The language-idiomatic package/distribution name 509 * `api_id` - The API ID associated with the service. Fully qualified identifier use to 510 enable a service in the cloud platform (e.g. monitoring.googleapis.com) 511 * `requires_billing` - Whether or not the API requires billing to be configured on the 512 customer's acocunt 513 514 Args: 515 metadata_file (str, optional): Path to the metadata json file 516 517 Returns: 518 A dictionary of metadata. This may not necessarily include all the defined fields above. 519 """ 520 if relative_dir is not None: 521 if os.path.exists(Path(relative_dir, metadata_file).resolve()): 522 with open(Path(relative_dir, metadata_file).resolve()) as f: 523 return json.load(f) 524 elif os.path.exists(metadata_file): 525 with open(metadata_file) as f: 526 return json.load(f) 527 return {} 528 529 530def _get_default_branch_name(repository_name: str) -> str: 531 """Read the default branch name from the environment. 532 533 First checks environment variable DEFAULT_BRANCH_PATH. If found, it 534 reads the contents of the file at DEFAULT_BRANCH_PATH and returns it. 535 536 Then checks environment varabile DEFAULT_BRANCH, and returns it if found. 537 """ 538 default_branch_path = os.getenv("DEFAULT_BRANCH_PATH") 539 if default_branch_path: 540 return Path(default_branch_path).read_text().strip() 541 542 # This default should be switched to "main" once we've migrated 543 # the majority of our repositories: 544 return os.getenv("DEFAULT_BRANCH", "master") 545