1# Copyright 2019 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14from pathlib import Path 15from typing import Optional, Union 16import os 17import shutil 18import tempfile 19 20from synthtool import _tracked_paths, metadata, shell 21from synthtool.log import logger 22from synthtool.sources import git 23 24GOOGLEAPIS_URL: str = git.make_repo_clone_url("googleapis/googleapis") 25GOOGLEAPIS_PRIVATE_URL: str = git.make_repo_clone_url("googleapis/googleapis-private") 26DISCOVERY_ARTIFACT_MANAGER_URL: str = git.make_repo_clone_url( 27 "googleapis/discovery-artifact-manager" 28) 29LOCAL_GOOGLEAPIS: Optional[str] = os.environ.get("SYNTHTOOL_GOOGLEAPIS") 30LOCAL_GOOGLEAPIS_DISCOVERY: Optional[str] = os.environ.get( 31 "SYNTHTOOL_GOOGLEAPIS_DISCOVERY" 32) 33LOCAL_DISCOVERY_ARTIFACT_MANAGER: Optional[str] = os.environ.get( 34 "SYNTHTOOL_DISCOVERY_ARTIFACT_MANAGER" 35) 36 37 38class GAPICBazel: 39 """A synthtool component that can produce libraries using bazel build.""" 40 41 def __init__(self): 42 self._ensure_dependencies_installed() 43 self._googleapis = None 44 self._googleapis_private = None 45 self._googleapis_discovery = None 46 self._discovery_artifact_manager = None 47 48 def py_library(self, service: str, version: str, **kwargs) -> Path: 49 return self._generate_code(service, version, "python", False, **kwargs) 50 51 def go_library(self, service: str, version: str, **kwargs) -> Path: 52 return self._generate_code(service, version, "go", False, **kwargs) 53 54 def node_library(self, service: str, version: str, **kwargs) -> Path: 55 return self._generate_code(service, version, "nodejs", False, **kwargs) 56 57 def csharp_library(self, service: str, version: str, **kwargs) -> Path: 58 return self._generate_code(service, version, "csharp", False, **kwargs) 59 60 def php_library( 61 self, service: str, version: str, clean_build: bool = False, **kwargs 62 ) -> Path: 63 return self._generate_code(service, version, "php", clean_build, **kwargs) 64 65 def java_library(self, service: str, version: str, **kwargs) -> Path: 66 return self._generate_code( 67 service, version, "java", False, tar_strip_components=0, **kwargs 68 ) 69 70 def ruby_library(self, service: str, version: str, **kwargs) -> Path: 71 return self._generate_code(service, version, "ruby", False, **kwargs) 72 73 def _generate_code( 74 self, 75 service: str, 76 version: str, 77 language: str, 78 clean_build: bool = False, 79 *, 80 private: bool = False, 81 discogapic: bool = False, 82 diregapic: bool = False, 83 proto_path: Union[str, Path] = None, 84 output_dir: Union[str, Path] = None, 85 bazel_target: str = None, 86 include_protos: bool = False, 87 proto_output_path: Union[str, Path] = None, 88 tar_strip_components: int = 1, 89 ): 90 # Determine which googleapis repo to use 91 if discogapic: 92 api_definitions_repo = self._clone_discovery_artifact_manager() 93 api_definitions_repo_name = "discovery-artifact-manager" 94 elif private: 95 api_definitions_repo = self._clone_googleapis_private() 96 api_definitions_repo_name = "googleapis_private" 97 else: 98 api_definitions_repo = self._clone_googleapis() 99 api_definitions_repo_name = "googleapis" 100 101 # Confidence check: We should have a googleapis repo; if we do not, 102 # something went wrong, and we should abort. 103 if not api_definitions_repo: 104 raise RuntimeError( 105 f"Unable to generate {service}, the sources repository repository" 106 "is unavailable." 107 ) 108 109 # Calculate proto_path if necessary. 110 if not bazel_target or include_protos: 111 # If bazel_target is not specified explicitly, we will need 112 # proto_path to calculate it. If include_protos is True, 113 # we will need the proto_path to copy the protos. 114 if not proto_path: 115 if bazel_target: 116 # Calculate proto_path from the full bazel target, which is 117 # in the format "//proto_path:target_name 118 proto_path = bazel_target.split(":")[0][2:] 119 else: 120 # If bazel_target is not specified, assume the protos are 121 # simply under google/cloud, where the most of the protos 122 # usually are. 123 proto_path = f"google/cloud/{service}/{version}" 124 protos = Path(proto_path) 125 if protos.is_absolute(): 126 protos = protos.relative_to("/") 127 128 # Determine bazel target based on per-language patterns 129 # Java: google-cloud-{{assembly_name}}-{{version}}-java 130 # Go: gapi-cloud-{{assembly_name}}-{{version}}-go 131 # Python: {{assembly_name}}-{{version}}-py 132 # PHP: google-cloud-{{assembly_name}}-{{version}}-php 133 # Node.js: {{assembly_name}}-{{version}}-nodejs 134 # Ruby: google-cloud-{{assembly_name}}-{{version}}-ruby 135 # C#: google-cloud-{{assembly_name}}-{{version}}-csharp 136 if not bazel_target: 137 # Determine where the protos we are generating actually live. 138 # We can sometimes (but not always) determine this from the service 139 # and version; in other cases, the user must provide it outright. 140 parts = list(protos.parts) 141 while len(parts) > 0 and parts[0] != "google": 142 parts.pop(0) 143 if len(parts) == 0: 144 raise RuntimeError( 145 f"Cannot determine bazel_target from proto_path {protos}." 146 "Please set bazel_target explicitly." 147 ) 148 if language == "python": 149 suffix = f"{service}-{version}-py" 150 elif language == "nodejs": 151 suffix = f"{service}-{version}-nodejs" 152 elif language == "go": 153 suffix = f"gapi-{'-'.join(parts[1:])}-go" 154 else: 155 suffix = f"{'-'.join(parts)}-{language}" 156 bazel_target = f"//{os.path.sep.join(parts)}:{suffix}" 157 158 # Confidence check: Do we have protos where we think we should? 159 if not (api_definitions_repo / protos).exists(): 160 raise FileNotFoundError( 161 f"Unable to find directory for protos: {(api_definitions_repo / protos)}." 162 ) 163 if not tuple((api_definitions_repo / protos).glob("*.proto")): 164 raise FileNotFoundError( 165 f"Directory {(api_definitions_repo / protos)} exists, but no protos found." 166 ) 167 if not (api_definitions_repo / protos / "BUILD.bazel"): 168 raise FileNotFoundError( 169 f"File {(api_definitions_repo / protos / 'BUILD.bazel')} does not exist." 170 ) 171 172 # Ensure the desired output directory exists. 173 # If none was provided, create a temporary directory. 174 if not output_dir: 175 output_dir = tempfile.mkdtemp() 176 output_dir = Path(output_dir).resolve() 177 178 # Let's build some stuff now. 179 cwd = os.getcwd() 180 os.chdir(str(api_definitions_repo)) 181 182 if clean_build: 183 logger.debug("Cleaning Bazel cache") 184 shell.run(["bazel", "clean", "--expunge", "--async"]) 185 186 # Log which version of bazel that we're using for easier debugging. 187 logger.debug("Which version of bazel will I run?") 188 shell.run(["bazel", "--version"], hide_output=False) 189 190 bazel_run_args = [ 191 "bazel", 192 "--max_idle_secs=240", 193 "build", 194 bazel_target, 195 ] 196 197 logger.debug(f"Generating code for: {bazel_target}.") 198 shell.run(bazel_run_args, hide_output=False) 199 200 # We've got tar file! 201 # its location: bazel-bin/google/cloud/language/v1/language-v1-nodejs.tar.gz 202 # bazel_target: //google/cloud/language/v1:language-v1-nodejs 203 tar_file = ( 204 f"bazel-bin{os.path.sep}{bazel_target[2:].replace(':', os.path.sep)}.tar.gz" 205 ) 206 207 tar_run_args = [ 208 "tar", 209 "-C", 210 str(output_dir), 211 f"--strip-components={tar_strip_components}", 212 "-xzf", 213 tar_file, 214 ] 215 shell.run(tar_run_args) 216 217 # Get the *.protos files and put them in a protos dir in the output 218 if include_protos: 219 proto_files = protos.glob("**/*.proto") 220 # By default, put the protos at the root in a folder named 'protos'. 221 # Specific languages can be cased here to put them in a more language 222 # appropriate place. 223 if not proto_output_path: 224 proto_output_path = output_dir / "protos" 225 if language == "python": 226 # place protos alongsize the *_pb2.py files 227 proto_output_path = ( 228 output_dir / f"google/cloud/{service}_{version}/proto" 229 ) 230 else: 231 proto_output_path = Path(output_dir / proto_output_path) 232 os.makedirs(proto_output_path, exist_ok=True) 233 234 for i in proto_files: 235 logger.debug(f"Copy: {i} to {proto_output_path / i.name}") 236 shutil.copyfile(i, proto_output_path / i.name) 237 logger.success(f"Placed proto files into {proto_output_path}.") 238 239 os.chdir(cwd) 240 241 # Confidence check: Does the output location have code in it? 242 # If not, complain. 243 if not tuple(output_dir.iterdir()): 244 raise RuntimeError( 245 f"Code generation seemed to succeed, but {output_dir} is empty." 246 ) 247 248 # Huzzah, it worked. 249 logger.success(f"Generated code into {output_dir}.") 250 251 # Record this in the synthtool metadata. 252 metadata.add_client_destination( 253 source=api_definitions_repo_name, 254 api_name=service, 255 api_version=version, 256 language=language, 257 generator="bazel", 258 ) 259 260 _tracked_paths.add(output_dir) 261 return output_dir 262 263 def _clone_googleapis(self): 264 if self._googleapis: 265 return self._googleapis 266 267 if LOCAL_GOOGLEAPIS: 268 self._googleapis = Path(LOCAL_GOOGLEAPIS).expanduser() 269 logger.debug(f"Using local googleapis at {self._googleapis}") 270 271 else: 272 logger.debug("Cloning googleapis.") 273 self._googleapis = git.clone(GOOGLEAPIS_URL) 274 275 return self._googleapis 276 277 def _clone_googleapis_private(self): 278 if self._googleapis_private: 279 return self._googleapis_private 280 281 if LOCAL_GOOGLEAPIS: 282 self._googleapis_private = Path(LOCAL_GOOGLEAPIS).expanduser() 283 logger.debug( 284 f"Using local googleapis at {self._googleapis_private} for googleapis-private" 285 ) 286 287 else: 288 logger.debug("Cloning googleapis-private.") 289 self._googleapis_private = git.clone(GOOGLEAPIS_PRIVATE_URL) 290 291 return self._googleapis_private 292 293 def _clone_discovery_artifact_manager(self): 294 if self._discovery_artifact_manager: 295 return self._discovery_artifact_manager 296 297 if LOCAL_DISCOVERY_ARTIFACT_MANAGER: 298 self._discovery_artifact_manager = Path( 299 LOCAL_DISCOVERY_ARTIFACT_MANAGER 300 ).expanduser() 301 logger.debug( 302 f"Using local discovery_artifact_manager at {self._discovery_artifact_manager} for googleapis-private" 303 ) 304 else: 305 logger.debug("Cloning discovery-artifact-manager.") 306 self._discovery_artifact_manager = git.clone(DISCOVERY_ARTIFACT_MANAGER_URL) 307 308 return self._discovery_artifact_manager 309 310 def _ensure_dependencies_installed(self): 311 logger.debug("Ensuring dependencies.") 312 313 dependencies = ["bazel", "zip", "unzip", "tar"] 314 failed_dependencies = [] 315 for dependency in dependencies: 316 return_code = shell.run(["which", dependency], check=False).returncode 317 if return_code: 318 failed_dependencies.append(dependency) 319 320 if failed_dependencies: 321 raise EnvironmentError( 322 f"Dependencies missing: {', '.join(failed_dependencies)}" 323 ) 324