1# Copyright 2019 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15load("@com_google_api_gax_java_properties//:dependencies.properties.bzl", "PROPERTIES") 16 17def _wrapPropertyNamesInBraces(properties): 18 wrappedProperties = {} 19 for k, v in properties.items(): 20 wrappedProperties["{{%s}}" % k] = v 21 return wrappedProperties 22 23_PROPERTIES = _wrapPropertyNamesInBraces(PROPERTIES) 24 25# ======================================================================== 26# General packaging helpers. 27# ======================================================================== 28 29def _construct_package_dir_paths(attr_package_dir, out_pkg, label_name): 30 if attr_package_dir: 31 package_dir = attr_package_dir 32 package_dir_expr = "../{}/".format(package_dir) 33 tar_cd_suffix = ".." 34 tar_prefix = attr_package_dir 35 else: 36 package_dir = label_name 37 package_dir_expr = "./" 38 tar_cd_suffix = "." 39 tar_prefix = "." 40 41 # We need to include label in the path to eliminate possible output files duplicates 42 # (labels are guaranteed to be unique by bazel itself) 43 package_dir_path = "%s/%s/%s" % (out_pkg.dirname, label_name, package_dir) 44 return struct( 45 package_dir = package_dir, 46 package_dir_expr = package_dir_expr, 47 package_dir_path = package_dir_path, 48 package_dir_sibling_parent = out_pkg, 49 package_dir_sibling_basename = label_name, 50 tar_cd_suffix = tar_cd_suffix, 51 tar_prefix = tar_prefix, 52 ) 53 54def _put_dep_in_a_bucket(dep, dep_bucket, processed_deps): 55 if processed_deps.get(dep): 56 return 57 dep_bucket.append(dep) 58 processed_deps[dep] = True 59 60def _gapic_pkg_tar_impl(ctx): 61 deps = [] 62 for dep in ctx.attr.deps: 63 for f in dep.files.to_list(): 64 deps.append(f) 65 66 samples =[] 67 for s in ctx.attr.samples: 68 for f in s.files.to_list(): 69 samples.append(f) 70 71 paths = _construct_package_dir_paths( 72 ctx.attr.package_dir, 73 ctx.outputs.pkg, 74 ctx.label.name, 75 ) 76 77 script = """ 78 for s in {samples}; do 79 mkdir -p {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/ 80 unzip -q ./$s -d {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/ 81 done 82 83 mkdir -p {package_dir_path} 84 for dep in {deps}; do 85 tar -xzpf $dep -C {package_dir_path} 86 done 87 cd {package_dir_path}/{tar_cd_suffix} 88 89 tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/* 90 cd - > /dev/null 91 mv {package_dir_path}/{package_dir}.tar.gz {pkg} 92 rm -rf {package_dir_path} 93 """.format( 94 deps = " ".join(["'%s'" % d.path for d in deps]), 95 package_dir_path = paths.package_dir_path, 96 package_dir = paths.package_dir, 97 samples = " ".join(["'%s'" % s.path for s in samples]), 98 pkg = ctx.outputs.pkg.path, 99 tar_cd_suffix = paths.tar_cd_suffix, 100 tar_prefix = paths.tar_prefix, 101 ) 102 103 ctx.actions.run_shell( 104 inputs = deps + samples, 105 command = script, 106 outputs = [ctx.outputs.pkg], 107 ) 108 109# The Bazel's native gapic_pkg_tar rule behaves weirdly when package_dir parameter 110# is specified (at least on some Linux machines it does not put all the files 111# under the package_dir). As a workaround for that bug we provide the custom 112# implementation of the gapic_pkg_tar rule. 113gapic_pkg_tar = rule( 114 attrs = { 115 "deps": attr.label_list(mandatory = True), 116 "samples": attr.label_list(mandatory = False), 117 "package_dir": attr.string(mandatory = False, default = ""), 118 "extension": attr.string(mandatory = False, default = "tar.gz"), 119 }, 120 outputs = {"pkg": "%{name}.%{extension}"}, 121 implementation = _gapic_pkg_tar_impl, 122) 123 124# ======================================================================== 125# Java Gapic package helpers. 126# ======================================================================== 127def _construct_extra_deps(scope_to_deps, versions_map): 128 label_name_to_maven_artifact = { 129 "policy_proto": "maven.com_google_api_grpc_proto_google_iam_v1", 130 "iam_policy_proto": "maven.com_google_api_grpc_proto_google_iam_v1", 131 "iam_java_proto": "maven.com_google_api_grpc_proto_google_iam_v1", 132 "iam_java_grpc": "maven.com_google_api_grpc_grpc_google_iam_v1", 133 "iam_policy_java_grpc": "maven.com_google_api_grpc_grpc_google_iam_v1", 134 "location_java_grpc": "maven.com_google_api_grpc_grpc_google_common_protos", 135 } 136 extra_deps = {} 137 for scope, deps in scope_to_deps.items(): 138 for dep in deps: 139 pkg_dependency = _get_gapic_pkg_dependency_name(dep) 140 if pkg_dependency: 141 key = "{{%s}}" % pkg_dependency 142 if not extra_deps.get(key): 143 extra_deps[key] = "%s project(':%s')" % (scope, pkg_dependency) 144 elif _is_java_dependency(dep): 145 for f in dep[JavaInfo].transitive_deps.to_list(): 146 maven_artifact = label_name_to_maven_artifact.get(f.owner.name) 147 if not maven_artifact: 148 continue 149 key = "{{%s}}" % maven_artifact 150 if not extra_deps.get(key): 151 extra_deps[key] = "%s '%s'" % (scope, versions_map[key]) 152 153 return "\n ".join(extra_deps.values()) 154 155def _is_java_dependency(dep): 156 return JavaInfo in dep 157 158def _is_source_dependency(dep): 159 return _is_java_dependency(dep) and hasattr(dep[JavaInfo], "source_jars") and dep.label.package != "jar" 160 161def _is_proto_dependency(dep): 162 return ProtoInfo in dep 163 164def _get_gapic_pkg_dependency_name(dep): 165 files_list = dep.files.to_list() 166 if not files_list or len(files_list) != 1: 167 return None 168 for ext in (".tar.gz", ".gz", ".tgz"): 169 if files_list[0].basename.endswith(ext): 170 return files_list[0].basename[:-len(ext)] 171 return None 172 173# ======================================================================== 174# Java Gapic package rules. 175# ======================================================================== 176 177def _java_gapic_build_configs_pkg_impl(ctx): 178 expanded_templates = [] 179 paths = _construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name) 180 181 substitutions = dict(ctx.attr.static_substitutions) 182 substitutions["{{extra_deps}}"] = _construct_extra_deps({ 183 "api": ctx.attr.deps, 184 "testImplementation": ctx.attr.test_deps, 185 }, substitutions) 186 187 for template in ctx.attr.templates.items(): 188 expanded_template = ctx.actions.declare_file( 189 "%s/%s" % (paths.package_dir_sibling_basename, template[1]), 190 sibling = paths.package_dir_sibling_parent, 191 ) 192 expanded_templates.append(expanded_template) 193 ctx.actions.expand_template( 194 template = template[0].files.to_list()[0], 195 substitutions = substitutions, 196 output = expanded_template, 197 ) 198 199 # Note the script is more complicated than it intuitively should be because of the limitations 200 # inherent to bazel execution environment: no absolute paths allowed, the generated artifacts 201 # must ensure uniqueness within a build. The template output directory manipulations are 202 # to modify default 555 file permissions on any generated by bazel file (exectuable read-only, 203 # which is not at all what we need for build files). There is no bazel built-in way to change 204 # the generated files permissions, also the actual files accessible by the script are symlinks 205 # and `chmod`, when applied to a directory, does not change the attributes of symlink targets 206 # inside the directory. Chaning the symlink target's permissions is also not an option, because 207 # they are on a read-only file system. 208 script = """ 209 mkdir -p {package_dir_path} 210 for templ in {templates}; do 211 cp $templ {package_dir_path}/ 212 done 213 chmod 644 {package_dir_path}/* 214 cd {package_dir_path}/{tar_cd_suffix} 215 tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/* 216 cd - > /dev/null 217 mv {package_dir_path}/{package_dir}.tar.gz {pkg} 218 """.format( 219 templates = " ".join(["'%s'" % f.path for f in expanded_templates]), 220 package_dir_path = paths.package_dir_path, 221 package_dir = paths.package_dir, 222 pkg = ctx.outputs.pkg.path, 223 tar_cd_suffix = paths.tar_cd_suffix, 224 tar_prefix = paths.tar_prefix, 225 ) 226 227 ctx.actions.run_shell( 228 inputs = expanded_templates, 229 command = script, 230 outputs = [ctx.outputs.pkg], 231 ) 232 233java_gapic_build_configs_pkg = rule( 234 attrs = { 235 "deps": attr.label_list(mandatory = True), 236 "test_deps": attr.label_list(mandatory = False, allow_empty = True), 237 "package_dir": attr.string(mandatory = False), 238 "templates": attr.label_keyed_string_dict(mandatory = False, allow_files = True), 239 "static_substitutions": attr.string_dict(mandatory = False, allow_empty = True, default = {}), 240 }, 241 outputs = {"pkg": "%{name}.tar.gz"}, 242 implementation = _java_gapic_build_configs_pkg_impl, 243) 244 245def _java_gapic_srcs_pkg_impl(ctx): 246 srcs = [] 247 proto_srcs = [] 248 for src_dep in ctx.attr.deps: 249 if _is_source_dependency(src_dep): 250 srcs.extend(src_dep[JavaInfo].source_jars) 251 if _is_proto_dependency(src_dep): 252 proto_srcs.extend(src_dep[ProtoInfo].check_deps_sources.to_list()) 253 254 test_srcs = [] 255 for test_src_dep in ctx.attr.test_deps: 256 if _is_source_dependency(test_src_dep): 257 test_srcs.extend(test_src_dep[JavaInfo].source_jars) 258 259 paths = _construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name) 260 261 # Note the script is more complicated than it intuitively should be because of limitations 262 # inherent to bazel execution environment: no absolute paths allowed, the generated artifacts 263 # must ensure uniqueness within a build. 264 script = """ 265 for src in {srcs}; do 266 mkdir -p {package_dir_path}/src/main/java 267 unzip -q -o $src -d {package_dir_path}/src/main/java 268 rm -r -f {package_dir_path}/src/main/java/META-INF 269 270 # Remove empty files. If there are no resource names, one such file might have 271 # been created. See java_gapic.bzl. 272 find {package_dir_path}/src/main/java -type f -size 0 | while read f; do rm -f $f; done 273 274 if [ -d {package_dir_path}/src/main/java/samples ]; then 275 mv {package_dir_path}/src/main/java/samples {package_dir_path} 276 fi 277 done 278 for proto_src in {proto_srcs}; do 279 mkdir -p {package_dir_path}/src/main/proto 280 cp -f --parents $proto_src {package_dir_path}/src/main/proto 281 done 282 for test_src in {test_srcs}; do 283 mkdir -p {package_dir_path}/src/test/java 284 unzip -q -o $test_src -d {package_dir_path}/src/test/java 285 rm -r -f {package_dir_path}/src/test/java/META-INF 286 done 287 cd {package_dir_path}/{tar_cd_suffix} 288 tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/* 289 cd - > /dev/null 290 mv {package_dir_path}/{package_dir}.tar.gz {pkg} 291 """.format( 292 srcs = " ".join(["'%s'" % f.path for f in srcs]), 293 proto_srcs = " ".join(["'%s'" % f.path for f in proto_srcs]), 294 test_srcs = " ".join(["'%s'" % f.path for f in test_srcs]), 295 package_dir_path = paths.package_dir_path, 296 package_dir = paths.package_dir, 297 pkg = ctx.outputs.pkg.path, 298 tar_cd_suffix = paths.tar_cd_suffix, 299 tar_prefix = paths.tar_prefix, 300 ) 301 302 ctx.actions.run_shell( 303 inputs = srcs + proto_srcs + test_srcs, 304 command = script, 305 outputs = [ctx.outputs.pkg], 306 ) 307 308java_gapic_srcs_pkg = rule( 309 attrs = { 310 "deps": attr.label_list(mandatory = True), 311 "test_deps": attr.label_list(mandatory = False, allow_empty = True), 312 "package_dir": attr.string(mandatory = True), 313 }, 314 outputs = {"pkg": "%{name}.tar.gz"}, 315 implementation = _java_gapic_srcs_pkg_impl, 316) 317 318def java_gapic_assembly_gradle_pkg( 319 name, 320 deps, 321 include_samples = False, 322 assembly_name = None, 323 transport = None, 324 **kwargs): 325 package_dir = name 326 if assembly_name: 327 package_dir = "google-cloud-%s-%s" % (assembly_name, name) 328 329 # Rename to avoid target conflicts with the monolith. 330 proto_target = "proto-%s" % package_dir 331 proto_target_dep = [] 332 grpc_target = "grpc-%s" % package_dir 333 grpc_target_dep = [] 334 client_target = "gapic-%s" % package_dir 335 client_target_dep = [] 336 337 client_deps = [] 338 client_test_deps = [] 339 grpc_deps = [] 340 proto_deps = [] 341 samples = [] 342 343 processed_deps = {} #there is no proper Set in Starlark 344 for dep in deps: 345 # Use contains instead of endswith since microgenerator testing may use differently-named targets. 346 if "_java_gapic" in dep: 347 if include_samples: 348 samples.append(dep + "_samples") 349 _put_dep_in_a_bucket(dep, client_deps, processed_deps) 350 _put_dep_in_a_bucket("%s_test" % dep, client_test_deps, processed_deps) 351 _put_dep_in_a_bucket("%s_resource_name" % dep, proto_deps, processed_deps) 352 elif dep.endswith("_java_grpc"): 353 _put_dep_in_a_bucket(dep, grpc_deps, processed_deps) 354 else: 355 _put_dep_in_a_bucket(dep, proto_deps, processed_deps) 356 357 if proto_deps: 358 _java_gapic_gradle_pkg( 359 name = proto_target, 360 template_label = Label("//rules_java_gapic:resources/gradle/proto.gradle.tmpl"), 361 deps = proto_deps, 362 **kwargs 363 ) 364 proto_target_dep = [":%s" % proto_target] 365 366 if grpc_deps: 367 _java_gapic_gradle_pkg( 368 name = grpc_target, 369 template_label = Label("//rules_java_gapic:resources/gradle/grpc.gradle.tmpl"), 370 deps = proto_target_dep + grpc_deps, 371 **kwargs 372 ) 373 grpc_target_dep = ["%s" % grpc_target] 374 375 if client_deps: 376 if not transport or transport == "grpc": 377 template_label = Label("//rules_java_gapic:resources/gradle/client_grpc.gradle.tmpl") 378 elif transport == "rest": 379 template_label = Label("//rules_java_gapic:resources/gradle/client_rest.gradle.tmpl") 380 elif transport == "grpc+rest": 381 template_label = Label("//rules_java_gapic:resources/gradle/client_grpcrest.gradle.tmpl") 382 383 _java_gapic_gradle_pkg( 384 name = client_target, 385 template_label = template_label, 386 deps = proto_target_dep + client_deps, 387 test_deps = grpc_target_dep + client_test_deps, 388 **kwargs 389 ) 390 client_target_dep = ["%s" % client_target] 391 392 _java_gapic_assembly_gradle_pkg( 393 name = name, 394 assembly_name = package_dir, 395 deps = proto_target_dep + grpc_target_dep + client_target_dep, 396 samples = samples, 397 ) 398 399def _java_gapic_gradle_pkg( 400 name, 401 template_label, 402 deps, 403 test_deps = None, 404 project_deps = None, 405 test_project_deps = None, 406 **kwargs): 407 resource_target_name = "%s-resources" % name 408 409 static_substitutions = dict(_PROPERTIES) 410 static_substitutions["{{name}}"] = name 411 412 java_gapic_build_configs_pkg( 413 name = resource_target_name, 414 deps = deps, 415 test_deps = test_deps, 416 package_dir = name, 417 templates = { 418 template_label: "build.gradle", 419 }, 420 static_substitutions = static_substitutions, 421 ) 422 423 srcs_gapic_pkg_target_name = "%s-srcs_pkg" % name 424 java_gapic_srcs_pkg( 425 name = srcs_gapic_pkg_target_name, 426 deps = deps, 427 test_deps = test_deps, 428 package_dir = name, 429 **kwargs 430 ) 431 432 gapic_pkg_tar( 433 name = name, 434 extension = "tar.gz", 435 deps = [ 436 resource_target_name, 437 srcs_gapic_pkg_target_name, 438 ], 439 **kwargs 440 ) 441 442def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, samples = None, visibility = None): 443 resource_target_name = "%s-resources" % assembly_name 444 java_gapic_build_configs_pkg( 445 name = resource_target_name, 446 deps = deps, 447 templates = { 448 Label("//rules_java_gapic:resources/gradle/assembly.gradle.tmpl"): "build.gradle", 449 Label("//rules_java_gapic:resources/gradle/settings.gradle.tmpl"): "settings.gradle", 450 }, 451 ) 452 453 gapic_pkg_tar( 454 name = name, 455 extension = "tar.gz", 456 deps = [ 457 Label("//rules_java_gapic:gradlew"), 458 resource_target_name, 459 ] + deps, 460 samples = samples, 461 package_dir = assembly_name, 462 visibility = visibility, 463 ) 464