1#! /usr/bin/env python3 2# Copyright 2021 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Builds the content of xds-protos package""" 16 17import os 18import sys 19 20from grpc_tools import protoc 21 22if sys.version_info >= (3, 9, 0): 23 from importlib import resources 24else: 25 import pkg_resources 26 27 28def localize_path(p): 29 return os.path.join(*p.split("/")) 30 31 32def _get_resource_file_name( 33 package_or_requirement: str, resource_name: str 34) -> str: 35 """Obtain the filename for a resource on the file system.""" 36 file_name = None 37 if sys.version_info >= (3, 9, 0): 38 file_name = ( 39 resources.files(package_or_requirement) / resource_name 40 ).resolve() 41 else: 42 file_name = pkg_resources.resource_filename( 43 package_or_requirement, resource_name 44 ) 45 return str(file_name) 46 47 48# We might not want to compile all the protos 49EXCLUDE_PROTO_PACKAGES_LIST = tuple( 50 localize_path(p) 51 for p in ( 52 # Requires extra dependency to Prometheus protos 53 "envoy/service/metrics/v2", 54 "envoy/service/metrics/v3", 55 "envoy/service/metrics/v4alpha", 56 ) 57) 58 59# Compute the pathes 60WORK_DIR = os.path.dirname(os.path.abspath(__file__)) 61GRPC_ROOT = os.path.abspath(os.path.join(WORK_DIR, "..", "..", "..", "..")) 62ENVOY_API_PROTO_ROOT = os.path.join(GRPC_ROOT, "third_party", "envoy-api") 63XDS_PROTO_ROOT = os.path.join(GRPC_ROOT, "third_party", "xds") 64GOOGLEAPIS_ROOT = os.path.join(GRPC_ROOT, "third_party", "googleapis") 65VALIDATE_ROOT = os.path.join(GRPC_ROOT, "third_party", "protoc-gen-validate") 66OPENCENSUS_PROTO_ROOT = os.path.join( 67 GRPC_ROOT, "third_party", "opencensus-proto", "src" 68) 69OPENTELEMETRY_PROTO_ROOT = os.path.join(GRPC_ROOT, "third_party", "opentelemetry") 70WELL_KNOWN_PROTOS_INCLUDE = _get_resource_file_name("grpc_tools", "_proto") 71 72OUTPUT_PATH = WORK_DIR 73 74# Prepare the test file generation 75TEST_FILE_NAME = "generated_file_import_test.py" 76TEST_IMPORTS = [] 77 78# The pkgutil-style namespace packaging __init__.py 79PKGUTIL_STYLE_INIT = ( 80 "__path__ = __import__('pkgutil').extend_path(__path__, __name__)\n" 81) 82NAMESPACE_PACKAGES = ["google"] 83 84 85def add_test_import(proto_package_path: str, file_name: str, service: bool = False): 86 TEST_IMPORTS.append( 87 "from %s import %s\n" 88 % ( 89 proto_package_path.replace("/", ".").replace("-", "_"), 90 file_name.replace(".proto", "_pb2").replace("-", "_"), 91 ) 92 ) 93 if service: 94 TEST_IMPORTS.append( 95 "from %s import %s\n" 96 % ( 97 proto_package_path.replace("/", ".").replace("-", "_"), 98 file_name.replace(".proto", "_pb2_grpc").replace("-", "_"), 99 ) 100 ) 101 102 103# Prepare Protoc command 104COMPILE_PROTO_ONLY = [ 105 "grpc_tools.protoc", 106 "--proto_path={}".format(ENVOY_API_PROTO_ROOT), 107 "--proto_path={}".format(XDS_PROTO_ROOT), 108 "--proto_path={}".format(GOOGLEAPIS_ROOT), 109 "--proto_path={}".format(VALIDATE_ROOT), 110 "--proto_path={}".format(WELL_KNOWN_PROTOS_INCLUDE), 111 "--proto_path={}".format(OPENCENSUS_PROTO_ROOT), 112 "--proto_path={}".format(OPENTELEMETRY_PROTO_ROOT), 113 "--python_out={}".format(OUTPUT_PATH), 114] 115COMPILE_BOTH = COMPILE_PROTO_ONLY + ["--grpc_python_out={}".format(OUTPUT_PATH)] 116 117 118def has_grpc_service(proto_package_path: str) -> bool: 119 return proto_package_path.startswith(os.path.join("envoy", "service")) 120 121 122def compile_protos(proto_root: str, sub_dir: str = ".") -> None: 123 compiled_any = False 124 for root, _, files in os.walk(os.path.join(proto_root, sub_dir)): 125 proto_package_path = os.path.relpath(root, proto_root) 126 if proto_package_path in EXCLUDE_PROTO_PACKAGES_LIST: 127 print(f"Skipping package {proto_package_path}") 128 continue 129 for file_name in files: 130 if file_name.endswith(".proto"): 131 # Compile proto 132 compiled_any = True 133 if has_grpc_service(proto_package_path): 134 return_code = protoc.main( 135 COMPILE_BOTH + [os.path.join(root, file_name)] 136 ) 137 add_test_import(proto_package_path, file_name, service=True) 138 else: 139 return_code = protoc.main( 140 COMPILE_PROTO_ONLY + [os.path.join(root, file_name)] 141 ) 142 add_test_import(proto_package_path, file_name, service=False) 143 if return_code != 0: 144 raise Exception("error: {} failed".format(COMPILE_BOTH)) 145 # Ensure a deterministic order. 146 TEST_IMPORTS.sort() 147 if not compiled_any: 148 raise Exception( 149 "No proto files found at {}. Did you update git submodules?".format( 150 proto_root, sub_dir 151 ) 152 ) 153 154 155def create_init_file(path: str, package_path: str = "") -> None: 156 with open(os.path.join(path, "__init__.py"), "w") as f: 157 # Apply the pkgutil-style namespace packaging, which is compatible for 2 158 # and 3. Here is the full table of namespace compatibility: 159 # https://github.com/pypa/sample-namespace-packages/blob/master/table.md 160 if package_path in NAMESPACE_PACKAGES: 161 f.write(PKGUTIL_STYLE_INIT) 162 163 164def main(): 165 # Compile xDS protos 166 compile_protos(ENVOY_API_PROTO_ROOT) 167 compile_protos(XDS_PROTO_ROOT) 168 # We don't want to compile the entire GCP surface API, just the essential ones 169 compile_protos(GOOGLEAPIS_ROOT, os.path.join("google", "api")) 170 compile_protos(GOOGLEAPIS_ROOT, os.path.join("google", "rpc")) 171 compile_protos(GOOGLEAPIS_ROOT, os.path.join("google", "longrunning")) 172 compile_protos(GOOGLEAPIS_ROOT, os.path.join("google", "logging")) 173 compile_protos(GOOGLEAPIS_ROOT, os.path.join("google", "type")) 174 compile_protos(VALIDATE_ROOT, "validate") 175 compile_protos(OPENCENSUS_PROTO_ROOT) 176 compile_protos(OPENTELEMETRY_PROTO_ROOT) 177 178 # Generate __init__.py files for all modules 179 create_init_file(WORK_DIR) 180 for proto_root_module in [ 181 "envoy", 182 "google", 183 "opencensus", 184 "udpa", 185 "validate", 186 "xds", 187 "opentelemetry", 188 ]: 189 for root, _, _ in os.walk(os.path.join(WORK_DIR, proto_root_module)): 190 package_path = os.path.relpath(root, WORK_DIR) 191 create_init_file(root, package_path) 192 193 # Generate test file 194 with open(os.path.join(WORK_DIR, TEST_FILE_NAME), "w") as f: 195 f.writelines(TEST_IMPORTS) 196 197 198if __name__ == "__main__": 199 main() 200