1#!/usr/bin/env python3 2# Copyright 2017 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run tests using docker images in Google Container Registry per matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import atexit 21import json 22import multiprocessing 23import os 24import re 25import subprocess 26import sys 27import uuid 28 29# Language Runtime Matrix 30import client_matrix 31 32python_util_dir = os.path.abspath( 33 os.path.join(os.path.dirname(__file__), "../run_tests/python_utils") 34) 35sys.path.append(python_util_dir) 36import dockerjob 37import jobset 38import report_utils 39import upload_test_results 40 41_TEST_TIMEOUT_SECONDS = 60 42_PULL_IMAGE_TIMEOUT_SECONDS = 15 * 60 43_MAX_PARALLEL_DOWNLOADS = 6 44_LANGUAGES = list(client_matrix.LANG_RUNTIME_MATRIX.keys()) 45# All gRPC release tags, flattened, deduped and sorted. 46_RELEASES = sorted( 47 list( 48 set( 49 release 50 for release_dict in list(client_matrix.LANG_RELEASE_MATRIX.values()) 51 for release in list(release_dict.keys()) 52 ) 53 ) 54) 55 56argp = argparse.ArgumentParser(description="Run interop tests.") 57argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) 58argp.add_argument( 59 "--gcr_path", 60 default="gcr.io/grpc-testing", 61 help="Path of docker images in Google Container Registry", 62) 63argp.add_argument( 64 "--release", 65 default="all", 66 choices=["all"] + _RELEASES, 67 help=( 68 "Release tags to test. When testing all " 69 'releases defined in client_matrix.py, use "all".' 70 ), 71) 72argp.add_argument( 73 "-l", 74 "--language", 75 choices=["all"] + sorted(_LANGUAGES), 76 nargs="+", 77 default=["all"], 78 help="Languages to test", 79) 80argp.add_argument( 81 "--keep", 82 action="store_true", 83 help="keep the created local images after finishing the tests.", 84) 85argp.add_argument( 86 "--report_file", default="report.xml", help="The result file to create." 87) 88argp.add_argument( 89 "--allow_flakes", 90 default=False, 91 action="store_const", 92 const=True, 93 help=( 94 "Allow flaky tests to show as passing (re-runs failed " 95 "tests up to five times)" 96 ), 97) 98argp.add_argument( 99 "--bq_result_table", 100 default="", 101 type=str, 102 nargs="?", 103 help="Upload test results to a specified BQ table.", 104) 105# Requests will be routed through specified VIP by default. 106# See go/grpc-interop-tests (internal-only) for details. 107argp.add_argument( 108 "--server_host", 109 default="74.125.206.210", 110 type=str, 111 nargs="?", 112 help="The gateway to backend services.", 113) 114 115 116def _get_test_images_for_lang(lang, release_arg, image_path_prefix): 117 """Find docker images for a language across releases and runtimes. 118 119 Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime. 120 """ 121 if release_arg == "all": 122 # Use all defined releases for given language 123 releases = client_matrix.get_release_tags(lang) 124 else: 125 # Look for a particular release. 126 if release_arg not in client_matrix.get_release_tags(lang): 127 jobset.message( 128 "SKIPPED", 129 "release %s for %s is not defined" % (release_arg, lang), 130 do_newline=True, 131 ) 132 return {} 133 releases = [release_arg] 134 135 # Image tuples keyed by runtime. 136 images = {} 137 for tag in releases: 138 for runtime in client_matrix.get_runtimes_for_lang_release(lang, tag): 139 image_name = "%s/grpc_interop_%s:%s" % ( 140 image_path_prefix, 141 runtime, 142 tag, 143 ) 144 image_tuple = (tag, image_name) 145 146 if runtime not in images: 147 images[runtime] = [] 148 images[runtime].append(image_tuple) 149 return images 150 151 152def _read_test_cases_file(lang, runtime, release): 153 """Read test cases from a bash-like file and return a list of commands""" 154 # Check to see if we need to use a particular version of test cases. 155 release_info = client_matrix.LANG_RELEASE_MATRIX[lang].get(release) 156 if release_info: 157 testcases_file = release_info.testcases_file 158 if not testcases_file: 159 # TODO(jtattermusch): remove the double-underscore, it is pointless 160 testcases_file = "%s__master" % lang 161 162 # For csharp, the testcases file used depends on the runtime 163 # TODO(jtattermusch): remove this odd specialcase 164 if lang == "csharp" and runtime == "csharpcoreclr": 165 testcases_file = testcases_file.replace("csharp_", "csharpcoreclr_") 166 167 testcases_filepath = os.path.join( 168 os.path.dirname(__file__), "testcases", testcases_file 169 ) 170 lines = [] 171 with open(testcases_filepath) as f: 172 for line in f.readlines(): 173 line = re.sub("\\#.*$", "", line) # remove hash comments 174 line = line.strip() 175 if line and not line.startswith("echo"): 176 # Each non-empty line is a treated as a test case command 177 lines.append(line) 178 return lines 179 180 181def _cleanup_docker_image(image): 182 jobset.message("START", "Cleanup docker image %s" % image, do_newline=True) 183 dockerjob.remove_image(image, skip_nonexistent=True) 184 185 186args = argp.parse_args() 187 188 189# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime. 190def _generate_test_case_jobspecs(lang, runtime, release, suite_name): 191 """Returns the list of test cases from testcase files per lang/release.""" 192 testcase_lines = _read_test_cases_file(lang, runtime, release) 193 194 job_spec_list = [] 195 for line in testcase_lines: 196 print("Creating jobspec with cmdline '{}'".format(line)) 197 # TODO(jtattermusch): revisit the logic for updating test case commands 198 # what it currently being done seems fragile. 199 200 # Extract test case name from the command line 201 m = re.search(r"--test_case=(\w+)", line) 202 testcase_name = m.group(1) if m else "unknown_test" 203 204 # Extract the server name from the command line 205 if "--server_host_override=" in line: 206 m = re.search( 207 r"--server_host_override=((.*).sandbox.googleapis.com)", line 208 ) 209 else: 210 m = re.search(r"--server_host=((.*).sandbox.googleapis.com)", line) 211 server = m.group(1) if m else "unknown_server" 212 server_short = m.group(2) if m else "unknown_server" 213 214 # replace original server_host argument 215 assert "--server_host=" in line 216 line = re.sub( 217 r"--server_host=[^ ]*", r"--server_host=%s" % args.server_host, line 218 ) 219 220 # some interop tests don't set server_host_override (see #17407), 221 # but we need to use it if different host is set via cmdline args. 222 if args.server_host != server and not "--server_host_override=" in line: 223 line = re.sub( 224 r"(--server_host=[^ ]*)", 225 r"\1 --server_host_override=%s" % server, 226 line, 227 ) 228 229 spec = jobset.JobSpec( 230 cmdline=line, 231 shortname="%s:%s:%s:%s" 232 % (suite_name, lang, server_short, testcase_name), 233 timeout_seconds=_TEST_TIMEOUT_SECONDS, 234 shell=True, 235 flake_retries=5 if args.allow_flakes else 0, 236 ) 237 job_spec_list.append(spec) 238 return job_spec_list 239 240 241def _pull_image_for_lang(lang, image, release): 242 """Pull an image for a given language form the image registry.""" 243 cmdline = [ 244 "time gcloud docker -- pull %s && time docker run --rm=true %s" 245 " /bin/true" % (image, image) 246 ] 247 return jobset.JobSpec( 248 cmdline=cmdline, 249 shortname="pull_image_{}".format(image), 250 timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS, 251 shell=True, 252 flake_retries=2, 253 ) 254 255 256def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests): 257 total_num_failures = 0 258 suite_name = "%s__%s_%s" % (lang, runtime, release) 259 job_spec_list = _generate_test_case_jobspecs( 260 lang, runtime, release, suite_name 261 ) 262 263 if not job_spec_list: 264 jobset.message("FAILED", "No test cases were found.", do_newline=True) 265 total_num_failures += 1 266 else: 267 num_failures, resultset = jobset.run( 268 job_spec_list, 269 newline_on_success=True, 270 add_env={"docker_image": image}, 271 maxjobs=args.jobs, 272 skip_jobs=skip_tests, 273 ) 274 if args.bq_result_table and resultset: 275 upload_test_results.upload_interop_results_to_bq( 276 resultset, args.bq_result_table 277 ) 278 if skip_tests: 279 jobset.message("FAILED", "Tests were skipped", do_newline=True) 280 total_num_failures += 1 281 if num_failures: 282 total_num_failures += num_failures 283 284 report_utils.append_junit_xml_results( 285 xml_report_tree, 286 resultset, 287 "grpc_interop_matrix", 288 suite_name, 289 str(uuid.uuid4()), 290 ) 291 return total_num_failures 292 293 294def _run_tests_for_lang(lang, runtime, images, xml_report_tree): 295 """Find and run all test cases for a language. 296 297 images is a list of (<release-tag>, <image-full-path>) tuple. 298 """ 299 skip_tests = False 300 total_num_failures = 0 301 302 max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS) 303 max_chunk_size = max_pull_jobs 304 chunk_count = (len(images) + max_chunk_size) // max_chunk_size 305 306 for chunk_index in range(chunk_count): 307 chunk_start = chunk_index * max_chunk_size 308 chunk_size = min(max_chunk_size, len(images) - chunk_start) 309 chunk_end = chunk_start + chunk_size 310 pull_specs = [] 311 if not skip_tests: 312 for release, image in images[chunk_start:chunk_end]: 313 pull_specs.append(_pull_image_for_lang(lang, image, release)) 314 315 # NOTE(rbellevi): We batch docker pull operations to maximize 316 # parallelism, without letting the disk usage grow unbounded. 317 pull_failures, _ = jobset.run( 318 pull_specs, newline_on_success=True, maxjobs=max_pull_jobs 319 ) 320 if pull_failures: 321 jobset.message( 322 "FAILED", 323 'Image download failed. Skipping tests for language "%s"' 324 % lang, 325 do_newline=True, 326 ) 327 skip_tests = True 328 for release, image in images[chunk_start:chunk_end]: 329 total_num_failures += _test_release( 330 lang, runtime, release, image, xml_report_tree, skip_tests 331 ) 332 if not args.keep: 333 for _, image in images[chunk_start:chunk_end]: 334 _cleanup_docker_image(image) 335 if not total_num_failures: 336 jobset.message( 337 "SUCCESS", "All {} tests passed".format(lang), do_newline=True 338 ) 339 else: 340 jobset.message( 341 "FAILED", "Some {} tests failed".format(lang), do_newline=True 342 ) 343 344 return total_num_failures 345 346 347languages = args.language if args.language != ["all"] else _LANGUAGES 348total_num_failures = 0 349_xml_report_tree = report_utils.new_junit_xml_tree() 350for lang in languages: 351 docker_images = _get_test_images_for_lang(lang, args.release, args.gcr_path) 352 for runtime in sorted(docker_images.keys()): 353 total_num_failures += _run_tests_for_lang( 354 lang, runtime, docker_images[runtime], _xml_report_tree 355 ) 356 357report_utils.create_xml_report_file(_xml_report_tree, args.report_file) 358 359if total_num_failures: 360 sys.exit(1) 361sys.exit(0) 362