1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run tests in parallel.""" 16 17from __future__ import print_function 18 19import argparse 20import ast 21import collections 22import glob 23import itertools 24import json 25import logging 26import multiprocessing 27import os 28import os.path 29import platform 30import random 31import re 32import shlex 33import socket 34import subprocess 35import sys 36import tempfile 37import time 38import traceback 39import uuid 40 41import six 42from six.moves import urllib 43 44import python_utils.jobset as jobset 45import python_utils.report_utils as report_utils 46import python_utils.start_port_server as start_port_server 47import python_utils.watch_dirs as watch_dirs 48 49try: 50 from python_utils.upload_test_results import upload_results_to_bq 51except ImportError: 52 pass # It's ok to not import because this is only necessary to upload results to BQ. 53 54gcp_utils_dir = os.path.abspath( 55 os.path.join(os.path.dirname(__file__), "../gcp/utils") 56) 57sys.path.append(gcp_utils_dir) 58 59_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 60os.chdir(_ROOT) 61 62_FORCE_ENVIRON_FOR_WRAPPERS = { 63 "GRPC_VERBOSITY": "DEBUG", 64} 65 66_POLLING_STRATEGIES = { 67 "linux": ["epoll1", "poll"], 68 "mac": ["poll"], 69} 70 71 72def platform_string(): 73 return jobset.platform_string() 74 75 76_DEFAULT_TIMEOUT_SECONDS = 5 * 60 77_PRE_BUILD_STEP_TIMEOUT_SECONDS = 10 * 60 78 79 80def run_shell_command(cmd, env=None, cwd=None): 81 try: 82 subprocess.check_output(cmd, shell=True, env=env, cwd=cwd) 83 except subprocess.CalledProcessError as e: 84 logging.exception( 85 "Error while running command '%s'. Exit status %d. Output:\n%s", 86 e.cmd, 87 e.returncode, 88 e.output, 89 ) 90 raise 91 92 93def max_parallel_tests_for_current_platform(): 94 # Too much test parallelization has only been seen to be a problem 95 # so far on windows. 96 if jobset.platform_string() == "windows": 97 return 64 98 return 1024 99 100 101def _print_debug_info_epilogue(dockerfile_dir=None): 102 """Use to print useful info for debug/repro just before exiting.""" 103 print("") 104 print("=== run_tests.py DEBUG INFO ===") 105 print('command: "%s"' % " ".join(sys.argv)) 106 if dockerfile_dir: 107 print("dockerfile: %s" % dockerfile_dir) 108 kokoro_job_name = os.getenv("KOKORO_JOB_NAME") 109 if kokoro_job_name: 110 print("kokoro job name: %s" % kokoro_job_name) 111 print("===============================") 112 113 114# SimpleConfig: just compile with CONFIG=config, and run the binary to test 115class Config(object): 116 def __init__( 117 self, 118 config, 119 environ=None, 120 timeout_multiplier=1, 121 tool_prefix=[], 122 iomgr_platform="native", 123 ): 124 if environ is None: 125 environ = {} 126 self.build_config = config 127 self.environ = environ 128 self.environ["CONFIG"] = config 129 self.tool_prefix = tool_prefix 130 self.timeout_multiplier = timeout_multiplier 131 self.iomgr_platform = iomgr_platform 132 133 def job_spec( 134 self, 135 cmdline, 136 timeout_seconds=_DEFAULT_TIMEOUT_SECONDS, 137 shortname=None, 138 environ={}, 139 cpu_cost=1.0, 140 flaky=False, 141 ): 142 """Construct a jobset.JobSpec for a test under this config 143 144 Args: 145 cmdline: a list of strings specifying the command line the test 146 would like to run 147 """ 148 actual_environ = self.environ.copy() 149 for k, v in environ.items(): 150 actual_environ[k] = v 151 if not flaky and shortname and shortname in flaky_tests: 152 flaky = True 153 if shortname in shortname_to_cpu: 154 cpu_cost = shortname_to_cpu[shortname] 155 return jobset.JobSpec( 156 cmdline=self.tool_prefix + cmdline, 157 shortname=shortname, 158 environ=actual_environ, 159 cpu_cost=cpu_cost, 160 timeout_seconds=( 161 self.timeout_multiplier * timeout_seconds 162 if timeout_seconds 163 else None 164 ), 165 flake_retries=4 if flaky or args.allow_flakes else 0, 166 timeout_retries=1 if flaky or args.allow_flakes else 0, 167 ) 168 169 170def get_c_tests(travis, test_lang): 171 out = [] 172 platforms_str = "ci_platforms" if travis else "platforms" 173 with open("tools/run_tests/generated/tests.json") as f: 174 js = json.load(f) 175 return [ 176 tgt 177 for tgt in js 178 if tgt["language"] == test_lang 179 and platform_string() in tgt[platforms_str] 180 and not (travis and tgt["flaky"]) 181 ] 182 183 184def _check_compiler(compiler, supported_compilers): 185 if compiler not in supported_compilers: 186 raise Exception( 187 "Compiler %s not supported (on this platform)." % compiler 188 ) 189 190 191def _check_arch(arch, supported_archs): 192 if arch not in supported_archs: 193 raise Exception("Architecture %s not supported." % arch) 194 195 196def _is_use_docker_child(): 197 """Returns True if running running as a --use_docker child.""" 198 return True if os.getenv("DOCKER_RUN_SCRIPT_COMMAND") else False 199 200 201_PythonConfigVars = collections.namedtuple( 202 "_ConfigVars", 203 [ 204 "shell", 205 "builder", 206 "builder_prefix_arguments", 207 "venv_relative_python", 208 "toolchain", 209 "runner", 210 ], 211) 212 213 214def _python_config_generator(name, major, minor, bits, config_vars): 215 build = ( 216 config_vars.shell 217 + config_vars.builder 218 + config_vars.builder_prefix_arguments 219 + [_python_pattern_function(major=major, minor=minor, bits=bits)] 220 + [name] 221 + config_vars.venv_relative_python 222 + config_vars.toolchain 223 ) 224 # run: [tools/run_tests/helper_scripts/run_python.sh py37/bin/python] 225 python_path = os.path.join(name, config_vars.venv_relative_python[0]) 226 run = config_vars.shell + config_vars.runner + [python_path] 227 return PythonConfig(name, build, run, python_path) 228 229 230def _pypy_config_generator(name, major, config_vars): 231 # Something like "py37/bin/python" 232 python_path = os.path.join(name, config_vars.venv_relative_python[0]) 233 return PythonConfig( 234 name, 235 config_vars.shell 236 + config_vars.builder 237 + config_vars.builder_prefix_arguments 238 + [_pypy_pattern_function(major=major)] 239 + [name] 240 + config_vars.venv_relative_python 241 + config_vars.toolchain, 242 config_vars.shell + config_vars.runner + [python_path], 243 python_path, 244 ) 245 246 247def _python_pattern_function(major, minor, bits): 248 # Bit-ness is handled by the test machine's environment 249 if os.name == "nt": 250 if bits == "64": 251 return "/c/Python{major}{minor}/python.exe".format( 252 major=major, minor=minor, bits=bits 253 ) 254 else: 255 return "/c/Python{major}{minor}_{bits}bits/python.exe".format( 256 major=major, minor=minor, bits=bits 257 ) 258 else: 259 return "python{major}.{minor}".format(major=major, minor=minor) 260 261 262def _pypy_pattern_function(major): 263 if major == "2": 264 return "pypy" 265 elif major == "3": 266 return "pypy3" 267 else: 268 raise ValueError("Unknown PyPy major version") 269 270 271class CLanguage(object): 272 def __init__(self, lang_suffix, test_lang): 273 self.lang_suffix = lang_suffix 274 self.platform = platform_string() 275 self.test_lang = test_lang 276 277 def configure(self, config, args): 278 self.config = config 279 self.args = args 280 if self.platform == "windows": 281 _check_compiler( 282 self.args.compiler, 283 [ 284 "default", 285 "cmake", 286 "cmake_ninja_vs2019", 287 "cmake_ninja_vs2022", 288 "cmake_vs2019", 289 "cmake_vs2022", 290 ], 291 ) 292 _check_arch(self.args.arch, ["default", "x64", "x86"]) 293 294 activate_vs_tools = "" 295 if ( 296 self.args.compiler == "cmake_ninja_vs2019" 297 or self.args.compiler == "cmake" 298 or self.args.compiler == "default" 299 ): 300 # cmake + ninja build is the default because it is faster and supports boringssl assembly optimizations 301 # the compiler used is exactly the same as for cmake_vs2017 302 cmake_generator = "Ninja" 303 activate_vs_tools = "2019" 304 elif self.args.compiler == "cmake_ninja_vs2022": 305 cmake_generator = "Ninja" 306 activate_vs_tools = "2022" 307 elif self.args.compiler == "cmake_vs2019": 308 cmake_generator = "Visual Studio 16 2019" 309 elif self.args.compiler == "cmake_vs2022": 310 cmake_generator = "Visual Studio 17 2022" 311 else: 312 print("should never reach here.") 313 sys.exit(1) 314 315 self._cmake_configure_extra_args = list( 316 self.args.cmake_configure_extra_args 317 ) 318 self._cmake_generator_windows = cmake_generator 319 # required to pass as cmake "-A" configuration for VS builds (but not for Ninja) 320 self._cmake_architecture_windows = ( 321 "x64" if self.args.arch == "x64" else "Win32" 322 ) 323 # when builing with Ninja, the VS common tools need to be activated first 324 self._activate_vs_tools_windows = activate_vs_tools 325 # "x64_x86" means create 32bit binaries, but use 64bit toolkit to secure more memory for the build 326 self._vs_tools_architecture_windows = ( 327 "x64" if self.args.arch == "x64" else "x64_x86" 328 ) 329 330 else: 331 if self.platform == "linux": 332 # Allow all the known architectures. _check_arch_option has already checked that we're not doing 333 # something illegal when not running under docker. 334 _check_arch(self.args.arch, ["default", "x64", "x86", "arm64"]) 335 else: 336 _check_arch(self.args.arch, ["default"]) 337 338 ( 339 self._docker_distro, 340 self._cmake_configure_extra_args, 341 ) = self._compiler_options( 342 self.args.use_docker, 343 self.args.compiler, 344 self.args.cmake_configure_extra_args, 345 ) 346 347 def test_specs(self): 348 out = [] 349 binaries = get_c_tests(self.args.travis, self.test_lang) 350 for target in binaries: 351 if target.get("boringssl", False): 352 # cmake doesn't build boringssl tests 353 continue 354 auto_timeout_scaling = target.get("auto_timeout_scaling", True) 355 polling_strategies = ( 356 _POLLING_STRATEGIES.get(self.platform, ["all"]) 357 if target.get("uses_polling", True) 358 else ["none"] 359 ) 360 for polling_strategy in polling_strategies: 361 env = { 362 "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH": _ROOT 363 + "/src/core/tsi/test_creds/ca.pem", 364 "GRPC_POLL_STRATEGY": polling_strategy, 365 "GRPC_VERBOSITY": "DEBUG", 366 } 367 resolver = os.environ.get("GRPC_DNS_RESOLVER", None) 368 if resolver: 369 env["GRPC_DNS_RESOLVER"] = resolver 370 shortname_ext = ( 371 "" 372 if polling_strategy == "all" 373 else " GRPC_POLL_STRATEGY=%s" % polling_strategy 374 ) 375 if polling_strategy in target.get("excluded_poll_engines", []): 376 continue 377 378 timeout_scaling = 1 379 if auto_timeout_scaling: 380 config = self.args.config 381 if ( 382 "asan" in config 383 or config == "msan" 384 or config == "tsan" 385 or config == "ubsan" 386 or config == "helgrind" 387 or config == "memcheck" 388 ): 389 # Scale overall test timeout if running under various sanitizers. 390 # scaling value is based on historical data analysis 391 timeout_scaling *= 3 392 393 if self.config.build_config in target["exclude_configs"]: 394 continue 395 if self.args.iomgr_platform in target.get("exclude_iomgrs", []): 396 continue 397 398 if self.platform == "windows": 399 if self._cmake_generator_windows == "Ninja": 400 binary = "cmake/build/%s.exe" % target["name"] 401 else: 402 binary = "cmake/build/%s/%s.exe" % ( 403 _MSBUILD_CONFIG[self.config.build_config], 404 target["name"], 405 ) 406 else: 407 binary = "cmake/build/%s" % target["name"] 408 409 cpu_cost = target["cpu_cost"] 410 if cpu_cost == "capacity": 411 cpu_cost = multiprocessing.cpu_count() 412 if os.path.isfile(binary): 413 list_test_command = None 414 filter_test_command = None 415 416 # these are the flag defined by gtest and benchmark framework to list 417 # and filter test runs. We use them to split each individual test 418 # into its own JobSpec, and thus into its own process. 419 if "benchmark" in target and target["benchmark"]: 420 with open(os.devnull, "w") as fnull: 421 tests = subprocess.check_output( 422 [binary, "--benchmark_list_tests"], stderr=fnull 423 ) 424 for line in tests.decode().split("\n"): 425 test = line.strip() 426 if not test: 427 continue 428 cmdline = [ 429 binary, 430 "--benchmark_filter=%s$" % test, 431 ] + target["args"] 432 out.append( 433 self.config.job_spec( 434 cmdline, 435 shortname="%s %s" 436 % (" ".join(cmdline), shortname_ext), 437 cpu_cost=cpu_cost, 438 timeout_seconds=target.get( 439 "timeout_seconds", 440 _DEFAULT_TIMEOUT_SECONDS, 441 ) 442 * timeout_scaling, 443 environ=env, 444 ) 445 ) 446 elif "gtest" in target and target["gtest"]: 447 # here we parse the output of --gtest_list_tests to build up a complete 448 # list of the tests contained in a binary for each test, we then 449 # add a job to run, filtering for just that test. 450 with open(os.devnull, "w") as fnull: 451 tests = subprocess.check_output( 452 [binary, "--gtest_list_tests"], stderr=fnull 453 ) 454 base = None 455 for line in tests.decode().split("\n"): 456 i = line.find("#") 457 if i >= 0: 458 line = line[:i] 459 if not line: 460 continue 461 if line[0] != " ": 462 base = line.strip() 463 else: 464 assert base is not None 465 assert line[1] == " " 466 test = base + line.strip() 467 cmdline = [ 468 binary, 469 "--gtest_filter=%s" % test, 470 ] + target["args"] 471 out.append( 472 self.config.job_spec( 473 cmdline, 474 shortname="%s %s" 475 % (" ".join(cmdline), shortname_ext), 476 cpu_cost=cpu_cost, 477 timeout_seconds=target.get( 478 "timeout_seconds", 479 _DEFAULT_TIMEOUT_SECONDS, 480 ) 481 * timeout_scaling, 482 environ=env, 483 ) 484 ) 485 else: 486 cmdline = [binary] + target["args"] 487 shortname = target.get( 488 "shortname", 489 " ".join(shlex.quote(arg) for arg in cmdline), 490 ) 491 shortname += shortname_ext 492 out.append( 493 self.config.job_spec( 494 cmdline, 495 shortname=shortname, 496 cpu_cost=cpu_cost, 497 flaky=target.get("flaky", False), 498 timeout_seconds=target.get( 499 "timeout_seconds", _DEFAULT_TIMEOUT_SECONDS 500 ) 501 * timeout_scaling, 502 environ=env, 503 ) 504 ) 505 elif self.args.regex == ".*" or self.platform == "windows": 506 print("\nWARNING: binary not found, skipping", binary) 507 return sorted(out) 508 509 def pre_build_steps(self): 510 return [] 511 512 def build_steps(self): 513 if self.platform == "windows": 514 return [ 515 [ 516 "tools\\run_tests\\helper_scripts\\build_cxx.bat", 517 "-DgRPC_BUILD_MSVC_MP_COUNT=%d" % self.args.jobs, 518 ] 519 + self._cmake_configure_extra_args 520 ] 521 else: 522 return [ 523 ["tools/run_tests/helper_scripts/build_cxx.sh"] 524 + self._cmake_configure_extra_args 525 ] 526 527 def build_steps_environ(self): 528 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 529 environ = {"GRPC_RUN_TESTS_CXX_LANGUAGE_SUFFIX": self.lang_suffix} 530 if self.platform == "windows": 531 environ["GRPC_CMAKE_GENERATOR"] = self._cmake_generator_windows 532 environ[ 533 "GRPC_CMAKE_ARCHITECTURE" 534 ] = self._cmake_architecture_windows 535 environ[ 536 "GRPC_BUILD_ACTIVATE_VS_TOOLS" 537 ] = self._activate_vs_tools_windows 538 environ[ 539 "GRPC_BUILD_VS_TOOLS_ARCHITECTURE" 540 ] = self._vs_tools_architecture_windows 541 elif self.platform == "linux": 542 environ["GRPC_RUNTESTS_ARCHITECTURE"] = self.args.arch 543 return environ 544 545 def post_tests_steps(self): 546 if self.platform == "windows": 547 return [] 548 else: 549 return [["tools/run_tests/helper_scripts/post_tests_c.sh"]] 550 551 def _clang_cmake_configure_extra_args(self, version_suffix=""): 552 return [ 553 "-DCMAKE_C_COMPILER=clang%s" % version_suffix, 554 "-DCMAKE_CXX_COMPILER=clang++%s" % version_suffix, 555 ] 556 557 def _compiler_options( 558 self, use_docker, compiler, cmake_configure_extra_args 559 ): 560 """Returns docker distro and cmake configure args to use for given compiler.""" 561 if cmake_configure_extra_args: 562 # only allow specifying extra cmake args for "vanilla" compiler 563 _check_compiler(compiler, ["default", "cmake"]) 564 return ("nonexistent_docker_distro", cmake_configure_extra_args) 565 if not use_docker and not _is_use_docker_child(): 566 # if not running under docker, we cannot ensure the right compiler version will be used, 567 # so we only allow the non-specific choices. 568 _check_compiler(compiler, ["default", "cmake"]) 569 570 if compiler == "default" or compiler == "cmake": 571 return ("debian11", []) 572 elif compiler == "gcc8": 573 return ("gcc_8", []) 574 elif compiler == "gcc10.2": 575 return ("debian11", []) 576 elif compiler == "gcc10.2_openssl102": 577 return ( 578 "debian11_openssl102", 579 [ 580 "-DgRPC_SSL_PROVIDER=package", 581 ], 582 ) 583 elif compiler == "gcc10.2_openssl111": 584 return ( 585 "debian11_openssl111", 586 [ 587 "-DgRPC_SSL_PROVIDER=package", 588 ], 589 ) 590 elif compiler == "gcc12": 591 return ("gcc_12", ["-DCMAKE_CXX_STANDARD=20"]) 592 elif compiler == "gcc12_openssl309": 593 return ( 594 "debian12_openssl309", 595 [ 596 "-DgRPC_SSL_PROVIDER=package", 597 ], 598 ) 599 elif compiler == "gcc_musl": 600 return ("alpine", []) 601 elif compiler == "clang6": 602 return ("clang_6", self._clang_cmake_configure_extra_args()) 603 elif compiler == "clang17": 604 return ("clang_17", self._clang_cmake_configure_extra_args()) 605 else: 606 raise Exception("Compiler %s not supported." % compiler) 607 608 def dockerfile_dir(self): 609 return "tools/dockerfile/test/cxx_%s_%s" % ( 610 self._docker_distro, 611 _docker_arch_suffix(self.args.arch), 612 ) 613 614 def __str__(self): 615 return self.lang_suffix 616 617 618class Php7Language(object): 619 def configure(self, config, args): 620 self.config = config 621 self.args = args 622 _check_compiler(self.args.compiler, ["default"]) 623 624 def test_specs(self): 625 return [ 626 self.config.job_spec( 627 ["src/php/bin/run_tests.sh"], 628 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 629 ) 630 ] 631 632 def pre_build_steps(self): 633 return [] 634 635 def build_steps(self): 636 return [["tools/run_tests/helper_scripts/build_php.sh"]] 637 638 def build_steps_environ(self): 639 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 640 return {} 641 642 def post_tests_steps(self): 643 return [["tools/run_tests/helper_scripts/post_tests_php.sh"]] 644 645 def dockerfile_dir(self): 646 return "tools/dockerfile/test/php7_debian11_%s" % _docker_arch_suffix( 647 self.args.arch 648 ) 649 650 def __str__(self): 651 return "php7" 652 653 654class PythonConfig( 655 collections.namedtuple( 656 "PythonConfig", ["name", "build", "run", "python_path"] 657 ) 658): 659 """Tuple of commands (named s.t. 'what it says on the tin' applies)""" 660 661 662class PythonLanguage(object): 663 _TEST_SPECS_FILE = { 664 "native": ["src/python/grpcio_tests/tests/tests.json"], 665 "asyncio": ["src/python/grpcio_tests/tests_aio/tests.json"], 666 } 667 668 _TEST_COMMAND = { 669 "native": "test_lite", 670 "asyncio": "test_aio", 671 } 672 673 def configure(self, config, args): 674 self.config = config 675 self.args = args 676 self.pythons = self._get_pythons(self.args) 677 678 def test_specs(self): 679 # load list of known test suites 680 jobs = [] 681 682 # Run tests across all supported interpreters. 683 for python_config in self.pythons: 684 # Run non-io-manager-specific tests. 685 if os.name != "nt": 686 jobs.append( 687 self.config.job_spec( 688 [ 689 python_config.python_path, 690 "tools/distrib/python/xds_protos/generated_file_import_test.py", 691 ], 692 timeout_seconds=60, 693 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 694 shortname=f"{python_config.name}.xds_protos", 695 ) 696 ) 697 698 # Run main test suite across all support IO managers. 699 for io_platform in self._TEST_SPECS_FILE: 700 test_cases = [] 701 for tests_json_file_name in self._TEST_SPECS_FILE[io_platform]: 702 with open(tests_json_file_name) as tests_json_file: 703 test_cases.extend(json.load(tests_json_file)) 704 705 environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) 706 # TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not 707 # designed for non-native IO manager. It has a side-effect that 708 # overrides threading settings in C-Core. 709 if io_platform != "native": 710 environment["GRPC_ENABLE_FORK_SUPPORT"] = "0" 711 jobs.extend( 712 [ 713 self.config.job_spec( 714 python_config.run 715 + [self._TEST_COMMAND[io_platform]], 716 timeout_seconds=8 * 60, 717 environ=dict( 718 GRPC_PYTHON_TESTRUNNER_FILTER=str(test_case), 719 **environment, 720 ), 721 shortname=f"{python_config.name}.{io_platform}.{test_case}", 722 ) 723 for test_case in test_cases 724 ] 725 ) 726 return jobs 727 728 def pre_build_steps(self): 729 return [] 730 731 def build_steps(self): 732 return [config.build for config in self.pythons] 733 734 def build_steps_environ(self): 735 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 736 return {} 737 738 def post_tests_steps(self): 739 if self.config.build_config != "gcov": 740 return [] 741 else: 742 return [["tools/run_tests/helper_scripts/post_tests_python.sh"]] 743 744 def dockerfile_dir(self): 745 return "tools/dockerfile/test/python_%s_%s" % ( 746 self._python_docker_distro_name(), 747 _docker_arch_suffix(self.args.arch), 748 ) 749 750 def _python_docker_distro_name(self): 751 """Choose the docker image to use based on python version.""" 752 if self.args.compiler == "python_alpine": 753 return "alpine" 754 else: 755 return "debian11_default" 756 757 def _get_pythons(self, args): 758 """Get python runtimes to test with, based on current platform, architecture, compiler etc.""" 759 if args.iomgr_platform != "native": 760 raise ValueError( 761 "Python builds no longer differentiate IO Manager platforms," 762 ' please use "native"' 763 ) 764 765 if args.arch == "x86": 766 bits = "32" 767 else: 768 bits = "64" 769 770 if os.name == "nt": 771 shell = ["bash"] 772 builder = [ 773 os.path.abspath( 774 "tools/run_tests/helper_scripts/build_python_msys2.sh" 775 ) 776 ] 777 builder_prefix_arguments = ["MINGW{}".format(bits)] 778 venv_relative_python = ["Scripts/python.exe"] 779 toolchain = ["mingw32"] 780 else: 781 shell = [] 782 builder = [ 783 os.path.abspath( 784 "tools/run_tests/helper_scripts/build_python.sh" 785 ) 786 ] 787 builder_prefix_arguments = [] 788 venv_relative_python = ["bin/python"] 789 toolchain = ["unix"] 790 791 runner = [ 792 os.path.abspath("tools/run_tests/helper_scripts/run_python.sh") 793 ] 794 795 config_vars = _PythonConfigVars( 796 shell, 797 builder, 798 builder_prefix_arguments, 799 venv_relative_python, 800 toolchain, 801 runner, 802 ) 803 804 # TODO: Supported version range should be defined by a single 805 # source of truth. 806 python38_config = _python_config_generator( 807 name="py38", 808 major="3", 809 minor="8", 810 bits=bits, 811 config_vars=config_vars, 812 ) 813 python39_config = _python_config_generator( 814 name="py39", 815 major="3", 816 minor="9", 817 bits=bits, 818 config_vars=config_vars, 819 ) 820 python310_config = _python_config_generator( 821 name="py310", 822 major="3", 823 minor="10", 824 bits=bits, 825 config_vars=config_vars, 826 ) 827 python311_config = _python_config_generator( 828 name="py311", 829 major="3", 830 minor="11", 831 bits=bits, 832 config_vars=config_vars, 833 ) 834 python312_config = _python_config_generator( 835 name="py312", 836 major="3", 837 minor="12", 838 bits=bits, 839 config_vars=config_vars, 840 ) 841 pypy27_config = _pypy_config_generator( 842 name="pypy", major="2", config_vars=config_vars 843 ) 844 pypy32_config = _pypy_config_generator( 845 name="pypy3", major="3", config_vars=config_vars 846 ) 847 848 if args.compiler == "default": 849 if os.name == "nt": 850 return (python38_config,) 851 elif os.uname()[0] == "Darwin": 852 # NOTE(rbellevi): Testing takes significantly longer on 853 # MacOS, so we restrict the number of interpreter versions 854 # tested. 855 return (python38_config,) 856 elif platform.machine() == "aarch64": 857 # Currently the python_debian11_default_arm64 docker image 858 # only has python3.9 installed (and that seems sufficient 859 # for arm64 testing) 860 return (python39_config,) 861 else: 862 # Default set tested on master. Test oldest and newest. 863 return ( 864 python38_config, 865 python312_config, 866 ) 867 elif args.compiler == "python3.8": 868 return (python38_config,) 869 elif args.compiler == "python3.9": 870 return (python39_config,) 871 elif args.compiler == "python3.10": 872 return (python310_config,) 873 elif args.compiler == "python3.11": 874 return (python311_config,) 875 elif args.compiler == "python3.12": 876 return (python312_config,) 877 elif args.compiler == "pypy": 878 return (pypy27_config,) 879 elif args.compiler == "pypy3": 880 return (pypy32_config,) 881 elif args.compiler == "python_alpine": 882 return (python39_config,) 883 elif args.compiler == "all_the_cpythons": 884 return ( 885 python38_config, 886 python39_config, 887 python310_config, 888 python311_config, 889 python312_config, 890 ) 891 else: 892 raise Exception("Compiler %s not supported." % args.compiler) 893 894 def __str__(self): 895 return "python" 896 897 898class RubyLanguage(object): 899 def configure(self, config, args): 900 self.config = config 901 self.args = args 902 _check_compiler(self.args.compiler, ["default"]) 903 904 def test_specs(self): 905 tests = [ 906 self.config.job_spec( 907 ["tools/run_tests/helper_scripts/run_ruby.sh"], 908 timeout_seconds=10 * 60, 909 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 910 ) 911 ] 912 # TODO(apolcyn): re-enable the following tests after 913 # https://bugs.ruby-lang.org/issues/15499 is fixed: 914 # They previously worked on ruby 2.5 but needed to be disabled 915 # after dropping support for ruby 2.5: 916 # - src/ruby/end2end/channel_state_test.rb 917 # - src/ruby/end2end/sig_int_during_channel_watch_test.rb 918 # TODO(apolcyn): the following test is skipped because it sometimes 919 # hits "Bus Error" crashes while requiring the grpc/ruby C-extension. 920 # This crashes have been unreproducible outside of CI. Also see 921 # b/266212253. 922 # - src/ruby/end2end/grpc_class_init_test.rb 923 for test in [ 924 "src/ruby/end2end/fork_test.rb", 925 "src/ruby/end2end/simple_fork_test.rb", 926 "src/ruby/end2end/prefork_without_using_grpc_test.rb", 927 "src/ruby/end2end/prefork_postfork_loop_test.rb", 928 "src/ruby/end2end/secure_fork_test.rb", 929 "src/ruby/end2end/bad_usage_fork_test.rb", 930 "src/ruby/end2end/sig_handling_test.rb", 931 "src/ruby/end2end/channel_closing_test.rb", 932 "src/ruby/end2end/killed_client_thread_test.rb", 933 "src/ruby/end2end/forking_client_test.rb", 934 "src/ruby/end2end/multiple_killed_watching_threads_test.rb", 935 "src/ruby/end2end/load_grpc_with_gc_stress_test.rb", 936 "src/ruby/end2end/client_memory_usage_test.rb", 937 "src/ruby/end2end/package_with_underscore_test.rb", 938 "src/ruby/end2end/graceful_sig_handling_test.rb", 939 "src/ruby/end2end/graceful_sig_stop_test.rb", 940 "src/ruby/end2end/errors_load_before_grpc_lib_test.rb", 941 "src/ruby/end2end/logger_load_before_grpc_lib_test.rb", 942 "src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb", 943 "src/ruby/end2end/call_credentials_timeout_test.rb", 944 "src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb", 945 ]: 946 if test in [ 947 "src/ruby/end2end/fork_test.rb", 948 "src/ruby/end2end/simple_fork_test.rb", 949 "src/ruby/end2end/secure_fork_test.rb", 950 "src/ruby/end2end/bad_usage_fork_test.rb", 951 "src/ruby/end2end/prefork_without_using_grpc_test.rb", 952 "src/ruby/end2end/prefork_postfork_loop_test.rb", 953 "src/ruby/end2end/fork_test_repro_35489.rb", 954 ]: 955 # Skip fork tests in general until https://github.com/grpc/grpc/issues/34442 956 # is fixed. Otherwise we see too many flakes. 957 # After that's fixed, we should continue to skip on mac 958 # indefinitely, and on "dbg" builds until the Event Engine 959 # migration completes. 960 continue 961 tests.append( 962 self.config.job_spec( 963 ["ruby", test], 964 shortname=test, 965 timeout_seconds=20 * 60, 966 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 967 ) 968 ) 969 return tests 970 971 def pre_build_steps(self): 972 return [["tools/run_tests/helper_scripts/pre_build_ruby.sh"]] 973 974 def build_steps(self): 975 return [["tools/run_tests/helper_scripts/build_ruby.sh"]] 976 977 def build_steps_environ(self): 978 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 979 return {} 980 981 def post_tests_steps(self): 982 return [["tools/run_tests/helper_scripts/post_tests_ruby.sh"]] 983 984 def dockerfile_dir(self): 985 return "tools/dockerfile/test/ruby_debian11_%s" % _docker_arch_suffix( 986 self.args.arch 987 ) 988 989 def __str__(self): 990 return "ruby" 991 992 993class CSharpLanguage(object): 994 def __init__(self): 995 self.platform = platform_string() 996 997 def configure(self, config, args): 998 self.config = config 999 self.args = args 1000 _check_compiler(self.args.compiler, ["default", "coreclr", "mono"]) 1001 if self.args.compiler == "default": 1002 # test both runtimes by default 1003 self.test_runtimes = ["coreclr", "mono"] 1004 else: 1005 # only test the specified runtime 1006 self.test_runtimes = [self.args.compiler] 1007 1008 if self.platform == "windows": 1009 _check_arch(self.args.arch, ["default"]) 1010 self._cmake_arch_option = "x64" 1011 else: 1012 self._docker_distro = "debian11" 1013 1014 def test_specs(self): 1015 with open("src/csharp/tests.json") as f: 1016 tests_by_assembly = json.load(f) 1017 1018 msbuild_config = _MSBUILD_CONFIG[self.config.build_config] 1019 nunit_args = ["--labels=All", "--noresult", "--workers=1"] 1020 1021 specs = [] 1022 for test_runtime in self.test_runtimes: 1023 if test_runtime == "coreclr": 1024 assembly_extension = ".dll" 1025 assembly_subdir = "bin/%s/netcoreapp3.1" % msbuild_config 1026 runtime_cmd = ["dotnet", "exec"] 1027 elif test_runtime == "mono": 1028 assembly_extension = ".exe" 1029 assembly_subdir = "bin/%s/net45" % msbuild_config 1030 if self.platform == "windows": 1031 runtime_cmd = [] 1032 elif self.platform == "mac": 1033 # mono before version 5.2 on MacOS defaults to 32bit runtime 1034 runtime_cmd = ["mono", "--arch=64"] 1035 else: 1036 runtime_cmd = ["mono"] 1037 else: 1038 raise Exception('Illegal runtime "%s" was specified.') 1039 1040 for assembly in six.iterkeys(tests_by_assembly): 1041 assembly_file = "src/csharp/%s/%s/%s%s" % ( 1042 assembly, 1043 assembly_subdir, 1044 assembly, 1045 assembly_extension, 1046 ) 1047 1048 # normally, run each test as a separate process 1049 for test in tests_by_assembly[assembly]: 1050 cmdline = ( 1051 runtime_cmd 1052 + [assembly_file, "--test=%s" % test] 1053 + nunit_args 1054 ) 1055 specs.append( 1056 self.config.job_spec( 1057 cmdline, 1058 shortname="csharp.%s.%s" % (test_runtime, test), 1059 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 1060 ) 1061 ) 1062 return specs 1063 1064 def pre_build_steps(self): 1065 if self.platform == "windows": 1066 return [["tools\\run_tests\\helper_scripts\\pre_build_csharp.bat"]] 1067 else: 1068 return [["tools/run_tests/helper_scripts/pre_build_csharp.sh"]] 1069 1070 def build_steps(self): 1071 if self.platform == "windows": 1072 return [["tools\\run_tests\\helper_scripts\\build_csharp.bat"]] 1073 else: 1074 return [["tools/run_tests/helper_scripts/build_csharp.sh"]] 1075 1076 def build_steps_environ(self): 1077 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1078 if self.platform == "windows": 1079 return {"ARCHITECTURE": self._cmake_arch_option} 1080 else: 1081 return {} 1082 1083 def post_tests_steps(self): 1084 if self.platform == "windows": 1085 return [["tools\\run_tests\\helper_scripts\\post_tests_csharp.bat"]] 1086 else: 1087 return [["tools/run_tests/helper_scripts/post_tests_csharp.sh"]] 1088 1089 def dockerfile_dir(self): 1090 return "tools/dockerfile/test/csharp_%s_%s" % ( 1091 self._docker_distro, 1092 _docker_arch_suffix(self.args.arch), 1093 ) 1094 1095 def __str__(self): 1096 return "csharp" 1097 1098 1099class ObjCLanguage(object): 1100 def configure(self, config, args): 1101 self.config = config 1102 self.args = args 1103 _check_compiler(self.args.compiler, ["default"]) 1104 1105 def test_specs(self): 1106 out = [] 1107 out.append( 1108 self.config.job_spec( 1109 ["src/objective-c/tests/build_one_example.sh"], 1110 timeout_seconds=20 * 60, 1111 shortname="ios-buildtest-example-sample", 1112 cpu_cost=1e6, 1113 environ={ 1114 "SCHEME": "Sample", 1115 "EXAMPLE_PATH": "src/objective-c/examples/Sample", 1116 }, 1117 ) 1118 ) 1119 # TODO(jtattermusch): Create bazel target for the sample and remove the test task from here. 1120 out.append( 1121 self.config.job_spec( 1122 ["src/objective-c/tests/build_one_example.sh"], 1123 timeout_seconds=20 * 60, 1124 shortname="ios-buildtest-example-switftsample", 1125 cpu_cost=1e6, 1126 environ={ 1127 "SCHEME": "SwiftSample", 1128 "EXAMPLE_PATH": "src/objective-c/examples/SwiftSample", 1129 }, 1130 ) 1131 ) 1132 out.append( 1133 self.config.job_spec( 1134 ["src/objective-c/tests/build_one_example.sh"], 1135 timeout_seconds=20 * 60, 1136 shortname="ios-buildtest-example-switft-use-frameworks", 1137 cpu_cost=1e6, 1138 environ={ 1139 "SCHEME": "SwiftUseFrameworks", 1140 "EXAMPLE_PATH": "src/objective-c/examples/SwiftUseFrameworks", 1141 }, 1142 ) 1143 ) 1144 1145 # Disabled due to #20258 1146 # TODO (mxyan): Reenable this test when #20258 is resolved. 1147 # out.append( 1148 # self.config.job_spec( 1149 # ['src/objective-c/tests/build_one_example_bazel.sh'], 1150 # timeout_seconds=20 * 60, 1151 # shortname='ios-buildtest-example-watchOS-sample', 1152 # cpu_cost=1e6, 1153 # environ={ 1154 # 'SCHEME': 'watchOS-sample-WatchKit-App', 1155 # 'EXAMPLE_PATH': 'src/objective-c/examples/watchOS-sample', 1156 # 'FRAMEWORKS': 'NO' 1157 # })) 1158 1159 # TODO(jtattermusch): move the test out of the test/core/iomgr/CFStreamTests directory? 1160 # How does one add the cfstream dependency in bazel? 1161 out.append( 1162 self.config.job_spec( 1163 ["test/core/iomgr/ios/CFStreamTests/build_and_run_tests.sh"], 1164 timeout_seconds=60 * 60, 1165 shortname="ios-test-cfstream-tests", 1166 cpu_cost=1e6, 1167 environ=_FORCE_ENVIRON_FOR_WRAPPERS, 1168 ) 1169 ) 1170 return sorted(out) 1171 1172 def pre_build_steps(self): 1173 return [] 1174 1175 def build_steps(self): 1176 return [] 1177 1178 def build_steps_environ(self): 1179 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1180 return {} 1181 1182 def post_tests_steps(self): 1183 return [] 1184 1185 def dockerfile_dir(self): 1186 return None 1187 1188 def __str__(self): 1189 return "objc" 1190 1191 1192class Sanity(object): 1193 def __init__(self, config_file): 1194 self.config_file = config_file 1195 1196 def configure(self, config, args): 1197 self.config = config 1198 self.args = args 1199 _check_compiler(self.args.compiler, ["default"]) 1200 1201 def test_specs(self): 1202 import yaml 1203 1204 with open("tools/run_tests/sanity/%s" % self.config_file, "r") as f: 1205 environ = {"TEST": "true"} 1206 if _is_use_docker_child(): 1207 environ["CLANG_FORMAT_SKIP_DOCKER"] = "true" 1208 environ["CLANG_TIDY_SKIP_DOCKER"] = "true" 1209 environ["IWYU_SKIP_DOCKER"] = "true" 1210 # sanity tests run tools/bazel wrapper concurrently 1211 # and that can result in a download/run race in the wrapper. 1212 # under docker we already have the right version of bazel 1213 # so we can just disable the wrapper. 1214 environ["DISABLE_BAZEL_WRAPPER"] = "true" 1215 return [ 1216 self.config.job_spec( 1217 cmd["script"].split(), 1218 timeout_seconds=45 * 60, 1219 environ=environ, 1220 cpu_cost=cmd.get("cpu_cost", 1), 1221 ) 1222 for cmd in yaml.safe_load(f) 1223 ] 1224 1225 def pre_build_steps(self): 1226 return [] 1227 1228 def build_steps(self): 1229 return [] 1230 1231 def build_steps_environ(self): 1232 """Extra environment variables set for pre_build_steps and build_steps jobs.""" 1233 return {} 1234 1235 def post_tests_steps(self): 1236 return [] 1237 1238 def dockerfile_dir(self): 1239 return "tools/dockerfile/test/sanity" 1240 1241 def __str__(self): 1242 return "sanity" 1243 1244 1245# different configurations we can run under 1246with open("tools/run_tests/generated/configs.json") as f: 1247 _CONFIGS = dict( 1248 (cfg["config"], Config(**cfg)) for cfg in ast.literal_eval(f.read()) 1249 ) 1250 1251_LANGUAGES = { 1252 "c++": CLanguage("cxx", "c++"), 1253 "c": CLanguage("c", "c"), 1254 "php7": Php7Language(), 1255 "python": PythonLanguage(), 1256 "ruby": RubyLanguage(), 1257 "csharp": CSharpLanguage(), 1258 "objc": ObjCLanguage(), 1259 "sanity": Sanity("sanity_tests.yaml"), 1260 "clang-tidy": Sanity("clang_tidy_tests.yaml"), 1261} 1262 1263_MSBUILD_CONFIG = { 1264 "dbg": "Debug", 1265 "opt": "Release", 1266 "gcov": "Debug", 1267} 1268 1269 1270def _build_step_environ(cfg, extra_env={}): 1271 """Environment variables set for each build step.""" 1272 environ = {"CONFIG": cfg, "GRPC_RUN_TESTS_JOBS": str(args.jobs)} 1273 msbuild_cfg = _MSBUILD_CONFIG.get(cfg) 1274 if msbuild_cfg: 1275 environ["MSBUILD_CONFIG"] = msbuild_cfg 1276 environ.update(extra_env) 1277 return environ 1278 1279 1280def _windows_arch_option(arch): 1281 """Returns msbuild cmdline option for selected architecture.""" 1282 if arch == "default" or arch == "x86": 1283 return "/p:Platform=Win32" 1284 elif arch == "x64": 1285 return "/p:Platform=x64" 1286 else: 1287 print("Architecture %s not supported." % arch) 1288 sys.exit(1) 1289 1290 1291def _check_arch_option(arch): 1292 """Checks that architecture option is valid.""" 1293 if platform_string() == "windows": 1294 _windows_arch_option(arch) 1295 elif platform_string() == "linux": 1296 # On linux, we need to be running under docker with the right architecture. 1297 runtime_machine = platform.machine() 1298 runtime_arch = platform.architecture()[0] 1299 if arch == "default": 1300 return 1301 elif ( 1302 runtime_machine == "x86_64" 1303 and runtime_arch == "64bit" 1304 and arch == "x64" 1305 ): 1306 return 1307 elif ( 1308 runtime_machine == "x86_64" 1309 and runtime_arch == "32bit" 1310 and arch == "x86" 1311 ): 1312 return 1313 elif ( 1314 runtime_machine == "aarch64" 1315 and runtime_arch == "64bit" 1316 and arch == "arm64" 1317 ): 1318 return 1319 else: 1320 print( 1321 "Architecture %s does not match current runtime architecture." 1322 % arch 1323 ) 1324 sys.exit(1) 1325 else: 1326 if args.arch != "default": 1327 print( 1328 "Architecture %s not supported on current platform." % args.arch 1329 ) 1330 sys.exit(1) 1331 1332 1333def _docker_arch_suffix(arch): 1334 """Returns suffix to dockerfile dir to use.""" 1335 if arch == "default" or arch == "x64": 1336 return "x64" 1337 elif arch == "x86": 1338 return "x86" 1339 elif arch == "arm64": 1340 return "arm64" 1341 else: 1342 print("Architecture %s not supported with current settings." % arch) 1343 sys.exit(1) 1344 1345 1346def runs_per_test_type(arg_str): 1347 """Auxiliary function to parse the "runs_per_test" flag. 1348 1349 Returns: 1350 A positive integer or 0, the latter indicating an infinite number of 1351 runs. 1352 1353 Raises: 1354 argparse.ArgumentTypeError: Upon invalid input. 1355 """ 1356 if arg_str == "inf": 1357 return 0 1358 try: 1359 n = int(arg_str) 1360 if n <= 0: 1361 raise ValueError 1362 return n 1363 except: 1364 msg = "'{}' is not a positive integer or 'inf'".format(arg_str) 1365 raise argparse.ArgumentTypeError(msg) 1366 1367 1368def percent_type(arg_str): 1369 pct = float(arg_str) 1370 if pct > 100 or pct < 0: 1371 raise argparse.ArgumentTypeError( 1372 "'%f' is not a valid percentage in the [0, 100] range" % pct 1373 ) 1374 return pct 1375 1376 1377# This is math.isclose in python >= 3.5 1378def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): 1379 return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) 1380 1381 1382def _shut_down_legacy_server(legacy_server_port): 1383 """Shut down legacy version of port server.""" 1384 try: 1385 version = int( 1386 urllib.request.urlopen( 1387 "http://localhost:%d/version_number" % legacy_server_port, 1388 timeout=10, 1389 ).read() 1390 ) 1391 except: 1392 pass 1393 else: 1394 urllib.request.urlopen( 1395 "http://localhost:%d/quitquitquit" % legacy_server_port 1396 ).read() 1397 1398 1399def _calculate_num_runs_failures(list_of_results): 1400 """Calculate number of runs and failures for a particular test. 1401 1402 Args: 1403 list_of_results: (List) of JobResult object. 1404 Returns: 1405 A tuple of total number of runs and failures. 1406 """ 1407 num_runs = len(list_of_results) # By default, there is 1 run per JobResult. 1408 num_failures = 0 1409 for jobresult in list_of_results: 1410 if jobresult.retries > 0: 1411 num_runs += jobresult.retries 1412 if jobresult.num_failures > 0: 1413 num_failures += jobresult.num_failures 1414 return num_runs, num_failures 1415 1416 1417class BuildAndRunError(object): 1418 """Represents error type in _build_and_run.""" 1419 1420 BUILD = object() 1421 TEST = object() 1422 POST_TEST = object() 1423 1424 1425# returns a list of things that failed (or an empty list on success) 1426def _build_and_run( 1427 check_cancelled, newline_on_success, xml_report=None, build_only=False 1428): 1429 """Do one pass of building & running tests.""" 1430 # build latest sequentially 1431 num_failures, resultset = jobset.run( 1432 build_steps, 1433 maxjobs=1, 1434 stop_on_failure=True, 1435 newline_on_success=newline_on_success, 1436 travis=args.travis, 1437 ) 1438 if num_failures: 1439 return [BuildAndRunError.BUILD] 1440 1441 if build_only: 1442 if xml_report: 1443 report_utils.render_junit_xml_report( 1444 resultset, xml_report, suite_name=args.report_suite_name 1445 ) 1446 return [] 1447 1448 # start antagonists 1449 antagonists = [ 1450 subprocess.Popen(["tools/run_tests/python_utils/antagonist.py"]) 1451 for _ in range(0, args.antagonists) 1452 ] 1453 start_port_server.start_port_server() 1454 resultset = None 1455 num_test_failures = 0 1456 try: 1457 infinite_runs = runs_per_test == 0 1458 one_run = set( 1459 spec 1460 for language in languages 1461 for spec in language.test_specs() 1462 if ( 1463 re.search(args.regex, spec.shortname) 1464 and ( 1465 args.regex_exclude == "" 1466 or not re.search(args.regex_exclude, spec.shortname) 1467 ) 1468 ) 1469 ) 1470 # When running on travis, we want out test runs to be as similar as possible 1471 # for reproducibility purposes. 1472 if args.travis and args.max_time <= 0: 1473 massaged_one_run = sorted(one_run, key=lambda x: x.cpu_cost) 1474 else: 1475 # whereas otherwise, we want to shuffle things up to give all tests a 1476 # chance to run. 1477 massaged_one_run = list( 1478 one_run 1479 ) # random.sample needs an indexable seq. 1480 num_jobs = len(massaged_one_run) 1481 # for a random sample, get as many as indicated by the 'sample_percent' 1482 # argument. By default this arg is 100, resulting in a shuffle of all 1483 # jobs. 1484 sample_size = int(num_jobs * args.sample_percent / 100.0) 1485 massaged_one_run = random.sample(massaged_one_run, sample_size) 1486 if not isclose(args.sample_percent, 100.0): 1487 assert ( 1488 args.runs_per_test == 1 1489 ), "Can't do sampling (-p) over multiple runs (-n)." 1490 print( 1491 "Running %d tests out of %d (~%d%%)" 1492 % (sample_size, num_jobs, args.sample_percent) 1493 ) 1494 if infinite_runs: 1495 assert ( 1496 len(massaged_one_run) > 0 1497 ), "Must have at least one test for a -n inf run" 1498 runs_sequence = ( 1499 itertools.repeat(massaged_one_run) 1500 if infinite_runs 1501 else itertools.repeat(massaged_one_run, runs_per_test) 1502 ) 1503 all_runs = itertools.chain.from_iterable(runs_sequence) 1504 1505 if args.quiet_success: 1506 jobset.message( 1507 "START", 1508 "Running tests quietly, only failing tests will be reported", 1509 do_newline=True, 1510 ) 1511 num_test_failures, resultset = jobset.run( 1512 all_runs, 1513 check_cancelled, 1514 newline_on_success=newline_on_success, 1515 travis=args.travis, 1516 maxjobs=args.jobs, 1517 maxjobs_cpu_agnostic=max_parallel_tests_for_current_platform(), 1518 stop_on_failure=args.stop_on_failure, 1519 quiet_success=args.quiet_success, 1520 max_time=args.max_time, 1521 ) 1522 if resultset: 1523 for k, v in sorted(resultset.items()): 1524 num_runs, num_failures = _calculate_num_runs_failures(v) 1525 if num_failures > 0: 1526 if num_failures == num_runs: # what about infinite_runs??? 1527 jobset.message("FAILED", k, do_newline=True) 1528 else: 1529 jobset.message( 1530 "FLAKE", 1531 "%s [%d/%d runs flaked]" 1532 % (k, num_failures, num_runs), 1533 do_newline=True, 1534 ) 1535 finally: 1536 for antagonist in antagonists: 1537 antagonist.kill() 1538 if args.bq_result_table and resultset: 1539 upload_extra_fields = { 1540 "compiler": args.compiler, 1541 "config": args.config, 1542 "iomgr_platform": args.iomgr_platform, 1543 "language": args.language[ 1544 0 1545 ], # args.language is a list but will always have one element when uploading to BQ is enabled. 1546 "platform": platform_string(), 1547 } 1548 try: 1549 upload_results_to_bq( 1550 resultset, args.bq_result_table, upload_extra_fields 1551 ) 1552 except NameError as e: 1553 logging.warning( 1554 e 1555 ) # It's fine to ignore since this is not critical 1556 if xml_report and resultset: 1557 report_utils.render_junit_xml_report( 1558 resultset, 1559 xml_report, 1560 suite_name=args.report_suite_name, 1561 multi_target=args.report_multi_target, 1562 ) 1563 1564 number_failures, _ = jobset.run( 1565 post_tests_steps, 1566 maxjobs=1, 1567 stop_on_failure=False, 1568 newline_on_success=newline_on_success, 1569 travis=args.travis, 1570 ) 1571 1572 out = [] 1573 if number_failures: 1574 out.append(BuildAndRunError.POST_TEST) 1575 if num_test_failures: 1576 out.append(BuildAndRunError.TEST) 1577 1578 return out 1579 1580 1581# parse command line 1582argp = argparse.ArgumentParser(description="Run grpc tests.") 1583argp.add_argument( 1584 "-c", "--config", choices=sorted(_CONFIGS.keys()), default="opt" 1585) 1586argp.add_argument( 1587 "-n", 1588 "--runs_per_test", 1589 default=1, 1590 type=runs_per_test_type, 1591 help=( 1592 'A positive integer or "inf". If "inf", all tests will run in an ' 1593 'infinite loop. Especially useful in combination with "-f"' 1594 ), 1595) 1596argp.add_argument("-r", "--regex", default=".*", type=str) 1597argp.add_argument("--regex_exclude", default="", type=str) 1598argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) 1599argp.add_argument("-s", "--slowdown", default=1.0, type=float) 1600argp.add_argument( 1601 "-p", 1602 "--sample_percent", 1603 default=100.0, 1604 type=percent_type, 1605 help="Run a random sample with that percentage of tests", 1606) 1607argp.add_argument( 1608 "-t", 1609 "--travis", 1610 default=False, 1611 action="store_const", 1612 const=True, 1613 help=( 1614 "When set, indicates that the script is running on CI (= not locally)." 1615 ), 1616) 1617argp.add_argument( 1618 "--newline_on_success", default=False, action="store_const", const=True 1619) 1620argp.add_argument( 1621 "-l", 1622 "--language", 1623 choices=sorted(_LANGUAGES.keys()), 1624 nargs="+", 1625 required=True, 1626) 1627argp.add_argument( 1628 "-S", "--stop_on_failure", default=False, action="store_const", const=True 1629) 1630argp.add_argument( 1631 "--use_docker", 1632 default=False, 1633 action="store_const", 1634 const=True, 1635 help="Run all the tests under docker. That provides " 1636 + "additional isolation and prevents the need to install " 1637 + "language specific prerequisites. Only available on Linux.", 1638) 1639argp.add_argument( 1640 "--allow_flakes", 1641 default=False, 1642 action="store_const", 1643 const=True, 1644 help=( 1645 "Allow flaky tests to show as passing (re-runs failed tests up to five" 1646 " times)" 1647 ), 1648) 1649argp.add_argument( 1650 "--arch", 1651 choices=["default", "x86", "x64", "arm64"], 1652 default="default", 1653 help=( 1654 'Selects architecture to target. For some platforms "default" is the' 1655 " only supported choice." 1656 ), 1657) 1658argp.add_argument( 1659 "--compiler", 1660 choices=[ 1661 "default", 1662 "gcc8", 1663 "gcc10.2", 1664 "gcc10.2_openssl102", 1665 "gcc10.2_openssl111", 1666 "gcc12", 1667 "gcc12_openssl309", 1668 "gcc_musl", 1669 "clang6", 1670 "clang17", 1671 # TODO: Automatically populate from supported version 1672 "python3.7", 1673 "python3.8", 1674 "python3.9", 1675 "python3.10", 1676 "python3.11", 1677 "python3.12", 1678 "pypy", 1679 "pypy3", 1680 "python_alpine", 1681 "all_the_cpythons", 1682 "coreclr", 1683 "cmake", 1684 "cmake_ninja_vs2019", 1685 "cmake_ninja_vs2022", 1686 "cmake_vs2019", 1687 "cmake_vs2022", 1688 "mono", 1689 ], 1690 default="default", 1691 help=( 1692 "Selects compiler to use. Allowed values depend on the platform and" 1693 " language." 1694 ), 1695) 1696argp.add_argument( 1697 "--iomgr_platform", 1698 choices=["native", "gevent", "asyncio"], 1699 default="native", 1700 help="Selects iomgr platform to build on", 1701) 1702argp.add_argument( 1703 "--build_only", 1704 default=False, 1705 action="store_const", 1706 const=True, 1707 help="Perform all the build steps but don't run any tests.", 1708) 1709argp.add_argument( 1710 "--measure_cpu_costs", 1711 default=False, 1712 action="store_const", 1713 const=True, 1714 help="Measure the cpu costs of tests", 1715) 1716argp.add_argument("-a", "--antagonists", default=0, type=int) 1717argp.add_argument( 1718 "-x", 1719 "--xml_report", 1720 default=None, 1721 type=str, 1722 help="Generates a JUnit-compatible XML report", 1723) 1724argp.add_argument( 1725 "--report_suite_name", 1726 default="tests", 1727 type=str, 1728 help="Test suite name to use in generated JUnit XML report", 1729) 1730argp.add_argument( 1731 "--report_multi_target", 1732 default=False, 1733 const=True, 1734 action="store_const", 1735 help=( 1736 "Generate separate XML report for each test job (Looks better in UIs)." 1737 ), 1738) 1739argp.add_argument( 1740 "--quiet_success", 1741 default=False, 1742 action="store_const", 1743 const=True, 1744 help=( 1745 "Don't print anything when a test passes. Passing tests also will not" 1746 " be reported in XML report. " 1747 ) 1748 + "Useful when running many iterations of each test (argument -n).", 1749) 1750argp.add_argument( 1751 "--force_default_poller", 1752 default=False, 1753 action="store_const", 1754 const=True, 1755 help="Don't try to iterate over many polling strategies when they exist", 1756) 1757argp.add_argument( 1758 "--force_use_pollers", 1759 default=None, 1760 type=str, 1761 help=( 1762 "Only use the specified comma-delimited list of polling engines. " 1763 "Example: --force_use_pollers epoll1,poll " 1764 " (This flag has no effect if --force_default_poller flag is also used)" 1765 ), 1766) 1767argp.add_argument( 1768 "--max_time", default=-1, type=int, help="Maximum test runtime in seconds" 1769) 1770argp.add_argument( 1771 "--bq_result_table", 1772 default="", 1773 type=str, 1774 nargs="?", 1775 help="Upload test results to a specified BQ table.", 1776) 1777argp.add_argument( 1778 "--cmake_configure_extra_args", 1779 default=[], 1780 nargs="+", 1781 help="Extra arguments that will be passed to the cmake configure command. Only works for C/C++.", 1782) 1783args = argp.parse_args() 1784 1785flaky_tests = set() 1786shortname_to_cpu = {} 1787 1788if args.force_default_poller: 1789 _POLLING_STRATEGIES = {} 1790elif args.force_use_pollers: 1791 _POLLING_STRATEGIES[platform_string()] = args.force_use_pollers.split(",") 1792 1793jobset.measure_cpu_costs = args.measure_cpu_costs 1794 1795# grab config 1796run_config = _CONFIGS[args.config] 1797build_config = run_config.build_config 1798 1799languages = set(_LANGUAGES[l] for l in args.language) 1800for l in languages: 1801 l.configure(run_config, args) 1802 1803if len(languages) != 1: 1804 print("Building multiple languages simultaneously is not supported!") 1805 sys.exit(1) 1806 1807# If --use_docker was used, respawn the run_tests.py script under a docker container 1808# instead of continuing. 1809if args.use_docker: 1810 if not args.travis: 1811 print("Seen --use_docker flag, will run tests under docker.") 1812 print("") 1813 print( 1814 "IMPORTANT: The changes you are testing need to be locally" 1815 " committed" 1816 ) 1817 print( 1818 "because only the committed changes in the current branch will be" 1819 ) 1820 print("copied to the docker environment.") 1821 time.sleep(5) 1822 1823 dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) 1824 if len(dockerfile_dirs) > 1: 1825 print( 1826 "Languages to be tested require running under different docker " 1827 "images." 1828 ) 1829 sys.exit(1) 1830 else: 1831 dockerfile_dir = next(iter(dockerfile_dirs)) 1832 1833 child_argv = [arg for arg in sys.argv if not arg == "--use_docker"] 1834 run_tests_cmd = "python3 tools/run_tests/run_tests.py %s" % " ".join( 1835 child_argv[1:] 1836 ) 1837 1838 env = os.environ.copy() 1839 env["DOCKERFILE_DIR"] = dockerfile_dir 1840 env["DOCKER_RUN_SCRIPT"] = "tools/run_tests/dockerize/docker_run.sh" 1841 env["DOCKER_RUN_SCRIPT_COMMAND"] = run_tests_cmd 1842 1843 retcode = subprocess.call( 1844 "tools/run_tests/dockerize/build_and_run_docker.sh", shell=True, env=env 1845 ) 1846 _print_debug_info_epilogue(dockerfile_dir=dockerfile_dir) 1847 sys.exit(retcode) 1848 1849_check_arch_option(args.arch) 1850 1851# collect pre-build steps (which get retried if they fail, e.g. to avoid 1852# flakes on downloading dependencies etc.) 1853build_steps = list( 1854 set( 1855 jobset.JobSpec( 1856 cmdline, 1857 environ=_build_step_environ( 1858 build_config, extra_env=l.build_steps_environ() 1859 ), 1860 timeout_seconds=_PRE_BUILD_STEP_TIMEOUT_SECONDS, 1861 flake_retries=2, 1862 ) 1863 for l in languages 1864 for cmdline in l.pre_build_steps() 1865 ) 1866) 1867 1868# collect build steps 1869build_steps.extend( 1870 set( 1871 jobset.JobSpec( 1872 cmdline, 1873 environ=_build_step_environ( 1874 build_config, extra_env=l.build_steps_environ() 1875 ), 1876 timeout_seconds=None, 1877 ) 1878 for l in languages 1879 for cmdline in l.build_steps() 1880 ) 1881) 1882 1883# collect post test steps 1884post_tests_steps = list( 1885 set( 1886 jobset.JobSpec( 1887 cmdline, 1888 environ=_build_step_environ( 1889 build_config, extra_env=l.build_steps_environ() 1890 ), 1891 ) 1892 for l in languages 1893 for cmdline in l.post_tests_steps() 1894 ) 1895) 1896runs_per_test = args.runs_per_test 1897 1898errors = _build_and_run( 1899 check_cancelled=lambda: False, 1900 newline_on_success=args.newline_on_success, 1901 xml_report=args.xml_report, 1902 build_only=args.build_only, 1903) 1904if not errors: 1905 jobset.message("SUCCESS", "All tests passed", do_newline=True) 1906else: 1907 jobset.message("FAILED", "Some tests failed", do_newline=True) 1908 1909if not _is_use_docker_child(): 1910 # if --use_docker was used, the outer invocation of run_tests.py will 1911 # print the debug info instead. 1912 _print_debug_info_epilogue() 1913 1914exit_code = 0 1915if BuildAndRunError.BUILD in errors: 1916 exit_code |= 1 1917if BuildAndRunError.TEST in errors: 1918 exit_code |= 2 1919if BuildAndRunError.POST_TEST in errors: 1920 exit_code |= 4 1921sys.exit(exit_code) 1922