1#!/usr/bin/env python3 2# Copyright 2015 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Run test matrix.""" 16 17from __future__ import print_function 18 19import argparse 20import multiprocessing 21import os 22import sys 23 24from python_utils.filter_pull_request_tests import filter_tests 25import python_utils.jobset as jobset 26import python_utils.report_utils as report_utils 27 28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 29os.chdir(_ROOT) 30 31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60 32 33# C/C++ tests can take long time 34_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60 35 36# Set timeout high for ObjC for Cocoapods to install pods 37_OBJC_RUNTESTS_TIMEOUT = 2 * 60 * 60 38 39# Number of jobs assigned to each run_tests.py instance 40_DEFAULT_INNER_JOBS = 2 41 42# Name of the top-level umbrella report that includes all the run_tests.py invocations 43# Note that the starting letter 't' matters so that the targets are listed AFTER 44# the per-test breakdown items that start with 'run_tests/' (it is more readable that way) 45_MATRIX_REPORT_NAME = "toplevel_run_tests_invocations" 46 47 48def _safe_report_name(name): 49 """Reports with '+' in target name won't show correctly in ResultStore""" 50 return name.replace("+", "p") 51 52 53def _report_filename(name): 54 """Generates report file name with directory structure that leads to better presentation by internal CI""" 55 # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. 56 return "%s/%s" % (_safe_report_name(name), "sponge_log.xml") 57 58 59def _matrix_job_logfilename(shortname_for_multi_target): 60 """Generate location for log file that will match the sponge_log.xml from the top-level matrix report.""" 61 # 'sponge_log.log' suffix must be there for log to get recognized as "target log" 62 # for the corresponding 'sponge_log.xml' report. 63 # the shortname_for_multi_target component must be set to match the sponge_log.xml location 64 # because the top-level render_junit_xml_report is called with multi_target=True 65 sponge_log_name = "%s/%s/%s" % ( 66 _MATRIX_REPORT_NAME, 67 shortname_for_multi_target, 68 "sponge_log.log", 69 ) 70 # env variable can be used to override the base location for the reports 71 # so we need to match that behavior here too 72 base_dir = os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) 73 if base_dir: 74 sponge_log_name = os.path.join(base_dir, sponge_log_name) 75 return sponge_log_name 76 77 78def _docker_jobspec( 79 name, 80 runtests_args=[], 81 runtests_envs={}, 82 inner_jobs=_DEFAULT_INNER_JOBS, 83 timeout_seconds=None, 84): 85 """Run a single instance of run_tests.py in a docker container""" 86 if not timeout_seconds: 87 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 88 shortname = "run_tests_%s" % name 89 test_job = jobset.JobSpec( 90 cmdline=[ 91 "python3", 92 "tools/run_tests/run_tests.py", 93 "--use_docker", 94 "-t", 95 "-j", 96 str(inner_jobs), 97 "-x", 98 "run_tests/%s" % _report_filename(name), 99 "--report_suite_name", 100 "%s" % _safe_report_name(name), 101 ] 102 + runtests_args, 103 environ=runtests_envs, 104 shortname=shortname, 105 timeout_seconds=timeout_seconds, 106 logfilename=_matrix_job_logfilename(shortname), 107 ) 108 return test_job 109 110 111def _workspace_jobspec( 112 name, 113 runtests_args=[], 114 workspace_name=None, 115 runtests_envs={}, 116 inner_jobs=_DEFAULT_INNER_JOBS, 117 timeout_seconds=None, 118): 119 """Run a single instance of run_tests.py in a separate workspace""" 120 if not workspace_name: 121 workspace_name = "workspace_%s" % name 122 if not timeout_seconds: 123 timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT 124 shortname = "run_tests_%s" % name 125 env = {"WORKSPACE_NAME": workspace_name} 126 env.update(runtests_envs) 127 # if report base dir is set, we don't need to ".." to come out of the workspace dir 128 report_dir_prefix = ( 129 "" if os.getenv("GRPC_TEST_REPORT_BASE_DIR", None) else "../" 130 ) 131 test_job = jobset.JobSpec( 132 cmdline=[ 133 "bash", 134 "tools/run_tests/helper_scripts/run_tests_in_workspace.sh", 135 "-t", 136 "-j", 137 str(inner_jobs), 138 "-x", 139 "%srun_tests/%s" % (report_dir_prefix, _report_filename(name)), 140 "--report_suite_name", 141 "%s" % _safe_report_name(name), 142 ] 143 + runtests_args, 144 environ=env, 145 shortname=shortname, 146 timeout_seconds=timeout_seconds, 147 logfilename=_matrix_job_logfilename(shortname), 148 ) 149 return test_job 150 151 152def _generate_jobs( 153 languages, 154 configs, 155 platforms, 156 iomgr_platforms=["native"], 157 arch=None, 158 compiler=None, 159 labels=[], 160 extra_args=[], 161 extra_envs={}, 162 inner_jobs=_DEFAULT_INNER_JOBS, 163 timeout_seconds=None, 164): 165 result = [] 166 for language in languages: 167 for platform in platforms: 168 for iomgr_platform in iomgr_platforms: 169 for config in configs: 170 name = "%s_%s_%s_%s" % ( 171 language, 172 platform, 173 config, 174 iomgr_platform, 175 ) 176 runtests_args = [ 177 "-l", 178 language, 179 "-c", 180 config, 181 "--iomgr_platform", 182 iomgr_platform, 183 ] 184 if arch or compiler: 185 name += "_%s_%s" % (arch, compiler) 186 runtests_args += [ 187 "--arch", 188 arch, 189 "--compiler", 190 compiler, 191 ] 192 if "--build_only" in extra_args: 193 name += "_buildonly" 194 for extra_env in extra_envs: 195 name += "_%s_%s" % (extra_env, extra_envs[extra_env]) 196 197 runtests_args += extra_args 198 if platform == "linux": 199 job = _docker_jobspec( 200 name=name, 201 runtests_args=runtests_args, 202 runtests_envs=extra_envs, 203 inner_jobs=inner_jobs, 204 timeout_seconds=timeout_seconds, 205 ) 206 else: 207 job = _workspace_jobspec( 208 name=name, 209 runtests_args=runtests_args, 210 runtests_envs=extra_envs, 211 inner_jobs=inner_jobs, 212 timeout_seconds=timeout_seconds, 213 ) 214 215 job.labels = [ 216 platform, 217 config, 218 language, 219 iomgr_platform, 220 ] + labels 221 result.append(job) 222 return result 223 224 225def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): 226 test_jobs = [] 227 # sanity tests 228 test_jobs += _generate_jobs( 229 languages=["sanity", "clang-tidy"], 230 configs=["dbg"], 231 platforms=["linux"], 232 labels=["basictests"], 233 extra_args=extra_args + ["--report_multi_target"], 234 inner_jobs=inner_jobs, 235 ) 236 237 # supported on all platforms. 238 test_jobs += _generate_jobs( 239 languages=["c"], 240 configs=["dbg", "opt"], 241 platforms=["linux", "macos", "windows"], 242 labels=["basictests", "corelang"], 243 extra_args=extra_args, # don't use multi_target report because C has too many test cases 244 inner_jobs=inner_jobs, 245 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 246 ) 247 248 # C# tests (both on .NET desktop/mono and .NET core) 249 test_jobs += _generate_jobs( 250 languages=["csharp"], 251 configs=["dbg", "opt"], 252 platforms=["linux", "macos", "windows"], 253 labels=["basictests", "multilang"], 254 extra_args=extra_args + ["--report_multi_target"], 255 inner_jobs=inner_jobs, 256 ) 257 258 # ARM64 Linux C# tests 259 test_jobs += _generate_jobs( 260 languages=["csharp"], 261 configs=["dbg", "opt"], 262 platforms=["linux"], 263 arch="arm64", 264 compiler="default", 265 labels=["basictests_arm64"], 266 extra_args=extra_args + ["--report_multi_target"], 267 inner_jobs=inner_jobs, 268 ) 269 270 test_jobs += _generate_jobs( 271 languages=["python"], 272 configs=["opt"], 273 platforms=["linux", "macos", "windows"], 274 iomgr_platforms=["native"], 275 labels=["basictests", "multilang"], 276 extra_args=extra_args + ["--report_multi_target"], 277 inner_jobs=inner_jobs, 278 ) 279 280 # ARM64 Linux Python tests 281 test_jobs += _generate_jobs( 282 languages=["python"], 283 configs=["opt"], 284 platforms=["linux"], 285 arch="arm64", 286 compiler="default", 287 iomgr_platforms=["native"], 288 labels=["basictests_arm64"], 289 extra_args=extra_args + ["--report_multi_target"], 290 inner_jobs=inner_jobs, 291 ) 292 293 # supported on linux and mac. 294 test_jobs += _generate_jobs( 295 languages=["c++"], 296 configs=["dbg", "opt"], 297 platforms=["linux", "macos"], 298 labels=["basictests", "corelang"], 299 extra_args=extra_args, # don't use multi_target report because C++ has too many test cases 300 inner_jobs=inner_jobs, 301 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 302 ) 303 304 test_jobs += _generate_jobs( 305 languages=["ruby", "php7"], 306 configs=["dbg", "opt"], 307 platforms=["linux", "macos"], 308 labels=["basictests", "multilang"], 309 extra_args=extra_args + ["--report_multi_target"], 310 inner_jobs=inner_jobs, 311 ) 312 313 # ARM64 Linux Ruby and PHP tests 314 test_jobs += _generate_jobs( 315 languages=["ruby", "php7"], 316 configs=["dbg", "opt"], 317 platforms=["linux"], 318 arch="arm64", 319 compiler="default", 320 labels=["basictests_arm64"], 321 extra_args=extra_args + ["--report_multi_target"], 322 inner_jobs=inner_jobs, 323 ) 324 325 # supported on mac only. 326 test_jobs += _generate_jobs( 327 languages=["objc"], 328 configs=["opt"], 329 platforms=["macos"], 330 labels=["basictests", "multilang"], 331 extra_args=extra_args + ["--report_multi_target"], 332 inner_jobs=inner_jobs, 333 timeout_seconds=_OBJC_RUNTESTS_TIMEOUT, 334 ) 335 336 return test_jobs 337 338 339def _create_portability_test_jobs( 340 extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS 341): 342 test_jobs = [] 343 # portability C x86 344 test_jobs += _generate_jobs( 345 languages=["c"], 346 configs=["dbg"], 347 platforms=["linux"], 348 arch="x86", 349 compiler="default", 350 labels=["portability", "corelang"], 351 extra_args=extra_args, 352 inner_jobs=inner_jobs, 353 ) 354 355 # portability C and C++ on x64 356 for compiler in [ 357 "gcc8", 358 # TODO(b/283304471): Tests using OpenSSL's engine APIs were broken and removed 359 "gcc10.2_openssl102", 360 "gcc10.2_openssl111", 361 "gcc12", 362 "gcc12_openssl309", 363 "gcc_musl", 364 "clang6", 365 "clang17", 366 ]: 367 test_jobs += _generate_jobs( 368 languages=["c", "c++"], 369 configs=["dbg"], 370 platforms=["linux"], 371 arch="x64", 372 compiler=compiler, 373 labels=["portability", "corelang"] 374 + (["openssl"] if "openssl" in compiler else []), 375 extra_args=extra_args, 376 inner_jobs=inner_jobs, 377 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 378 ) 379 380 # portability C & C++ on Windows 64-bit 381 test_jobs += _generate_jobs( 382 languages=["c", "c++"], 383 configs=["dbg"], 384 platforms=["windows"], 385 arch="default", 386 compiler="cmake_ninja_vs2019", 387 labels=["portability", "corelang"], 388 extra_args=extra_args, 389 inner_jobs=inner_jobs, 390 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 391 ) 392 393 # portability C and C++ on Windows with the "Visual Studio 2022" cmake 394 # generator, i.e. not using Ninja (to verify that we can still build with msbuild) 395 test_jobs += _generate_jobs( 396 languages=["c", "c++"], 397 configs=["dbg"], 398 platforms=["windows"], 399 arch="x64", 400 compiler="cmake_vs2022", 401 labels=["portability", "corelang"], 402 extra_args=extra_args, 403 inner_jobs=inner_jobs, 404 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 405 ) 406 407 # C and C++ with no-exceptions on Linux 408 test_jobs += _generate_jobs( 409 languages=["c", "c++"], 410 configs=["noexcept"], 411 platforms=["linux"], 412 labels=["portability", "corelang"], 413 extra_args=extra_args, 414 inner_jobs=inner_jobs, 415 timeout_seconds=_CPP_RUNTESTS_TIMEOUT, 416 ) 417 418 test_jobs += _generate_jobs( 419 languages=["python"], 420 configs=["dbg"], 421 platforms=["linux"], 422 arch="default", 423 compiler="python_alpine", 424 labels=["portability", "multilang"], 425 extra_args=extra_args + ["--report_multi_target"], 426 inner_jobs=inner_jobs, 427 ) 428 429 return test_jobs 430 431 432def _allowed_labels(): 433 """Returns a list of existing job labels.""" 434 all_labels = set() 435 for job in _create_test_jobs() + _create_portability_test_jobs(): 436 for label in job.labels: 437 all_labels.add(label) 438 return sorted(all_labels) 439 440 441def _runs_per_test_type(arg_str): 442 """Auxiliary function to parse the "runs_per_test" flag.""" 443 try: 444 n = int(arg_str) 445 if n <= 0: 446 raise ValueError 447 return n 448 except: 449 msg = "'{}' is not a positive integer".format(arg_str) 450 raise argparse.ArgumentTypeError(msg) 451 452 453if __name__ == "__main__": 454 argp = argparse.ArgumentParser( 455 description="Run a matrix of run_tests.py tests." 456 ) 457 argp.add_argument( 458 "-j", 459 "--jobs", 460 default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS, 461 type=int, 462 help="Number of concurrent run_tests.py instances.", 463 ) 464 argp.add_argument( 465 "-f", 466 "--filter", 467 choices=_allowed_labels(), 468 nargs="+", 469 default=[], 470 help="Filter targets to run by label with AND semantics.", 471 ) 472 argp.add_argument( 473 "--exclude", 474 choices=_allowed_labels(), 475 nargs="+", 476 default=[], 477 help="Exclude targets with any of given labels.", 478 ) 479 argp.add_argument( 480 "--build_only", 481 default=False, 482 action="store_const", 483 const=True, 484 help="Pass --build_only flag to run_tests.py instances.", 485 ) 486 argp.add_argument( 487 "--force_default_poller", 488 default=False, 489 action="store_const", 490 const=True, 491 help="Pass --force_default_poller to run_tests.py instances.", 492 ) 493 argp.add_argument( 494 "--dry_run", 495 default=False, 496 action="store_const", 497 const=True, 498 help="Only print what would be run.", 499 ) 500 argp.add_argument( 501 "--filter_pr_tests", 502 default=False, 503 action="store_const", 504 const=True, 505 help="Filters out tests irrelevant to pull request changes.", 506 ) 507 argp.add_argument( 508 "--base_branch", 509 default="origin/master", 510 type=str, 511 help="Branch that pull request is requesting to merge into", 512 ) 513 argp.add_argument( 514 "--inner_jobs", 515 default=_DEFAULT_INNER_JOBS, 516 type=int, 517 help="Number of jobs in each run_tests.py instance", 518 ) 519 argp.add_argument( 520 "-n", 521 "--runs_per_test", 522 default=1, 523 type=_runs_per_test_type, 524 help="How many times to run each tests. >1 runs implies " 525 + "omitting passing test from the output & reports.", 526 ) 527 argp.add_argument( 528 "--max_time", 529 default=-1, 530 type=int, 531 help="Maximum amount of time to run tests for" 532 + "(other tests will be skipped)", 533 ) 534 argp.add_argument( 535 "--internal_ci", 536 default=False, 537 action="store_const", 538 const=True, 539 help=( 540 "(Deprecated, has no effect) Put reports into subdirectories to" 541 " improve presentation of results by Kokoro." 542 ), 543 ) 544 argp.add_argument( 545 "--bq_result_table", 546 default="", 547 type=str, 548 nargs="?", 549 help="Upload test results to a specified BQ table.", 550 ) 551 argp.add_argument( 552 "--extra_args", 553 default="", 554 type=str, 555 nargs=argparse.REMAINDER, 556 help="Extra test args passed to each sub-script.", 557 ) 558 args = argp.parse_args() 559 560 extra_args = [] 561 if args.build_only: 562 extra_args.append("--build_only") 563 if args.force_default_poller: 564 extra_args.append("--force_default_poller") 565 if args.runs_per_test > 1: 566 extra_args.append("-n") 567 extra_args.append("%s" % args.runs_per_test) 568 extra_args.append("--quiet_success") 569 if args.max_time > 0: 570 extra_args.extend(("--max_time", "%d" % args.max_time)) 571 if args.bq_result_table: 572 extra_args.append("--bq_result_table") 573 extra_args.append("%s" % args.bq_result_table) 574 extra_args.append("--measure_cpu_costs") 575 if args.extra_args: 576 extra_args.extend(args.extra_args) 577 578 all_jobs = _create_test_jobs( 579 extra_args=extra_args, inner_jobs=args.inner_jobs 580 ) + _create_portability_test_jobs( 581 extra_args=extra_args, inner_jobs=args.inner_jobs 582 ) 583 584 jobs = [] 585 for job in all_jobs: 586 if not args.filter or all( 587 filter in job.labels for filter in args.filter 588 ): 589 if not any( 590 exclude_label in job.labels for exclude_label in args.exclude 591 ): 592 jobs.append(job) 593 594 if not jobs: 595 jobset.message( 596 "FAILED", "No test suites match given criteria.", do_newline=True 597 ) 598 sys.exit(1) 599 600 print("IMPORTANT: The changes you are testing need to be locally committed") 601 print("because only the committed changes in the current branch will be") 602 print("copied to the docker environment or into subworkspaces.") 603 604 skipped_jobs = [] 605 606 if args.filter_pr_tests: 607 print("Looking for irrelevant tests to skip...") 608 relevant_jobs = filter_tests(jobs, args.base_branch) 609 if len(relevant_jobs) == len(jobs): 610 print("No tests will be skipped.") 611 else: 612 print("These tests will be skipped:") 613 skipped_jobs = list(set(jobs) - set(relevant_jobs)) 614 # Sort by shortnames to make printing of skipped tests consistent 615 skipped_jobs.sort(key=lambda job: job.shortname) 616 for job in list(skipped_jobs): 617 print(" %s" % job.shortname) 618 jobs = relevant_jobs 619 620 print("Will run these tests:") 621 for job in jobs: 622 print(' %s: "%s"' % (job.shortname, " ".join(job.cmdline))) 623 print("") 624 625 if args.dry_run: 626 print("--dry_run was used, exiting") 627 sys.exit(1) 628 629 jobset.message("START", "Running test matrix.", do_newline=True) 630 num_failures, resultset = jobset.run( 631 jobs, newline_on_success=True, travis=True, maxjobs=args.jobs 632 ) 633 # Merge skipped tests into results to show skipped tests on report.xml 634 if skipped_jobs: 635 ignored_num_skipped_failures, skipped_results = jobset.run( 636 skipped_jobs, skip_jobs=True 637 ) 638 resultset.update(skipped_results) 639 report_utils.render_junit_xml_report( 640 resultset, 641 _report_filename(_MATRIX_REPORT_NAME), 642 suite_name=_MATRIX_REPORT_NAME, 643 multi_target=True, 644 ) 645 646 if num_failures == 0: 647 jobset.message( 648 "SUCCESS", 649 "All run_tests.py instances finished successfully.", 650 do_newline=True, 651 ) 652 else: 653 jobset.message( 654 "FAILED", 655 "Some run_tests.py instances have failed.", 656 do_newline=True, 657 ) 658 sys.exit(1) 659