1#!/usr/bin/env python3
2# Copyright 2015 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Run test matrix."""
16
17from __future__ import print_function
18
19import argparse
20import multiprocessing
21import os
22import sys
23
24from python_utils.filter_pull_request_tests import filter_tests
25import python_utils.jobset as jobset
26import python_utils.report_utils as report_utils
27
28_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
29os.chdir(_ROOT)
30
31_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60
32
33# C/C++ tests can take long time
34_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60
35
36# Set timeout high for ObjC for Cocoapods to install pods
37_OBJC_RUNTESTS_TIMEOUT = 2 * 60 * 60
38
39# Number of jobs assigned to each run_tests.py instance
40_DEFAULT_INNER_JOBS = 2
41
42# Name of the top-level umbrella report that includes all the run_tests.py invocations
43# Note that the starting letter 't' matters so that the targets are listed AFTER
44# the per-test breakdown items that start with 'run_tests/' (it is more readable that way)
45_MATRIX_REPORT_NAME = 'toplevel_run_tests_invocations'
46
47
48def _safe_report_name(name):
49    """Reports with '+' in target name won't show correctly in ResultStore"""
50    return name.replace('+', 'p')
51
52
53def _report_filename(name):
54    """Generates report file name with directory structure that leads to better presentation by internal CI"""
55    # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro.
56    return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml')
57
58
59def _matrix_job_logfilename(shortname_for_multi_target):
60    """Generate location for log file that will match the sponge_log.xml from the top-level matrix report."""
61    # 'sponge_log.log' suffix must be there for log to get recognized as "target log"
62    # for the corresponding 'sponge_log.xml' report.
63    # the shortname_for_multi_target component must be set to match the sponge_log.xml location
64    # because the top-level render_junit_xml_report is called with multi_target=True
65    sponge_log_name = '%s/%s/%s' % (
66        _MATRIX_REPORT_NAME, shortname_for_multi_target, 'sponge_log.log')
67    # env variable can be used to override the base location for the reports
68    # so we need to match that behavior here too
69    base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None)
70    if base_dir:
71        sponge_log_name = os.path.join(base_dir, sponge_log_name)
72    return sponge_log_name
73
74
75def _docker_jobspec(name,
76                    runtests_args=[],
77                    runtests_envs={},
78                    inner_jobs=_DEFAULT_INNER_JOBS,
79                    timeout_seconds=None):
80    """Run a single instance of run_tests.py in a docker container"""
81    if not timeout_seconds:
82        timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
83    shortname = 'run_tests_%s' % name
84    test_job = jobset.JobSpec(cmdline=[
85        'python3', 'tools/run_tests/run_tests.py', '--use_docker', '-t', '-j',
86        str(inner_jobs), '-x',
87        'run_tests/%s' % _report_filename(name), '--report_suite_name',
88        '%s' % _safe_report_name(name)
89    ] + runtests_args,
90                              environ=runtests_envs,
91                              shortname=shortname,
92                              timeout_seconds=timeout_seconds,
93                              logfilename=_matrix_job_logfilename(shortname))
94    return test_job
95
96
97def _workspace_jobspec(name,
98                       runtests_args=[],
99                       workspace_name=None,
100                       runtests_envs={},
101                       inner_jobs=_DEFAULT_INNER_JOBS,
102                       timeout_seconds=None):
103    """Run a single instance of run_tests.py in a separate workspace"""
104    if not workspace_name:
105        workspace_name = 'workspace_%s' % name
106    if not timeout_seconds:
107        timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
108    shortname = 'run_tests_%s' % name
109    env = {'WORKSPACE_NAME': workspace_name}
110    env.update(runtests_envs)
111    # if report base dir is set, we don't need to ".." to come out of the workspace dir
112    report_dir_prefix = '' if os.getenv('GRPC_TEST_REPORT_BASE_DIR',
113                                        None) else '../'
114    test_job = jobset.JobSpec(cmdline=[
115        'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
116        '-t', '-j',
117        str(inner_jobs), '-x',
118        '%srun_tests/%s' %
119        (report_dir_prefix, _report_filename(name)), '--report_suite_name',
120        '%s' % _safe_report_name(name)
121    ] + runtests_args,
122                              environ=env,
123                              shortname=shortname,
124                              timeout_seconds=timeout_seconds,
125                              logfilename=_matrix_job_logfilename(shortname))
126    return test_job
127
128
129def _generate_jobs(languages,
130                   configs,
131                   platforms,
132                   iomgr_platforms=['native'],
133                   arch=None,
134                   compiler=None,
135                   labels=[],
136                   extra_args=[],
137                   extra_envs={},
138                   inner_jobs=_DEFAULT_INNER_JOBS,
139                   timeout_seconds=None):
140    result = []
141    for language in languages:
142        for platform in platforms:
143            for iomgr_platform in iomgr_platforms:
144                for config in configs:
145                    name = '%s_%s_%s_%s' % (language, platform, config,
146                                            iomgr_platform)
147                    runtests_args = [
148                        '-l', language, '-c', config, '--iomgr_platform',
149                        iomgr_platform
150                    ]
151                    if arch or compiler:
152                        name += '_%s_%s' % (arch, compiler)
153                        runtests_args += [
154                            '--arch', arch, '--compiler', compiler
155                        ]
156                    if '--build_only' in extra_args:
157                        name += '_buildonly'
158                    for extra_env in extra_envs:
159                        name += '_%s_%s' % (extra_env, extra_envs[extra_env])
160
161                    runtests_args += extra_args
162                    if platform == 'linux':
163                        job = _docker_jobspec(name=name,
164                                              runtests_args=runtests_args,
165                                              runtests_envs=extra_envs,
166                                              inner_jobs=inner_jobs,
167                                              timeout_seconds=timeout_seconds)
168                    else:
169                        job = _workspace_jobspec(
170                            name=name,
171                            runtests_args=runtests_args,
172                            runtests_envs=extra_envs,
173                            inner_jobs=inner_jobs,
174                            timeout_seconds=timeout_seconds)
175
176                    job.labels = [platform, config, language, iomgr_platform
177                                 ] + labels
178                    result.append(job)
179    return result
180
181
182def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
183    test_jobs = []
184    # sanity tests
185    test_jobs += _generate_jobs(languages=['sanity', 'clang-tidy', 'iwyu'],
186                                configs=['dbg'],
187                                platforms=['linux'],
188                                labels=['basictests'],
189                                extra_args=extra_args +
190                                ['--report_multi_target'],
191                                inner_jobs=inner_jobs)
192
193    # supported on all platforms.
194    test_jobs += _generate_jobs(
195        languages=['c'],
196        configs=['dbg', 'opt'],
197        platforms=['linux', 'macos', 'windows'],
198        labels=['basictests', 'corelang'],
199        extra_args=
200        extra_args,  # don't use multi_target report because C has too many test cases
201        inner_jobs=inner_jobs,
202        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
203
204    # C# tests (both on .NET desktop/mono and .NET core)
205    test_jobs += _generate_jobs(languages=['csharp'],
206                                configs=['dbg', 'opt'],
207                                platforms=['linux', 'macos', 'windows'],
208                                labels=['basictests', 'multilang'],
209                                extra_args=extra_args +
210                                ['--report_multi_target'],
211                                inner_jobs=inner_jobs)
212
213    # ARM64 Linux C# tests
214    test_jobs += _generate_jobs(languages=['csharp'],
215                                configs=['dbg', 'opt'],
216                                platforms=['linux'],
217                                arch='arm64',
218                                compiler='default',
219                                labels=['basictests_arm64'],
220                                extra_args=extra_args +
221                                ['--report_multi_target'],
222                                inner_jobs=inner_jobs)
223
224    test_jobs += _generate_jobs(languages=['python'],
225                                configs=['opt'],
226                                platforms=['linux', 'macos', 'windows'],
227                                iomgr_platforms=['native'],
228                                labels=['basictests', 'multilang'],
229                                extra_args=extra_args +
230                                ['--report_multi_target'],
231                                inner_jobs=inner_jobs)
232
233    # ARM64 Linux Python tests
234    test_jobs += _generate_jobs(languages=['python'],
235                                configs=['opt'],
236                                platforms=['linux'],
237                                arch='arm64',
238                                compiler='default',
239                                iomgr_platforms=['native'],
240                                labels=['basictests_arm64'],
241                                extra_args=extra_args +
242                                ['--report_multi_target'],
243                                inner_jobs=inner_jobs)
244
245    # supported on linux and mac.
246    test_jobs += _generate_jobs(
247        languages=['c++'],
248        configs=['dbg', 'opt'],
249        platforms=['linux', 'macos'],
250        labels=['basictests', 'corelang'],
251        extra_args=
252        extra_args,  # don't use multi_target report because C++ has too many test cases
253        inner_jobs=inner_jobs,
254        timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
255
256    test_jobs += _generate_jobs(languages=['ruby', 'php7'],
257                                configs=['dbg', 'opt'],
258                                platforms=['linux', 'macos'],
259                                labels=['basictests', 'multilang'],
260                                extra_args=extra_args +
261                                ['--report_multi_target'],
262                                inner_jobs=inner_jobs)
263
264    # ARM64 Linux Ruby and PHP tests
265    test_jobs += _generate_jobs(languages=['ruby', 'php7'],
266                                configs=['dbg', 'opt'],
267                                platforms=['linux'],
268                                arch='arm64',
269                                compiler='default',
270                                labels=['basictests_arm64'],
271                                extra_args=extra_args +
272                                ['--report_multi_target'],
273                                inner_jobs=inner_jobs)
274
275    # supported on mac only.
276    test_jobs += _generate_jobs(languages=['objc'],
277                                configs=['opt'],
278                                platforms=['macos'],
279                                labels=['basictests', 'multilang'],
280                                extra_args=extra_args +
281                                ['--report_multi_target'],
282                                inner_jobs=inner_jobs,
283                                timeout_seconds=_OBJC_RUNTESTS_TIMEOUT)
284
285    return test_jobs
286
287
288def _create_portability_test_jobs(extra_args=[],
289                                  inner_jobs=_DEFAULT_INNER_JOBS):
290    test_jobs = []
291    # portability C x86
292    test_jobs += _generate_jobs(languages=['c'],
293                                configs=['dbg'],
294                                platforms=['linux'],
295                                arch='x86',
296                                compiler='default',
297                                labels=['portability', 'corelang'],
298                                extra_args=extra_args,
299                                inner_jobs=inner_jobs)
300
301    # portability C and C++ on x64
302    for compiler in [
303            'gcc7',
304            # 'gcc10.2_openssl102', // TODO(b/283304471): Enable this later
305            'gcc12',
306            'gcc_musl',
307            'clang6',
308            'clang15'
309    ]:
310        test_jobs += _generate_jobs(languages=['c', 'c++'],
311                                    configs=['dbg'],
312                                    platforms=['linux'],
313                                    arch='x64',
314                                    compiler=compiler,
315                                    labels=['portability', 'corelang'],
316                                    extra_args=extra_args,
317                                    inner_jobs=inner_jobs,
318                                    timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
319
320    # portability C on Windows 64-bit (x86 is the default)
321    test_jobs += _generate_jobs(languages=['c'],
322                                configs=['dbg'],
323                                platforms=['windows'],
324                                arch='x64',
325                                compiler='default',
326                                labels=['portability', 'corelang'],
327                                extra_args=extra_args,
328                                inner_jobs=inner_jobs)
329
330    # portability C on Windows with the "Visual Studio" cmake
331    # generator, i.e. not using Ninja (to verify that we can still build with msbuild)
332    test_jobs += _generate_jobs(languages=['c'],
333                                configs=['dbg'],
334                                platforms=['windows'],
335                                arch='default',
336                                compiler='cmake_vs2019',
337                                labels=['portability', 'corelang'],
338                                extra_args=extra_args,
339                                inner_jobs=inner_jobs)
340
341    # portability C++ on Windows
342    # TODO(jtattermusch): some of the tests are failing, so we force --build_only
343    test_jobs += _generate_jobs(languages=['c++'],
344                                configs=['dbg'],
345                                platforms=['windows'],
346                                arch='default',
347                                compiler='default',
348                                labels=['portability', 'corelang'],
349                                extra_args=extra_args + ['--build_only'],
350                                inner_jobs=inner_jobs,
351                                timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
352
353    # portability C and C++ on Windows using VS2019 (build only)
354    # TODO(jtattermusch): The C tests with exactly the same config are already running as part of the
355    # basictests_c suite (so we force --build_only to avoid running them twice).
356    # The C++ tests aren't all passing, so also force --build_only.
357    test_jobs += _generate_jobs(languages=['c', 'c++'],
358                                configs=['dbg'],
359                                platforms=['windows'],
360                                arch='x64',
361                                compiler='cmake_ninja_vs2019',
362                                labels=['portability', 'corelang'],
363                                extra_args=extra_args + ['--build_only'],
364                                inner_jobs=inner_jobs,
365                                timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
366
367    # C and C++ with no-exceptions on Linux
368    test_jobs += _generate_jobs(languages=['c', 'c++'],
369                                configs=['noexcept'],
370                                platforms=['linux'],
371                                labels=['portability', 'corelang'],
372                                extra_args=extra_args,
373                                inner_jobs=inner_jobs,
374                                timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
375
376    test_jobs += _generate_jobs(languages=['python'],
377                                configs=['dbg'],
378                                platforms=['linux'],
379                                arch='default',
380                                compiler='python_alpine',
381                                labels=['portability', 'multilang'],
382                                extra_args=extra_args +
383                                ['--report_multi_target'],
384                                inner_jobs=inner_jobs)
385
386    return test_jobs
387
388
389def _allowed_labels():
390    """Returns a list of existing job labels."""
391    all_labels = set()
392    for job in _create_test_jobs() + _create_portability_test_jobs():
393        for label in job.labels:
394            all_labels.add(label)
395    return sorted(all_labels)
396
397
398def _runs_per_test_type(arg_str):
399    """Auxiliary function to parse the "runs_per_test" flag."""
400    try:
401        n = int(arg_str)
402        if n <= 0:
403            raise ValueError
404        return n
405    except:
406        msg = '\'{}\' is not a positive integer'.format(arg_str)
407        raise argparse.ArgumentTypeError(msg)
408
409
410if __name__ == "__main__":
411    argp = argparse.ArgumentParser(
412        description='Run a matrix of run_tests.py tests.')
413    argp.add_argument('-j',
414                      '--jobs',
415                      default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS,
416                      type=int,
417                      help='Number of concurrent run_tests.py instances.')
418    argp.add_argument('-f',
419                      '--filter',
420                      choices=_allowed_labels(),
421                      nargs='+',
422                      default=[],
423                      help='Filter targets to run by label with AND semantics.')
424    argp.add_argument('--exclude',
425                      choices=_allowed_labels(),
426                      nargs='+',
427                      default=[],
428                      help='Exclude targets with any of given labels.')
429    argp.add_argument('--build_only',
430                      default=False,
431                      action='store_const',
432                      const=True,
433                      help='Pass --build_only flag to run_tests.py instances.')
434    argp.add_argument(
435        '--force_default_poller',
436        default=False,
437        action='store_const',
438        const=True,
439        help='Pass --force_default_poller to run_tests.py instances.')
440    argp.add_argument('--dry_run',
441                      default=False,
442                      action='store_const',
443                      const=True,
444                      help='Only print what would be run.')
445    argp.add_argument(
446        '--filter_pr_tests',
447        default=False,
448        action='store_const',
449        const=True,
450        help='Filters out tests irrelevant to pull request changes.')
451    argp.add_argument(
452        '--base_branch',
453        default='origin/master',
454        type=str,
455        help='Branch that pull request is requesting to merge into')
456    argp.add_argument('--inner_jobs',
457                      default=_DEFAULT_INNER_JOBS,
458                      type=int,
459                      help='Number of jobs in each run_tests.py instance')
460    argp.add_argument(
461        '-n',
462        '--runs_per_test',
463        default=1,
464        type=_runs_per_test_type,
465        help='How many times to run each tests. >1 runs implies ' +
466        'omitting passing test from the output & reports.')
467    argp.add_argument('--max_time',
468                      default=-1,
469                      type=int,
470                      help='Maximum amount of time to run tests for' +
471                      '(other tests will be skipped)')
472    argp.add_argument(
473        '--internal_ci',
474        default=False,
475        action='store_const',
476        const=True,
477        help=
478        '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of '
479        'results by Kokoro.')
480    argp.add_argument('--bq_result_table',
481                      default='',
482                      type=str,
483                      nargs='?',
484                      help='Upload test results to a specified BQ table.')
485    argp.add_argument('--extra_args',
486                      default='',
487                      type=str,
488                      nargs=argparse.REMAINDER,
489                      help='Extra test args passed to each sub-script.')
490    args = argp.parse_args()
491
492    extra_args = []
493    if args.build_only:
494        extra_args.append('--build_only')
495    if args.force_default_poller:
496        extra_args.append('--force_default_poller')
497    if args.runs_per_test > 1:
498        extra_args.append('-n')
499        extra_args.append('%s' % args.runs_per_test)
500        extra_args.append('--quiet_success')
501    if args.max_time > 0:
502        extra_args.extend(('--max_time', '%d' % args.max_time))
503    if args.bq_result_table:
504        extra_args.append('--bq_result_table')
505        extra_args.append('%s' % args.bq_result_table)
506        extra_args.append('--measure_cpu_costs')
507    if args.extra_args:
508        extra_args.extend(args.extra_args)
509
510    all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
511               _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
512
513    jobs = []
514    for job in all_jobs:
515        if not args.filter or all(
516                filter in job.labels for filter in args.filter):
517            if not any(exclude_label in job.labels
518                       for exclude_label in args.exclude):
519                jobs.append(job)
520
521    if not jobs:
522        jobset.message('FAILED',
523                       'No test suites match given criteria.',
524                       do_newline=True)
525        sys.exit(1)
526
527    print('IMPORTANT: The changes you are testing need to be locally committed')
528    print('because only the committed changes in the current branch will be')
529    print('copied to the docker environment or into subworkspaces.')
530
531    skipped_jobs = []
532
533    if args.filter_pr_tests:
534        print('Looking for irrelevant tests to skip...')
535        relevant_jobs = filter_tests(jobs, args.base_branch)
536        if len(relevant_jobs) == len(jobs):
537            print('No tests will be skipped.')
538        else:
539            print('These tests will be skipped:')
540            skipped_jobs = list(set(jobs) - set(relevant_jobs))
541            # Sort by shortnames to make printing of skipped tests consistent
542            skipped_jobs.sort(key=lambda job: job.shortname)
543            for job in list(skipped_jobs):
544                print('  %s' % job.shortname)
545        jobs = relevant_jobs
546
547    print('Will run these tests:')
548    for job in jobs:
549        print('  %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
550    print('')
551
552    if args.dry_run:
553        print('--dry_run was used, exiting')
554        sys.exit(1)
555
556    jobset.message('START', 'Running test matrix.', do_newline=True)
557    num_failures, resultset = jobset.run(jobs,
558                                         newline_on_success=True,
559                                         travis=True,
560                                         maxjobs=args.jobs)
561    # Merge skipped tests into results to show skipped tests on report.xml
562    if skipped_jobs:
563        ignored_num_skipped_failures, skipped_results = jobset.run(
564            skipped_jobs, skip_jobs=True)
565        resultset.update(skipped_results)
566    report_utils.render_junit_xml_report(resultset,
567                                         _report_filename(_MATRIX_REPORT_NAME),
568                                         suite_name=_MATRIX_REPORT_NAME,
569                                         multi_target=True)
570
571    if num_failures == 0:
572        jobset.message('SUCCESS',
573                       'All run_tests.py instances finished successfully.',
574                       do_newline=True)
575    else:
576        jobset.message('FAILED',
577                       'Some run_tests.py instances have failed.',
578                       do_newline=True)
579        sys.exit(1)
580