1#!/usr/bin/env python3
2# Copyright 2017 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Run tests using docker images in Google Container Registry per matrix."""
16
17from __future__ import print_function
18
19import argparse
20import atexit
21import json
22import multiprocessing
23import os
24import re
25import subprocess
26import sys
27import uuid
28
29# Language Runtime Matrix
30import client_matrix
31
32python_util_dir = os.path.abspath(
33    os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
34sys.path.append(python_util_dir)
35import dockerjob
36import jobset
37import report_utils
38import upload_test_results
39
40_TEST_TIMEOUT_SECONDS = 60
41_PULL_IMAGE_TIMEOUT_SECONDS = 15 * 60
42_MAX_PARALLEL_DOWNLOADS = 6
43_LANGUAGES = list(client_matrix.LANG_RUNTIME_MATRIX.keys())
44# All gRPC release tags, flattened, deduped and sorted.
45_RELEASES = sorted(
46    list(
47        set(release
48            for release_dict in list(client_matrix.LANG_RELEASE_MATRIX.values())
49            for release in list(release_dict.keys()))))
50
51argp = argparse.ArgumentParser(description='Run interop tests.')
52argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
53argp.add_argument('--gcr_path',
54                  default='gcr.io/grpc-testing',
55                  help='Path of docker images in Google Container Registry')
56argp.add_argument('--release',
57                  default='all',
58                  choices=['all'] + _RELEASES,
59                  help='Release tags to test.  When testing all '
60                  'releases defined in client_matrix.py, use "all".')
61argp.add_argument('-l',
62                  '--language',
63                  choices=['all'] + sorted(_LANGUAGES),
64                  nargs='+',
65                  default=['all'],
66                  help='Languages to test')
67argp.add_argument(
68    '--keep',
69    action='store_true',
70    help='keep the created local images after finishing the tests.')
71argp.add_argument('--report_file',
72                  default='report.xml',
73                  help='The result file to create.')
74argp.add_argument('--allow_flakes',
75                  default=False,
76                  action='store_const',
77                  const=True,
78                  help=('Allow flaky tests to show as passing (re-runs failed '
79                        'tests up to five times)'))
80argp.add_argument('--bq_result_table',
81                  default='',
82                  type=str,
83                  nargs='?',
84                  help='Upload test results to a specified BQ table.')
85# Requests will be routed through specified VIP by default.
86# See go/grpc-interop-tests (internal-only) for details.
87argp.add_argument('--server_host',
88                  default='74.125.206.210',
89                  type=str,
90                  nargs='?',
91                  help='The gateway to backend services.')
92
93
94def _get_test_images_for_lang(lang, release_arg, image_path_prefix):
95    """Find docker images for a language across releases and runtimes.
96
97  Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime.
98  """
99    if release_arg == 'all':
100        # Use all defined releases for given language
101        releases = client_matrix.get_release_tags(lang)
102    else:
103        # Look for a particular release.
104        if release_arg not in client_matrix.get_release_tags(lang):
105            jobset.message('SKIPPED',
106                           'release %s for %s is not defined' %
107                           (release_arg, lang),
108                           do_newline=True)
109            return {}
110        releases = [release_arg]
111
112    # Image tuples keyed by runtime.
113    images = {}
114    for tag in releases:
115        for runtime in client_matrix.get_runtimes_for_lang_release(lang, tag):
116            image_name = '%s/grpc_interop_%s:%s' % (image_path_prefix, runtime,
117                                                    tag)
118            image_tuple = (tag, image_name)
119
120            if runtime not in images:
121                images[runtime] = []
122            images[runtime].append(image_tuple)
123    return images
124
125
126def _read_test_cases_file(lang, runtime, release):
127    """Read test cases from a bash-like file and return a list of commands"""
128    # Check to see if we need to use a particular version of test cases.
129    release_info = client_matrix.LANG_RELEASE_MATRIX[lang].get(release)
130    if release_info:
131        testcases_file = release_info.testcases_file
132    if not testcases_file:
133        # TODO(jtattermusch): remove the double-underscore, it is pointless
134        testcases_file = '%s__master' % lang
135
136    # For csharp, the testcases file used depends on the runtime
137    # TODO(jtattermusch): remove this odd specialcase
138    if lang == 'csharp' and runtime == 'csharpcoreclr':
139        testcases_file = testcases_file.replace('csharp_', 'csharpcoreclr_')
140
141    testcases_filepath = os.path.join(os.path.dirname(__file__), 'testcases',
142                                      testcases_file)
143    lines = []
144    with open(testcases_filepath) as f:
145        for line in f.readlines():
146            line = re.sub('\\#.*$', '', line)  # remove hash comments
147            line = line.strip()
148            if line and not line.startswith('echo'):
149                # Each non-empty line is a treated as a test case command
150                lines.append(line)
151    return lines
152
153
154def _cleanup_docker_image(image):
155    jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
156    dockerjob.remove_image(image, skip_nonexistent=True)
157
158
159args = argp.parse_args()
160
161
162# caches test cases (list of JobSpec) loaded from file.  Keyed by lang and runtime.
163def _generate_test_case_jobspecs(lang, runtime, release, suite_name):
164    """Returns the list of test cases from testcase files per lang/release."""
165    testcase_lines = _read_test_cases_file(lang, runtime, release)
166
167    job_spec_list = []
168    for line in testcase_lines:
169        print("Creating jobspec with cmdline '{}'".format(line))
170        # TODO(jtattermusch): revisit the logic for updating test case commands
171        # what it currently being done seems fragile.
172
173        # Extract test case name from the command line
174        m = re.search(r'--test_case=(\w+)', line)
175        testcase_name = m.group(1) if m else 'unknown_test'
176
177        # Extract the server name from the command line
178        if '--server_host_override=' in line:
179            m = re.search(
180                r'--server_host_override=((.*).sandbox.googleapis.com)', line)
181        else:
182            m = re.search(r'--server_host=((.*).sandbox.googleapis.com)', line)
183        server = m.group(1) if m else 'unknown_server'
184        server_short = m.group(2) if m else 'unknown_server'
185
186        # replace original server_host argument
187        assert '--server_host=' in line
188        line = re.sub(r'--server_host=[^ ]*',
189                      r'--server_host=%s' % args.server_host, line)
190
191        # some interop tests don't set server_host_override (see #17407),
192        # but we need to use it if different host is set via cmdline args.
193        if args.server_host != server and not '--server_host_override=' in line:
194            line = re.sub(r'(--server_host=[^ ]*)',
195                          r'\1 --server_host_override=%s' % server, line)
196
197        spec = jobset.JobSpec(cmdline=line,
198                              shortname='%s:%s:%s:%s' %
199                              (suite_name, lang, server_short, testcase_name),
200                              timeout_seconds=_TEST_TIMEOUT_SECONDS,
201                              shell=True,
202                              flake_retries=5 if args.allow_flakes else 0)
203        job_spec_list.append(spec)
204    return job_spec_list
205
206
207def _pull_image_for_lang(lang, image, release):
208    """Pull an image for a given language form the image registry."""
209    cmdline = [
210        'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true'
211        % (image, image)
212    ]
213    return jobset.JobSpec(cmdline=cmdline,
214                          shortname='pull_image_{}'.format(image),
215                          timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS,
216                          shell=True,
217                          flake_retries=2)
218
219
220def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests):
221    total_num_failures = 0
222    suite_name = '%s__%s_%s' % (lang, runtime, release)
223    job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
224                                                 suite_name)
225
226    if not job_spec_list:
227        jobset.message('FAILED', 'No test cases were found.', do_newline=True)
228        total_num_failures += 1
229    else:
230        num_failures, resultset = jobset.run(job_spec_list,
231                                             newline_on_success=True,
232                                             add_env={'docker_image': image},
233                                             maxjobs=args.jobs,
234                                             skip_jobs=skip_tests)
235        if args.bq_result_table and resultset:
236            upload_test_results.upload_interop_results_to_bq(
237                resultset, args.bq_result_table)
238        if skip_tests:
239            jobset.message('FAILED', 'Tests were skipped', do_newline=True)
240            total_num_failures += 1
241        if num_failures:
242            total_num_failures += num_failures
243
244        report_utils.append_junit_xml_results(xml_report_tree, resultset,
245                                              'grpc_interop_matrix', suite_name,
246                                              str(uuid.uuid4()))
247    return total_num_failures
248
249
250def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
251    """Find and run all test cases for a language.
252
253  images is a list of (<release-tag>, <image-full-path>) tuple.
254  """
255    skip_tests = False
256    total_num_failures = 0
257
258    max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS)
259    max_chunk_size = max_pull_jobs
260    chunk_count = (len(images) + max_chunk_size) // max_chunk_size
261
262    for chunk_index in range(chunk_count):
263        chunk_start = chunk_index * max_chunk_size
264        chunk_size = min(max_chunk_size, len(images) - chunk_start)
265        chunk_end = chunk_start + chunk_size
266        pull_specs = []
267        if not skip_tests:
268            for release, image in images[chunk_start:chunk_end]:
269                pull_specs.append(_pull_image_for_lang(lang, image, release))
270
271        # NOTE(rbellevi): We batch docker pull operations to maximize
272        # parallelism, without letting the disk usage grow unbounded.
273        pull_failures, _ = jobset.run(pull_specs,
274                                      newline_on_success=True,
275                                      maxjobs=max_pull_jobs)
276        if pull_failures:
277            jobset.message(
278                'FAILED',
279                'Image download failed. Skipping tests for language "%s"' %
280                lang,
281                do_newline=True)
282            skip_tests = True
283        for release, image in images[chunk_start:chunk_end]:
284            total_num_failures += _test_release(lang, runtime, release, image,
285                                                xml_report_tree, skip_tests)
286        if not args.keep:
287            for _, image in images[chunk_start:chunk_end]:
288                _cleanup_docker_image(image)
289    if not total_num_failures:
290        jobset.message('SUCCESS',
291                       'All {} tests passed'.format(lang),
292                       do_newline=True)
293    else:
294        jobset.message('FAILED',
295                       'Some {} tests failed'.format(lang),
296                       do_newline=True)
297
298    return total_num_failures
299
300
301languages = args.language if args.language != ['all'] else _LANGUAGES
302total_num_failures = 0
303_xml_report_tree = report_utils.new_junit_xml_tree()
304for lang in languages:
305    docker_images = _get_test_images_for_lang(lang, args.release, args.gcr_path)
306    for runtime in sorted(docker_images.keys()):
307        total_num_failures += _run_tests_for_lang(lang, runtime,
308                                                  docker_images[runtime],
309                                                  _xml_report_tree)
310
311report_utils.create_xml_report_file(_xml_report_tree, args.report_file)
312
313if total_num_failures:
314    sys.exit(1)
315sys.exit(0)
316