1#!/usr/bin/env python3
2# Copyright 2022 The gRPC Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Helps with running bazel with extra settings to generate structured test reports in CI."""
16
17import argparse
18import os
19import platform
20import sys
21import uuid
22
23_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
24os.chdir(_ROOT)
25
26# How long to sleep before querying Resultstore API and uploading to bigquery
27# (to let ResultStore finish writing results from the bazel invocation that has
28# just finished).
29_UPLOAD_RBE_RESULTS_DELAY_SECONDS = 60
30
31
32def _platform_string():
33    """Detect current platform"""
34    if platform.system() == 'Windows':
35        return 'windows'
36    elif platform.system()[:7] == 'MSYS_NT':
37        return 'windows'
38    elif platform.system() == 'Darwin':
39        return 'mac'
40    elif platform.system() == 'Linux':
41        return 'linux'
42    else:
43        return 'posix'
44
45
46def _append_to_kokoro_bazel_invocations(invocation_id: str) -> None:
47    """Kokoro can display "Bazel" result link on kokoro jobs if told so."""
48    # to get "bazel" link for kokoro build, we need to upload
49    # the "bazel_invocation_ids" file with bazel invocation ID as artifact.
50    kokoro_artifacts_dir = os.getenv('KOKORO_ARTIFACTS_DIR')
51    if kokoro_artifacts_dir:
52        # append the bazel invocation UUID to the bazel_invocation_ids file.
53        with open(os.path.join(kokoro_artifacts_dir, 'bazel_invocation_ids'),
54                  'a') as f:
55            f.write(invocation_id + '\n')
56        print(
57            'Added invocation ID %s to kokoro "bazel_invocation_ids" artifact' %
58            invocation_id,
59            file=sys.stderr)
60    else:
61        print(
62            'Skipped adding invocation ID %s to kokoro "bazel_invocation_ids" artifact'
63            % invocation_id,
64            file=sys.stderr)
65        pass
66
67
68def _generate_junit_report_string(report_suite_name: str, invocation_id: str,
69                                  success: bool) -> None:
70    """Generate sponge_log.xml formatted report, that will make the bazel invocation reachable as a target in resultstore UI / sponge."""
71    bazel_invocation_url = 'https://source.cloud.google.com/results/invocations/%s' % invocation_id
72    package_name = report_suite_name
73    # set testcase name to invocation URL. That way, the link will be displayed in some form
74    # resultstore UI and sponge even in case the bazel invocation succeeds.
75    testcase_name = bazel_invocation_url
76    if success:
77        # unfortunately, neither resultstore UI nor sponge display the "system-err" output (or any other tags)
78        # on a passing test case. But at least we tried.
79        test_output_tag = '<system-err>PASSED. See invocation results here: %s</system-err>' % bazel_invocation_url
80    else:
81        # The failure output will be displayes in both resultstore UI and sponge when clicking on the failing testcase.
82        test_output_tag = '<failure message="Failure">FAILED. See bazel invocation results here: %s</failure>' % bazel_invocation_url
83
84    lines = [
85        '<testsuites>',
86        '<testsuite id="1" name="%s" package="%s">' %
87        (report_suite_name, package_name),
88        '<testcase name="%s">' % testcase_name,
89        test_output_tag,
90        '</testcase>'
91        '</testsuite>',
92        '</testsuites>',
93    ]
94    return '\n'.join(lines)
95
96
97def _create_bazel_wrapper(report_path: str, report_suite_name: str,
98                          invocation_id: str, upload_results: bool) -> None:
99    """Create a "bazel wrapper" script that will execute bazel with extra settings and postprocessing."""
100
101    os.makedirs(report_path, exist_ok=True)
102
103    bazel_wrapper_filename = os.path.join(report_path, 'bazel_wrapper')
104    bazel_wrapper_bat_filename = bazel_wrapper_filename + '.bat'
105    bazel_rc_filename = os.path.join(report_path, 'bazel_wrapper.bazelrc')
106
107    # put xml reports in a separate directory if requested by GRPC_TEST_REPORT_BASE_DIR
108    report_base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None)
109    xml_report_path = os.path.abspath(
110        os.path.join(report_base_dir, report_path
111                    ) if report_base_dir else report_path)
112    os.makedirs(xml_report_path, exist_ok=True)
113
114    failing_report_filename = os.path.join(xml_report_path, 'sponge_log.xml')
115    success_report_filename = os.path.join(xml_report_path,
116                                           'success_log_to_rename.xml')
117
118    if _platform_string() == 'windows':
119        workspace_status_command = 'tools/remote_build/workspace_status_kokoro.bat'
120    else:
121        workspace_status_command = 'tools/remote_build/workspace_status_kokoro.sh'
122
123    # generate RC file with the bazel flags we want to use apply.
124    # Using an RC file solves problems with flag ordering in the wrapper.
125    # (e.g. some flags need to come after the build/test command)
126    with open(bazel_rc_filename, 'w') as f:
127        f.write('build --invocation_id="%s"\n' % invocation_id)
128        f.write('build --workspace_status_command="%s"\n' %
129                workspace_status_command)
130
131    # generate "failing" and "success" report
132    # the "failing" is named as "sponge_log.xml", which is the name picked up by sponge/resultstore
133    # so the failing report will be used by default (unless we later replace the report with
134    # one that says "success"). That way if something goes wrong before bazel is run,
135    # there will at least be a "failing" target that indicates that (we really don't want silent failures).
136    with open(failing_report_filename, 'w') as f:
137        f.write(
138            _generate_junit_report_string(report_suite_name,
139                                          invocation_id,
140                                          success=False))
141    with open(success_report_filename, 'w') as f:
142        f.write(
143            _generate_junit_report_string(report_suite_name,
144                                          invocation_id,
145                                          success=True))
146
147    # generate the bazel wrapper for linux/macos
148    with open(bazel_wrapper_filename, 'w') as f:
149        intro_lines = [
150            '#!/bin/bash',
151            'set -ex',
152            '',
153            'tools/bazel --bazelrc="%s" "$@" || FAILED=true' %
154            bazel_rc_filename,
155            '',
156        ]
157
158        if upload_results:
159            upload_results_lines = [
160                'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS,
161                'PYTHONHTTPSVERIFY=0 python3 ./tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s"'
162                % invocation_id,
163                '',
164            ]
165        else:
166            upload_results_lines = []
167
168        outro_lines = [
169            'if [ "$FAILED" != "" ]',
170            'then',
171            '  exit 1',
172            'else',
173            '  # success: plant the pre-generated xml report that says "success"',
174            '  mv -f %s %s' %
175            (success_report_filename, failing_report_filename),
176            'fi',
177        ]
178
179        lines = [
180            line + '\n'
181            for line in intro_lines + upload_results_lines + outro_lines
182        ]
183        f.writelines(lines)
184    os.chmod(bazel_wrapper_filename, 0o775)  # make the unix wrapper executable
185
186    # generate bazel wrapper for windows
187    with open(bazel_wrapper_bat_filename, 'w') as f:
188        intro_lines = [
189            '@echo on',
190            '',
191            'bazel --bazelrc="%s" %%*' % bazel_rc_filename,
192            'set BAZEL_EXITCODE=%errorlevel%',
193            '',
194        ]
195
196        if upload_results:
197            upload_results_lines = [
198                'sleep %s' % _UPLOAD_RBE_RESULTS_DELAY_SECONDS,
199                'python3 tools/run_tests/python_utils/upload_rbe_results.py --invocation_id="%s" || exit /b 1'
200                % invocation_id,
201                '',
202            ]
203        else:
204            upload_results_lines = []
205
206        outro_lines = [
207            'if %BAZEL_EXITCODE% == 0 (',
208            '  @rem success: plant the pre-generated xml report that says "success"',
209            '  mv -f %s %s' %
210            (success_report_filename, failing_report_filename),
211            ')',
212            'exit /b %BAZEL_EXITCODE%',
213        ]
214
215        lines = [
216            line + '\n'
217            for line in intro_lines + upload_results_lines + outro_lines
218        ]
219        f.writelines(lines)
220
221    print('Bazel invocation ID: %s' % invocation_id, file=sys.stderr)
222    print('Upload test results to BigQuery after bazel runs: %s' %
223          upload_results,
224          file=sys.stderr)
225    print('Generated bazel wrapper: %s' % bazel_wrapper_filename,
226          file=sys.stderr)
227    print('Generated bazel wrapper: %s' % bazel_wrapper_bat_filename,
228          file=sys.stderr)
229
230
231if __name__ == '__main__':
232    # parse command line
233    argp = argparse.ArgumentParser(
234        description=
235        'Generate bazel wrapper to help with bazel test reports in CI.')
236    argp.add_argument(
237        '--report_path',
238        required=True,
239        type=str,
240        help=
241        'Path under which the bazel wrapper and other files are going to be generated'
242    )
243    argp.add_argument('--report_suite_name',
244                      default='bazel_invocations',
245                      type=str,
246                      help='Test suite name to use in generated XML report')
247    args = argp.parse_args()
248
249    # generate new bazel invocation ID
250    invocation_id = str(uuid.uuid4())
251
252    report_path = args.report_path
253    report_suite_name = args.report_suite_name
254    upload_results = True if os.getenv('UPLOAD_TEST_RESULTS') else False
255
256    _append_to_kokoro_bazel_invocations(invocation_id)
257    _create_bazel_wrapper(report_path, report_suite_name, invocation_id,
258                          upload_results)
259