xref: /aosp_15_r20/external/grpc-grpc/tools/distrib/update_flakes.py (revision cc02d7e222339f7a4f6ba5f422e6413f4bd931f2) 
1#!/usr/bin/env python3
2
3# Copyright 2022 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import os
18import re
19import subprocess
20import sys
21
22from google.cloud import bigquery
23import run_buildozer
24import update_flakes_query
25
26lookback_hours = 24 * 7 * 4
27
28
29def include_test(test):
30    if "@" in test:
31        return False
32    if test.startswith("//test/cpp/qps:"):
33        return False
34    return True
35
36
37TEST_DIRS = ["test/core", "test/cpp"]
38tests = {}
39already_flaky = set()
40for test_dir in TEST_DIRS:
41    for line in subprocess.check_output(
42        ["bazel", "query", "tests({}/...)".format(test_dir)]
43    ).splitlines():
44        test = line.strip().decode("utf-8")
45        if not include_test(test):
46            continue
47        tests[test] = False
48for test_dir in TEST_DIRS:
49    for line in subprocess.check_output(
50        ["bazel", "query", "attr(flaky, 1, tests({}/...))".format(test_dir)]
51    ).splitlines():
52        test = line.strip().decode("utf-8")
53        if not include_test(test):
54            continue
55        already_flaky.add(test)
56
57flaky_e2e = set()
58
59client = bigquery.Client()
60for row in client.query(
61    update_flakes_query.QUERY.format(lookback_hours=lookback_hours)
62).result():
63    if "/macos/" in row.job_name:
64        continue  # we know mac stuff is flaky
65    if row.test_binary not in tests:
66        m = re.match(
67            r"^//test/core/end2end:([^@]*)@([^@]*)(.*)", row.test_binary
68        )
69        if m:
70            flaky_e2e.add("{}@{}{}".format(m.group(1), m.group(2), m.group(3)))
71            print("will mark end2end test {} as flaky".format(row.test_binary))
72        else:
73            print("skip obsolete test {}".format(row.test_binary))
74        continue
75    print("will mark {} as flaky".format(row.test_binary))
76    tests[row.test_binary] = True
77
78buildozer_commands = []
79for test, flaky in sorted(tests.items()):
80    if flaky:
81        buildozer_commands.append("set flaky True|{}".format(test))
82    elif test in already_flaky:
83        buildozer_commands.append("remove flaky|{}".format(test))
84
85with open("test/core/end2end/flaky.bzl", "w") as f:
86    with open(sys.argv[0]) as my_source:
87        for line in my_source:
88            if line[0] != "#":
89                break
90        for line in my_source:
91            if line[0] == "#":
92                print(line.strip(), file=f)
93                break
94        for line in my_source:
95            if line[0] != "#":
96                break
97            print(line.strip(), file=f)
98    print(
99        (
100            '"""A list of flaky tests, consumed by generate_tests.bzl to set'
101            ' flaky attrs."""'
102        ),
103        file=f,
104    )
105    print("FLAKY_TESTS = [", file=f)
106    for line in sorted(list(flaky_e2e)):
107        print('    "{}",'.format(line), file=f)
108    print("]", file=f)
109
110run_buildozer.run_buildozer(buildozer_commands)
111