xref: /aosp_15_r20/external/cronet/testing/scripts/rust/test_results.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1# Copyright 2021 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""This is a library for printing test result as specified by
5//docs/testing/test_executable_api.md (e.g. --isolated-script-test-output)
6and //docs/testing/json_test_results_format.md
7
8Typical usage:
9    import argparse
10    import test_results
11
12    cmdline_parser = argparse.ArgumentParser()
13    test_results.add_cmdline_args(cmdline_parser)
14    ... adding other cmdline parameter definitions ...
15    parsed_cmdline_args = cmdline_parser.parse_args()
16
17    test_results = []
18    test_results.append(test_results.TestResult(
19        'test-suite/test-name', 'PASS'))
20    ...
21
22    test_results.print_test_results(parsed_cmdline_args, test_results)
23"""
24
25import argparse
26import json
27import os
28
29
30class TestResult:
31    """TestResult represents a result of executing a single test once.
32    """
33
34    def __init__(self,
35                 test_name,
36                 actual_test_result,
37                 expected_test_result='PASS'):
38        self.test_name = test_name
39        self.actual_test_result = actual_test_result
40        self.expected_test_result = expected_test_result
41
42    def __eq__(self, other):
43        self_tuple = tuple(sorted(self.__dict__.items()))
44        other_tuple = tuple(sorted(other.__dict__.items()))
45        return self_tuple == other_tuple
46
47    def __hash__(self):
48        return hash(tuple(sorted(self.__dict__.items())))
49
50    def __repr__(self):
51        result = 'TestResult[{}: {}'.format(self.test_name,
52                                            self.actual_test_result)
53        if self.expected_test_result != 'PASS':
54            result += ' (expecting: {})]'.format(self.expected_test_result)
55        else:
56            result += ']'
57        return result
58
59
60def _validate_output_directory(outdir):
61    if not os.path.isdir(outdir):
62        raise argparse.ArgumentTypeError('No such directory: ' + outdir)
63    return outdir
64
65
66def add_cmdline_args(argparse_parser):
67    """Adds test-result-specific cmdline parameter definitions to
68    `argparse_parser`.
69
70    Args:
71        argparse_parser: An object of argparse.ArgumentParser type.
72    """
73    outdir_help = 'Directory where test results will be written into.'
74    argparse_parser.add_argument('--isolated-outdir',
75                                 dest='outdir',
76                                 help=outdir_help,
77                                 metavar='DIRPATH',
78                                 type=_validate_output_directory)
79
80    outfile_help = 'If this argument is provided, then test results in the ' \
81                   'JSON Test Results Format will be written here. See also ' \
82                   '//docs/testing/json_test_results_format.md'
83    argparse_parser.add_argument('--isolated-script-test-output',
84                                 dest='test_output',
85                                 default=None,
86                                 help=outfile_help,
87                                 metavar='FILENAME')
88
89    argparse_parser.add_argument('--isolated-script-test-perf-output',
90                                 dest='ignored_perf_output',
91                                 default=None,
92                                 help='Deprecated and ignored.',
93                                 metavar='IGNORED')
94
95
96def _build_json_data(list_of_test_results, seconds_since_epoch):
97    num_failures_by_type = {}
98    tests = {}
99    for res in list_of_test_results:
100        old_count = num_failures_by_type.get(res.actual_test_result, 0)
101        num_failures_by_type[res.actual_test_result] = old_count + 1
102
103        path = res.test_name.split('//')
104        group = tests
105        for group_name in path[:-1]:
106            if not group_name in group:
107                group[group_name] = {}
108            group = group[group_name]
109
110        group[path[-1]] = {
111            'expected': res.expected_test_result,
112            'actual': res.actual_test_result,
113        }
114
115    return {
116        'interrupted': False,
117        'path_delimiter': '//',
118        'seconds_since_epoch': seconds_since_epoch,
119        'version': 3,
120        'tests': tests,
121        'num_failures_by_type': num_failures_by_type,
122    }
123
124
125def print_test_results(argparse_parsed_args, list_of_test_results,
126                       testing_start_time_as_seconds_since_epoch):
127    """Prints `list_of_test_results` to a file specified on the cmdline.
128
129    Args:
130        argparse_parsed_arg: A result of an earlier call to
131          argparse_parser.parse_args() call (where `argparse_parser` has been
132          populated via an even earlier call to add_cmdline_args).
133        list_of_test_results: A list of TestResult objects.
134        testing_start_time_as_seconds_since_epoch: A number from an earlier
135          `time.time()` call.
136    """
137    if argparse_parsed_args.test_output is None:
138        return
139
140    json_data = _build_json_data(list_of_test_results,
141                                 testing_start_time_as_seconds_since_epoch)
142
143    filepath = argparse_parsed_args.test_output
144    with open(filepath, mode='w', encoding='utf-8') as f:
145        json.dump(json_data, f, indent=2)
146