xref: /aosp_15_r20/external/cronet/testing/flake_suppressor_common/results_unittest.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b)
1#!/usr/bin/env vpython3
2# Copyright 2021 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6# pylint: disable=protected-access
7
8import datetime
9import os
10import unittest
11import unittest.mock as mock
12
13from typing import List, Tuple
14
15from flake_suppressor_common import common_typing as ct
16from flake_suppressor_common import data_types
17from flake_suppressor_common import results
18from flake_suppressor_common import tag_utils as common_tag_utils
19from flake_suppressor_common import unittest_utils as uu
20
21GENERIC_EXPECTATION_FILE_CONTENTS = """\
22# tags: [ win ]
23# results: [ Failure ]
24crbug.com/1111 [ win ] foo_test [ Failure ]
25"""
26
27GPU_EXPECTATION_FILE_CONTENTS = """\
28# tags: [ win ]
29# tags: [ amd nvidia ]
30# results: [ Failure ]
31crbug.com/1111 [ win nvidia ] conformance/textures/misc/video-rotation.html [ Failure ]
32"""
33
34
35class BaseResultsUnittest(unittest.TestCase):
36  def setUp(self) -> None:
37    common_tag_utils.SetTagUtilsImplementation(uu.UnitTestTagUtils)
38    expectations_processor = uu.UnitTestExpectationProcessor()
39    self._results = uu.UnitTestResultProcessor(expectations_processor)
40    self._local_patcher = mock.patch(
41        'flake_suppressor_common.results.expectations.'
42        'ExpectationProcessor.GetLocalCheckoutExpectationFileContents')
43    self._local_mock = self._local_patcher.start()
44    self._local_mock.return_value = {}
45    self.addCleanup(self._local_patcher.stop)
46    self._expectation_file_patcher = mock.patch.object(
47        uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite')
48    self._expectation_file_mock = self._expectation_file_patcher.start()
49    self.addCleanup(self._expectation_file_patcher.stop)
50
51
52class AggregateResultsUnittest(BaseResultsUnittest):
53  def testBasic(self) -> None:
54    """Basic functionality test."""
55    query_results = [
56        {
57            'name': ('gpu_tests.webgl_conformance_integration_test.'
58                     'WebGLConformanceIntegrationTest.'
59                     'conformance/textures/misc/video-rotation.html'),
60            'id':
61            'build-1111',
62            # The win-laptop tag is ignored, and thus should be removed in the
63            # output.
64            'typ_tags': ['win', 'nvidia', 'win-laptop'],
65        },
66        {
67            'name': ('gpu_tests.webgl_conformance_integration_test.'
68                     'WebGLConformanceIntegrationTest.'
69                     'conformance/textures/misc/video-rotation.html'),
70            'id':
71            'build-2222',
72            'typ_tags': ['win', 'nvidia'],
73        },
74        {
75            'name': ('gpu_tests.webgl_conformance_integration_test.'
76                     'WebGLConformanceIntegrationTest.'
77                     'conformance/textures/misc/video-rotation.html'),
78            'id':
79            'build-3333',
80            'typ_tags': ['win', 'amd'],
81        },
82        {
83            'name': ('gpu_tests.webgl_conformance_integration_test.'
84                     'WebGLConformanceIntegrationTest.'
85                     'conformance/textures/misc/texture-npot-video.html'),
86            'id':
87            'build-4444',
88            'typ_tags': ['win', 'nvidia'],
89        },
90        {
91            'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.'
92                     'Pixel_CSS3DBlueBox'),
93            'id':
94            'build-5555',
95            'typ_tags': ['win', 'nvidia'],
96        },
97    ]
98    expected_output = {
99        'webgl_conformance_integration_test': {
100            'conformance/textures/misc/video-rotation.html': {
101                ('nvidia', 'win'): [
102                    'http://ci.chromium.org/b/1111',
103                    'http://ci.chromium.org/b/2222',
104                ],
105                ('amd', 'win'): ['http://ci.chromium.org/b/3333'],
106            },
107            'conformance/textures/misc/texture-npot-video.html': {
108                ('nvidia', 'win'): ['http://ci.chromium.org/b/4444'],
109            },
110        },
111        'pixel_integration_test': {
112            'Pixel_CSS3DBlueBox': {
113                ('nvidia', 'win'): ['http://ci.chromium.org/b/5555'],
114            },
115        },
116    }
117    self.assertEqual(self._results.AggregateResults(query_results),
118                     expected_output)
119
120
121class AggregateTestStatusResultsUnittest(BaseResultsUnittest):
122  def testBasic(self) -> None:
123    """Basic functionality test."""
124    query_results = [
125        {
126            'name': ('gpu_tests.webgl_conformance_integration_test.'
127                     'WebGLConformanceIntegrationTest.'
128                     'conformance/textures/misc/video-rotation.html'),
129            'id':
130            'build-1111',
131            # The win-laptop tag is ignored, and thus should be removed in the
132            # output.
133            'typ_tags': ['win', 'nvidia', 'win-laptop'],
134            'status':
135            ct.ResultStatus.FAIL,
136            'date':
137            '2023-03-01',
138            'is_slow':
139            False,
140            'typ_expectations': ['Pass'],
141        },
142        {
143            'name': ('gpu_tests.webgl_conformance_integration_test.'
144                     'WebGLConformanceIntegrationTest.'
145                     'conformance/textures/misc/video-rotation.html'),
146            'id':
147            'build-2222',
148            'typ_tags': ['win', 'nvidia'],
149            'status':
150            ct.ResultStatus.CRASH,
151            'date':
152            '2023-03-02',
153            'is_slow':
154            False,
155            'typ_expectations': ['Pass'],
156        },
157        {
158            'name': ('gpu_tests.webgl_conformance_integration_test.'
159                     'WebGLConformanceIntegrationTest.'
160                     'conformance/textures/misc/video-rotation.html'),
161            'id':
162            'build-3333',
163            'typ_tags': ['win', 'amd'],
164            'status':
165            ct.ResultStatus.FAIL,
166            'date':
167            '2023-03-03',
168            'is_slow':
169            True,
170            'typ_expectations': ['Pass', 'Slow'],
171        },
172        {
173            'name': ('gpu_tests.webgl_conformance_integration_test.'
174                     'WebGLConformanceIntegrationTest.'
175                     'conformance/textures/misc/texture-npot-video.html'),
176            'id':
177            'build-4444',
178            'typ_tags': ['win', 'nvidia'],
179            'status':
180            ct.ResultStatus.FAIL,
181            'date':
182            '2023-03-04',
183            'is_slow':
184            True,
185            'typ_expectations': ['Pass', 'Slow'],
186        },
187        {
188            'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.'
189                     'Pixel_CSS3DBlueBox'),
190            'id':
191            'build-5555',
192            'typ_tags': ['win', 'nvidia'],
193            'status':
194            ct.ResultStatus.FAIL,
195            'date':
196            '2023-03-05',
197            'is_slow':
198            False,
199            'typ_expectations': ['Pass'],
200        },
201    ]
202    expected_output = {
203        'webgl_conformance_integration_test': {
204            'conformance/textures/misc/video-rotation.html': {
205                ('nvidia', 'win'): [
206                    (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111',
207                     datetime.date.fromisoformat('2023-03-01'), False, ['Pass'
208                                                                        ]),
209                    (ct.ResultStatus.CRASH, 'http://ci.chromium.org/b/2222',
210                     datetime.date.fromisoformat('2023-03-02'), False, ['Pass'
211                                                                        ]),
212                ],
213                ('amd', 'win'): [
214                    (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/3333',
215                     datetime.date.fromisoformat('2023-03-03'), True,
216                     ['Pass', 'Slow']),
217                ],
218            },
219            'conformance/textures/misc/texture-npot-video.html': {
220                ('nvidia', 'win'):
221                [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/4444',
222                  datetime.date.fromisoformat('2023-03-04'), True,
223                  ['Pass', 'Slow'])],
224            },
225        },
226        'pixel_integration_test': {
227            'Pixel_CSS3DBlueBox': {
228                ('nvidia', 'win'):
229                [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/5555',
230                  datetime.date.fromisoformat('2023-03-05'), False, ['Pass'])],
231            },
232        },
233    }
234    self.assertEqual(self._results.AggregateTestStatusResults(query_results),
235                     expected_output)
236
237
238class ConvertJsonResultsToResultObjectsUnittest(BaseResultsUnittest):
239  def testBasic(self) -> None:
240    """Basic functionality test."""
241    r = [
242        {
243            'name': ('gpu_tests.webgl_conformance_integration_test.'
244                     'WebGLConformanceIntegrationTest.'
245                     'conformance/textures/misc/video-rotation.html'),
246            'id':
247            'build-1111',
248            # The win-laptop tag is ignored, and thus should be removed in the
249            # output.
250            'typ_tags': ['win', 'nvidia', 'win-laptop'],
251        },
252        {
253            'name': ('gpu_tests.webgl_conformance_integration_test.'
254                     'WebGLConformanceIntegrationTest.'
255                     'conformance/textures/misc/video-rotation.html'),
256            'id':
257            'build-2222',
258            'typ_tags': ['nvidia', 'win'],
259        },
260    ]
261    expected_results = [
262        data_types.Result('webgl_conformance_integration_test',
263                          'conformance/textures/misc/video-rotation.html',
264                          ('nvidia', 'win'), '1111'),
265        data_types.Result(
266            'webgl_conformance_integration_test',
267            'conformance/textures/misc/video-rotation.html',
268            ('nvidia', 'win'),
269            '2222',
270        ),
271    ]
272    self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r),
273                     expected_results)
274
275  def testOnQueryResultWithOptionalAttributes(self) -> None:
276    """Functionality test on query result with optional attributes."""
277    r = [
278        {
279            'name': ('gpu_tests.webgl_conformance_integration_test.'
280                     'WebGLConformanceIntegrationTest.'
281                     'conformance/textures/misc/video-rotation.html'),
282            'id':
283            'build-1111',
284            # The win-laptop tag is ignored, and thus should be removed in the
285            # output.
286            'typ_tags': ['win', 'nvidia', 'win-laptop'],
287            'status':
288            ct.ResultStatus.FAIL,
289            'date':
290            '2023-03-01',
291            'is_slow':
292            False,
293            'typ_expectations': ['Pass'],
294        },
295        {
296            'name': ('gpu_tests.webgl_conformance_integration_test.'
297                     'WebGLConformanceIntegrationTest.'
298                     'conformance/textures/misc/video-rotation.html'),
299            'id':
300            'build-2222',
301            'typ_tags': ['nvidia', 'win'],
302            'status':
303            ct.ResultStatus.CRASH,
304            'date':
305            '2023-03-02',
306            'is_slow':
307            True,
308            'typ_expectations': ['Pass', 'Slow'],
309        },
310    ]
311    expected_results = [
312        data_types.Result('webgl_conformance_integration_test',
313                          'conformance/textures/misc/video-rotation.html',
314                          ('nvidia', 'win'), '1111', ct.ResultStatus.FAIL,
315                          datetime.date.fromisoformat('2023-03-01'), False,
316                          ['Pass']),
317        data_types.Result('webgl_conformance_integration_test',
318                          'conformance/textures/misc/video-rotation.html',
319                          ('nvidia', 'win'), '2222', ct.ResultStatus.CRASH,
320                          datetime.date.fromisoformat('2023-03-02'), True,
321                          ['Pass', 'Slow']),
322    ]
323    self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r),
324                     expected_results)
325
326
327class FilterOutSuppressedResultsUnittest(BaseResultsUnittest):
328  def testNoSuppressedResults(self) -> None:
329    """Tests functionality when no expectations apply to the given results."""
330    self._local_mock.return_value = {
331        'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS,
332    }
333    r = [
334        data_types.Result('foo_integration_test', 'foo_test', tuple(['linux']),
335                          'id'),
336        data_types.Result('foo_integration_test', 'bar_test', tuple(['win']),
337                          'id'),
338        data_types.Result('bar_integration_test', 'foo_test', tuple(['win']),
339                          'id')
340    ]
341
342    self.assertEqual(self._results._FilterOutSuppressedResults(r), r)
343
344  def testSuppressedResults(self) -> None:
345    """Tests functionality when expectations apply to the given results."""
346    self._local_mock.return_value = {
347        'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS,
348    }
349    self._expectation_file_mock.return_value = os.path.join(
350        uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY, 'foo_expectations.txt')
351
352    r = [
353        data_types.Result('foo_integration_test', 'foo_test', ('win', 'nvidia'),
354                          'id'),
355        data_types.Result('foo_integration_test', 'foo_test', tuple(['win']),
356                          'id'),
357        data_types.Result('foo_integration_test', 'bar_test', tuple(['win']),
358                          'id'),
359    ]
360
361    expected_filtered_results = [
362        data_types.Result('foo_integration_test', 'bar_test', tuple(['win']),
363                          'id'),
364    ]
365
366    self.assertEqual(self._results._FilterOutSuppressedResults(r),
367                     expected_filtered_results)
368
369
370if __name__ == '__main__':
371  unittest.main(verbosity=2)
372