1#!/usr/bin/env vpython3 2# Copyright 2021 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# pylint: disable=protected-access 7 8import datetime 9import os 10import unittest 11import unittest.mock as mock 12 13from typing import List, Tuple 14 15from flake_suppressor_common import common_typing as ct 16from flake_suppressor_common import data_types 17from flake_suppressor_common import results 18from flake_suppressor_common import tag_utils as common_tag_utils 19from flake_suppressor_common import unittest_utils as uu 20 21GENERIC_EXPECTATION_FILE_CONTENTS = """\ 22# tags: [ win ] 23# results: [ Failure ] 24crbug.com/1111 [ win ] foo_test [ Failure ] 25""" 26 27GPU_EXPECTATION_FILE_CONTENTS = """\ 28# tags: [ win ] 29# tags: [ amd nvidia ] 30# results: [ Failure ] 31crbug.com/1111 [ win nvidia ] conformance/textures/misc/video-rotation.html [ Failure ] 32""" 33 34 35class BaseResultsUnittest(unittest.TestCase): 36 def setUp(self) -> None: 37 common_tag_utils.SetTagUtilsImplementation(uu.UnitTestTagUtils) 38 expectations_processor = uu.UnitTestExpectationProcessor() 39 self._results = uu.UnitTestResultProcessor(expectations_processor) 40 self._local_patcher = mock.patch( 41 'flake_suppressor_common.results.expectations.' 42 'ExpectationProcessor.GetLocalCheckoutExpectationFileContents') 43 self._local_mock = self._local_patcher.start() 44 self._local_mock.return_value = {} 45 self.addCleanup(self._local_patcher.stop) 46 self._expectation_file_patcher = mock.patch.object( 47 uu.UnitTestExpectationProcessor, 'GetExpectationFileForSuite') 48 self._expectation_file_mock = self._expectation_file_patcher.start() 49 self.addCleanup(self._expectation_file_patcher.stop) 50 51 52class AggregateResultsUnittest(BaseResultsUnittest): 53 def testBasic(self) -> None: 54 """Basic functionality test.""" 55 query_results = [ 56 { 57 'name': ('gpu_tests.webgl_conformance_integration_test.' 58 'WebGLConformanceIntegrationTest.' 59 'conformance/textures/misc/video-rotation.html'), 60 'id': 61 'build-1111', 62 # The win-laptop tag is ignored, and thus should be removed in the 63 # output. 64 'typ_tags': ['win', 'nvidia', 'win-laptop'], 65 }, 66 { 67 'name': ('gpu_tests.webgl_conformance_integration_test.' 68 'WebGLConformanceIntegrationTest.' 69 'conformance/textures/misc/video-rotation.html'), 70 'id': 71 'build-2222', 72 'typ_tags': ['win', 'nvidia'], 73 }, 74 { 75 'name': ('gpu_tests.webgl_conformance_integration_test.' 76 'WebGLConformanceIntegrationTest.' 77 'conformance/textures/misc/video-rotation.html'), 78 'id': 79 'build-3333', 80 'typ_tags': ['win', 'amd'], 81 }, 82 { 83 'name': ('gpu_tests.webgl_conformance_integration_test.' 84 'WebGLConformanceIntegrationTest.' 85 'conformance/textures/misc/texture-npot-video.html'), 86 'id': 87 'build-4444', 88 'typ_tags': ['win', 'nvidia'], 89 }, 90 { 91 'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.' 92 'Pixel_CSS3DBlueBox'), 93 'id': 94 'build-5555', 95 'typ_tags': ['win', 'nvidia'], 96 }, 97 ] 98 expected_output = { 99 'webgl_conformance_integration_test': { 100 'conformance/textures/misc/video-rotation.html': { 101 ('nvidia', 'win'): [ 102 'http://ci.chromium.org/b/1111', 103 'http://ci.chromium.org/b/2222', 104 ], 105 ('amd', 'win'): ['http://ci.chromium.org/b/3333'], 106 }, 107 'conformance/textures/misc/texture-npot-video.html': { 108 ('nvidia', 'win'): ['http://ci.chromium.org/b/4444'], 109 }, 110 }, 111 'pixel_integration_test': { 112 'Pixel_CSS3DBlueBox': { 113 ('nvidia', 'win'): ['http://ci.chromium.org/b/5555'], 114 }, 115 }, 116 } 117 self.assertEqual(self._results.AggregateResults(query_results), 118 expected_output) 119 120 121class AggregateTestStatusResultsUnittest(BaseResultsUnittest): 122 def testBasic(self) -> None: 123 """Basic functionality test.""" 124 query_results = [ 125 { 126 'name': ('gpu_tests.webgl_conformance_integration_test.' 127 'WebGLConformanceIntegrationTest.' 128 'conformance/textures/misc/video-rotation.html'), 129 'id': 130 'build-1111', 131 # The win-laptop tag is ignored, and thus should be removed in the 132 # output. 133 'typ_tags': ['win', 'nvidia', 'win-laptop'], 134 'status': 135 ct.ResultStatus.FAIL, 136 'date': 137 '2023-03-01', 138 'is_slow': 139 False, 140 'typ_expectations': ['Pass'], 141 }, 142 { 143 'name': ('gpu_tests.webgl_conformance_integration_test.' 144 'WebGLConformanceIntegrationTest.' 145 'conformance/textures/misc/video-rotation.html'), 146 'id': 147 'build-2222', 148 'typ_tags': ['win', 'nvidia'], 149 'status': 150 ct.ResultStatus.CRASH, 151 'date': 152 '2023-03-02', 153 'is_slow': 154 False, 155 'typ_expectations': ['Pass'], 156 }, 157 { 158 'name': ('gpu_tests.webgl_conformance_integration_test.' 159 'WebGLConformanceIntegrationTest.' 160 'conformance/textures/misc/video-rotation.html'), 161 'id': 162 'build-3333', 163 'typ_tags': ['win', 'amd'], 164 'status': 165 ct.ResultStatus.FAIL, 166 'date': 167 '2023-03-03', 168 'is_slow': 169 True, 170 'typ_expectations': ['Pass', 'Slow'], 171 }, 172 { 173 'name': ('gpu_tests.webgl_conformance_integration_test.' 174 'WebGLConformanceIntegrationTest.' 175 'conformance/textures/misc/texture-npot-video.html'), 176 'id': 177 'build-4444', 178 'typ_tags': ['win', 'nvidia'], 179 'status': 180 ct.ResultStatus.FAIL, 181 'date': 182 '2023-03-04', 183 'is_slow': 184 True, 185 'typ_expectations': ['Pass', 'Slow'], 186 }, 187 { 188 'name': ('gpu_tests.pixel_integration_test.PixelIntegrationTest.' 189 'Pixel_CSS3DBlueBox'), 190 'id': 191 'build-5555', 192 'typ_tags': ['win', 'nvidia'], 193 'status': 194 ct.ResultStatus.FAIL, 195 'date': 196 '2023-03-05', 197 'is_slow': 198 False, 199 'typ_expectations': ['Pass'], 200 }, 201 ] 202 expected_output = { 203 'webgl_conformance_integration_test': { 204 'conformance/textures/misc/video-rotation.html': { 205 ('nvidia', 'win'): [ 206 (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/1111', 207 datetime.date.fromisoformat('2023-03-01'), False, ['Pass' 208 ]), 209 (ct.ResultStatus.CRASH, 'http://ci.chromium.org/b/2222', 210 datetime.date.fromisoformat('2023-03-02'), False, ['Pass' 211 ]), 212 ], 213 ('amd', 'win'): [ 214 (ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/3333', 215 datetime.date.fromisoformat('2023-03-03'), True, 216 ['Pass', 'Slow']), 217 ], 218 }, 219 'conformance/textures/misc/texture-npot-video.html': { 220 ('nvidia', 'win'): 221 [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/4444', 222 datetime.date.fromisoformat('2023-03-04'), True, 223 ['Pass', 'Slow'])], 224 }, 225 }, 226 'pixel_integration_test': { 227 'Pixel_CSS3DBlueBox': { 228 ('nvidia', 'win'): 229 [(ct.ResultStatus.FAIL, 'http://ci.chromium.org/b/5555', 230 datetime.date.fromisoformat('2023-03-05'), False, ['Pass'])], 231 }, 232 }, 233 } 234 self.assertEqual(self._results.AggregateTestStatusResults(query_results), 235 expected_output) 236 237 238class ConvertJsonResultsToResultObjectsUnittest(BaseResultsUnittest): 239 def testBasic(self) -> None: 240 """Basic functionality test.""" 241 r = [ 242 { 243 'name': ('gpu_tests.webgl_conformance_integration_test.' 244 'WebGLConformanceIntegrationTest.' 245 'conformance/textures/misc/video-rotation.html'), 246 'id': 247 'build-1111', 248 # The win-laptop tag is ignored, and thus should be removed in the 249 # output. 250 'typ_tags': ['win', 'nvidia', 'win-laptop'], 251 }, 252 { 253 'name': ('gpu_tests.webgl_conformance_integration_test.' 254 'WebGLConformanceIntegrationTest.' 255 'conformance/textures/misc/video-rotation.html'), 256 'id': 257 'build-2222', 258 'typ_tags': ['nvidia', 'win'], 259 }, 260 ] 261 expected_results = [ 262 data_types.Result('webgl_conformance_integration_test', 263 'conformance/textures/misc/video-rotation.html', 264 ('nvidia', 'win'), '1111'), 265 data_types.Result( 266 'webgl_conformance_integration_test', 267 'conformance/textures/misc/video-rotation.html', 268 ('nvidia', 'win'), 269 '2222', 270 ), 271 ] 272 self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r), 273 expected_results) 274 275 def testOnQueryResultWithOptionalAttributes(self) -> None: 276 """Functionality test on query result with optional attributes.""" 277 r = [ 278 { 279 'name': ('gpu_tests.webgl_conformance_integration_test.' 280 'WebGLConformanceIntegrationTest.' 281 'conformance/textures/misc/video-rotation.html'), 282 'id': 283 'build-1111', 284 # The win-laptop tag is ignored, and thus should be removed in the 285 # output. 286 'typ_tags': ['win', 'nvidia', 'win-laptop'], 287 'status': 288 ct.ResultStatus.FAIL, 289 'date': 290 '2023-03-01', 291 'is_slow': 292 False, 293 'typ_expectations': ['Pass'], 294 }, 295 { 296 'name': ('gpu_tests.webgl_conformance_integration_test.' 297 'WebGLConformanceIntegrationTest.' 298 'conformance/textures/misc/video-rotation.html'), 299 'id': 300 'build-2222', 301 'typ_tags': ['nvidia', 'win'], 302 'status': 303 ct.ResultStatus.CRASH, 304 'date': 305 '2023-03-02', 306 'is_slow': 307 True, 308 'typ_expectations': ['Pass', 'Slow'], 309 }, 310 ] 311 expected_results = [ 312 data_types.Result('webgl_conformance_integration_test', 313 'conformance/textures/misc/video-rotation.html', 314 ('nvidia', 'win'), '1111', ct.ResultStatus.FAIL, 315 datetime.date.fromisoformat('2023-03-01'), False, 316 ['Pass']), 317 data_types.Result('webgl_conformance_integration_test', 318 'conformance/textures/misc/video-rotation.html', 319 ('nvidia', 'win'), '2222', ct.ResultStatus.CRASH, 320 datetime.date.fromisoformat('2023-03-02'), True, 321 ['Pass', 'Slow']), 322 ] 323 self.assertEqual(self._results._ConvertJsonResultsToResultObjects(r), 324 expected_results) 325 326 327class FilterOutSuppressedResultsUnittest(BaseResultsUnittest): 328 def testNoSuppressedResults(self) -> None: 329 """Tests functionality when no expectations apply to the given results.""" 330 self._local_mock.return_value = { 331 'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS, 332 } 333 r = [ 334 data_types.Result('foo_integration_test', 'foo_test', tuple(['linux']), 335 'id'), 336 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 337 'id'), 338 data_types.Result('bar_integration_test', 'foo_test', tuple(['win']), 339 'id') 340 ] 341 342 self.assertEqual(self._results._FilterOutSuppressedResults(r), r) 343 344 def testSuppressedResults(self) -> None: 345 """Tests functionality when expectations apply to the given results.""" 346 self._local_mock.return_value = { 347 'foo_expectations.txt': GENERIC_EXPECTATION_FILE_CONTENTS, 348 } 349 self._expectation_file_mock.return_value = os.path.join( 350 uu.ABSOLUTE_EXPECTATION_FILE_DIRECTORY, 'foo_expectations.txt') 351 352 r = [ 353 data_types.Result('foo_integration_test', 'foo_test', ('win', 'nvidia'), 354 'id'), 355 data_types.Result('foo_integration_test', 'foo_test', tuple(['win']), 356 'id'), 357 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 358 'id'), 359 ] 360 361 expected_filtered_results = [ 362 data_types.Result('foo_integration_test', 'bar_test', tuple(['win']), 363 'id'), 364 ] 365 366 self.assertEqual(self._results._FilterOutSuppressedResults(r), 367 expected_filtered_results) 368 369 370if __name__ == '__main__': 371 unittest.main(verbosity=2) 372