1#!/usr/bin/env python3 2# Copyright 2019 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# pylint: disable=protected-access 7 8"""Tests for LLVM bisection.""" 9 10import json 11import os 12import subprocess 13import unittest 14from unittest import mock 15 16import chroot 17import get_llvm_hash 18import git_llvm_rev 19import llvm_bisection 20import modify_a_tryjob 21import test_helpers 22 23 24class LLVMBisectionTest(unittest.TestCase): 25 """Unittests for LLVM bisection.""" 26 27 def testGetRemainingRangePassed(self): 28 start = 100 29 end = 150 30 31 test_tryjobs = [ 32 { 33 "rev": 110, 34 "status": "good", 35 "link": "https://some_tryjob_1_url.com", 36 }, 37 { 38 "rev": 120, 39 "status": "good", 40 "link": "https://some_tryjob_2_url.com", 41 }, 42 { 43 "rev": 130, 44 "status": "pending", 45 "link": "https://some_tryjob_3_url.com", 46 }, 47 { 48 "rev": 135, 49 "status": "skip", 50 "link": "https://some_tryjob_4_url.com", 51 }, 52 { 53 "rev": 140, 54 "status": "bad", 55 "link": "https://some_tryjob_5_url.com", 56 }, 57 ] 58 59 # Tuple consists of the new good revision, the new bad revision, a set 60 # of 'pending' revisions, and a set of 'skip' revisions. 61 expected_revisions_tuple = 120, 140, {130}, {135} 62 63 self.assertEqual( 64 llvm_bisection.GetRemainingRange(start, end, test_tryjobs), 65 expected_revisions_tuple, 66 ) 67 68 def testGetRemainingRangeFailedWithMissingStatus(self): 69 start = 100 70 end = 150 71 72 test_tryjobs = [ 73 { 74 "rev": 105, 75 "status": "good", 76 "link": "https://some_tryjob_1_url.com", 77 }, 78 { 79 "rev": 120, 80 "status": None, 81 "link": "https://some_tryjob_2_url.com", 82 }, 83 { 84 "rev": 140, 85 "status": "bad", 86 "link": "https://some_tryjob_3_url.com", 87 }, 88 ] 89 90 with self.assertRaises(ValueError) as err: 91 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 92 93 error_message = ( 94 '"status" is missing or has no value, please ' 95 "go to %s and update it" % test_tryjobs[1]["link"] 96 ) 97 self.assertEqual(str(err.exception), error_message) 98 99 def testGetRemainingRangeFailedWithInvalidRange(self): 100 start = 100 101 end = 150 102 103 test_tryjobs = [ 104 { 105 "rev": 110, 106 "status": "bad", 107 "link": "https://some_tryjob_1_url.com", 108 }, 109 { 110 "rev": 125, 111 "status": "skip", 112 "link": "https://some_tryjob_2_url.com", 113 }, 114 { 115 "rev": 140, 116 "status": "good", 117 "link": "https://some_tryjob_3_url.com", 118 }, 119 ] 120 121 with self.assertRaises(AssertionError) as err: 122 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 123 124 expected_error_message = ( 125 "Bisection is broken because %d (good) is >= " 126 "%d (bad)" % (test_tryjobs[2]["rev"], test_tryjobs[0]["rev"]) 127 ) 128 129 self.assertEqual(str(err.exception), expected_error_message) 130 131 @mock.patch.object(get_llvm_hash, "GetGitHashFrom") 132 def testGetCommitsBetweenPassed(self, mock_get_git_hash): 133 start = git_llvm_rev.base_llvm_revision 134 end = start + 10 135 test_pending_revisions = {start + 7} 136 test_skip_revisions = { 137 start + 1, 138 start + 2, 139 start + 4, 140 start + 8, 141 start + 9, 142 } 143 parallel = 3 144 abs_path_to_src = "/abs/path/to/src" 145 146 revs = ["a123testhash3", "a123testhash5"] 147 mock_get_git_hash.side_effect = revs 148 149 git_hashes = [ 150 git_llvm_rev.base_llvm_revision + 3, 151 git_llvm_rev.base_llvm_revision + 5, 152 ] 153 154 self.assertEqual( 155 llvm_bisection.GetCommitsBetween( 156 start, 157 end, 158 parallel, 159 abs_path_to_src, 160 test_pending_revisions, 161 test_skip_revisions, 162 ), 163 (git_hashes, revs), 164 ) 165 166 def testLoadStatusFilePassedWithExistingFile(self): 167 start = 100 168 end = 150 169 170 test_bisect_state = {"start": start, "end": end, "jobs": []} 171 172 # Simulate that the status file exists. 173 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 174 with open(temp_json_file, "w", encoding="utf-8") as f: 175 test_helpers.WritePrettyJsonFile(test_bisect_state, f) 176 177 self.assertEqual( 178 llvm_bisection.LoadStatusFile(temp_json_file, start, end), 179 test_bisect_state, 180 ) 181 182 def testLoadStatusFilePassedWithoutExistingFile(self): 183 start = 200 184 end = 250 185 186 expected_bisect_state = {"start": start, "end": end, "jobs": []} 187 188 last_tested = "/abs/path/to/file_that_does_not_exist.json" 189 190 self.assertEqual( 191 llvm_bisection.LoadStatusFile(last_tested, start, end), 192 expected_bisect_state, 193 ) 194 195 @mock.patch.object(modify_a_tryjob, "AddTryjob") 196 def testBisectPassed(self, mock_add_tryjob): 197 git_hash_list = ["a123testhash1", "a123testhash2", "a123testhash3"] 198 revisions_list = [102, 104, 106] 199 200 # Simulate behavior of `AddTryjob()` when successfully launched a 201 # tryjob for the updated packages. 202 @test_helpers.CallCountsToMockFunctions 203 def MockAddTryjob( 204 call_count, 205 _packages, 206 _git_hash, 207 _revision, 208 _chroot_path, 209 _extra_cls, 210 _options, 211 _builder, 212 _svn_revision, 213 ): 214 if call_count < 2: 215 return {"rev": revisions_list[call_count], "status": "pending"} 216 217 # Simulate an exception happened along the way when updating the 218 # packages' `LLVM_NEXT_HASH`. 219 if call_count == 2: 220 raise ValueError("Unable to launch tryjob") 221 222 assert False, "Called `AddTryjob()` more than expected." 223 224 # Use the test function to simulate `AddTryjob()`. 225 mock_add_tryjob.side_effect = MockAddTryjob 226 227 start = 100 228 end = 110 229 230 bisection_contents = {"start": start, "end": end, "jobs": []} 231 232 args_output = test_helpers.ArgsOutputTest() 233 234 packages = ["sys-devel/llvm"] 235 236 # Create a temporary .JSON file to simulate a status file for bisection. 237 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 238 with open(temp_json_file, "w", encoding="utf-8") as f: 239 test_helpers.WritePrettyJsonFile(bisection_contents, f) 240 241 # Verify that the status file is updated when an exception happened 242 # when attempting to launch a revision (i.e. progress is not lost). 243 with self.assertRaises(ValueError) as err: 244 llvm_bisection.Bisect( 245 revisions_list, 246 git_hash_list, 247 bisection_contents, 248 temp_json_file, 249 packages, 250 args_output.chromeos_path, 251 args_output.extra_change_lists, 252 args_output.options, 253 args_output.builders, 254 ) 255 256 expected_bisection_contents = { 257 "start": start, 258 "end": end, 259 "jobs": [ 260 {"rev": revisions_list[0], "status": "pending"}, 261 {"rev": revisions_list[1], "status": "pending"}, 262 ], 263 } 264 265 # Verify that the launched tryjobs were added to the status file 266 # when an exception happened. 267 with open(temp_json_file, encoding="utf-8") as f: 268 json_contents = json.load(f) 269 270 self.assertEqual(json_contents, expected_bisection_contents) 271 272 self.assertEqual(str(err.exception), "Unable to launch tryjob") 273 274 self.assertEqual(mock_add_tryjob.call_count, 3) 275 276 @mock.patch.object(subprocess, "check_output", return_value=None) 277 @mock.patch.object( 278 get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4" 279 ) 280 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 281 @mock.patch.object(llvm_bisection, "GetRemainingRange") 282 @mock.patch.object(llvm_bisection, "LoadStatusFile") 283 @mock.patch.object(chroot, "VerifyChromeOSRoot") 284 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 285 def testMainPassed( 286 self, 287 mock_outside_chroot, 288 mock_chromeos_root, 289 mock_load_status_file, 290 mock_get_range, 291 mock_get_revision_and_hash_list, 292 _mock_get_bad_llvm_hash, 293 mock_abandon_cl, 294 ): 295 start = 500 296 end = 502 297 cl = 1 298 299 bisect_state = { 300 "start": start, 301 "end": end, 302 "jobs": [{"rev": 501, "status": "bad", "cl": cl}], 303 } 304 305 skip_revisions = {501} 306 pending_revisions = {} 307 308 mock_load_status_file.return_value = bisect_state 309 310 mock_get_range.return_value = ( 311 start, 312 end, 313 pending_revisions, 314 skip_revisions, 315 ) 316 317 mock_get_revision_and_hash_list.return_value = [], [] 318 319 args_output = test_helpers.ArgsOutputTest() 320 args_output.start_rev = start 321 args_output.end_rev = end 322 args_output.parallel = 3 323 args_output.src_path = None 324 args_output.chromeos_path = "somepath" 325 args_output.cleanup = True 326 327 self.assertEqual( 328 llvm_bisection.main(args_output), 329 llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value, 330 ) 331 332 mock_chromeos_root.assert_called_once() 333 334 mock_outside_chroot.assert_called_once() 335 336 mock_load_status_file.assert_called_once() 337 338 mock_get_range.assert_called_once() 339 340 mock_get_revision_and_hash_list.assert_called_once() 341 342 mock_abandon_cl.assert_called_once() 343 self.assertEqual( 344 mock_abandon_cl.call_args, 345 mock.call( 346 [ 347 os.path.join( 348 args_output.chromeos_path, "chromite/bin/gerrit" 349 ), 350 "abandon", 351 str(cl), 352 ], 353 stderr=subprocess.STDOUT, 354 encoding="utf-8", 355 ), 356 ) 357 358 @mock.patch.object(llvm_bisection, "LoadStatusFile") 359 @mock.patch.object(chroot, "VerifyChromeOSRoot") 360 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 361 def testMainFailedWithInvalidRange( 362 self, mock_chromeos_root, mock_outside_chroot, mock_load_status_file 363 ): 364 start = 500 365 end = 502 366 367 bisect_state = { 368 "start": start - 1, 369 "end": end, 370 } 371 372 mock_load_status_file.return_value = bisect_state 373 374 args_output = test_helpers.ArgsOutputTest() 375 args_output.start_rev = start 376 args_output.end_rev = end 377 args_output.parallel = 3 378 args_output.src_path = None 379 380 with self.assertRaises(ValueError) as err: 381 llvm_bisection.main(args_output) 382 383 error_message = ( 384 f"The start {start} or the end {end} version provided is " 385 f'different than "start" {bisect_state["start"]} or "end" ' 386 f'{bisect_state["end"]} in the .JSON file' 387 ) 388 389 self.assertEqual(str(err.exception), error_message) 390 391 mock_chromeos_root.assert_called_once() 392 393 mock_outside_chroot.assert_called_once() 394 395 mock_load_status_file.assert_called_once() 396 397 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 398 @mock.patch.object(llvm_bisection, "GetRemainingRange") 399 @mock.patch.object(llvm_bisection, "LoadStatusFile") 400 @mock.patch.object(chroot, "VerifyChromeOSRoot") 401 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 402 def testMainFailedWithPendingBuilds( 403 self, 404 mock_chromeos_root, 405 mock_outside_chroot, 406 mock_load_status_file, 407 mock_get_range, 408 mock_get_revision_and_hash_list, 409 ): 410 start = 500 411 end = 502 412 rev = 501 413 414 bisect_state = { 415 "start": start, 416 "end": end, 417 "jobs": [{"rev": rev, "status": "pending"}], 418 } 419 420 skip_revisions = {} 421 pending_revisions = {rev} 422 423 mock_load_status_file.return_value = bisect_state 424 425 mock_get_range.return_value = ( 426 start, 427 end, 428 pending_revisions, 429 skip_revisions, 430 ) 431 432 mock_get_revision_and_hash_list.return_value = [], [] 433 434 args_output = test_helpers.ArgsOutputTest() 435 args_output.start_rev = start 436 args_output.end_rev = end 437 args_output.parallel = 3 438 args_output.src_path = None 439 440 with self.assertRaises(ValueError) as err: 441 llvm_bisection.main(args_output) 442 443 error_message = ( 444 f"No revisions between start {start} and end {end} to " 445 "create tryjobs\nThe following tryjobs are pending:\n" 446 f"{rev}\n" 447 ) 448 449 self.assertEqual(str(err.exception), error_message) 450 451 mock_chromeos_root.assert_called_once() 452 453 mock_outside_chroot.assert_called_once() 454 455 mock_load_status_file.assert_called_once() 456 457 mock_get_range.assert_called_once() 458 459 mock_get_revision_and_hash_list.assert_called_once() 460 461 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 462 @mock.patch.object(llvm_bisection, "GetRemainingRange") 463 @mock.patch.object(llvm_bisection, "LoadStatusFile") 464 @mock.patch.object(chroot, "VerifyChromeOSRoot") 465 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 466 def testMainFailedWithDuplicateBuilds( 467 self, 468 mock_outside_chroot, 469 mock_chromeos_root, 470 mock_load_status_file, 471 mock_get_range, 472 mock_get_revision_and_hash_list, 473 ): 474 start = 500 475 end = 502 476 rev = 501 477 git_hash = "a123testhash1" 478 479 bisect_state = { 480 "start": start, 481 "end": end, 482 "jobs": [{"rev": rev, "status": "pending"}], 483 } 484 485 skip_revisions = {} 486 pending_revisions = {rev} 487 488 mock_load_status_file.return_value = bisect_state 489 490 mock_get_range.return_value = ( 491 start, 492 end, 493 pending_revisions, 494 skip_revisions, 495 ) 496 497 mock_get_revision_and_hash_list.return_value = [rev], [git_hash] 498 499 args_output = test_helpers.ArgsOutputTest() 500 args_output.start_rev = start 501 args_output.end_rev = end 502 args_output.parallel = 3 503 args_output.src_path = None 504 505 with self.assertRaises(ValueError) as err: 506 llvm_bisection.main(args_output) 507 508 error_message = 'Revision %d exists already in "jobs"' % rev 509 self.assertEqual(str(err.exception), error_message) 510 511 mock_chromeos_root.assert_called_once() 512 513 mock_outside_chroot.assert_called_once() 514 515 mock_load_status_file.assert_called_once() 516 517 mock_get_range.assert_called_once() 518 519 mock_get_revision_and_hash_list.assert_called_once() 520 521 @mock.patch.object(subprocess, "check_output", return_value=None) 522 @mock.patch.object( 523 get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4" 524 ) 525 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 526 @mock.patch.object(llvm_bisection, "GetRemainingRange") 527 @mock.patch.object(llvm_bisection, "LoadStatusFile") 528 @mock.patch.object(chroot, "VerifyChromeOSRoot") 529 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 530 def testMainFailedToAbandonCL( 531 self, 532 mock_outside_chroot, 533 mock_chromeos_root, 534 mock_load_status_file, 535 mock_get_range, 536 mock_get_revision_and_hash_list, 537 _mock_get_bad_llvm_hash, 538 mock_abandon_cl, 539 ): 540 start = 500 541 end = 502 542 543 bisect_state = { 544 "start": start, 545 "end": end, 546 "jobs": [{"rev": 501, "status": "bad", "cl": 0}], 547 } 548 549 skip_revisions = {501} 550 pending_revisions = {} 551 552 mock_load_status_file.return_value = bisect_state 553 554 mock_get_range.return_value = ( 555 start, 556 end, 557 pending_revisions, 558 skip_revisions, 559 ) 560 561 mock_get_revision_and_hash_list.return_value = ([], []) 562 563 error_message = "Error message." 564 mock_abandon_cl.side_effect = subprocess.CalledProcessError( 565 returncode=1, cmd=[], output=error_message 566 ) 567 568 args_output = test_helpers.ArgsOutputTest() 569 args_output.start_rev = start 570 args_output.end_rev = end 571 args_output.parallel = 3 572 args_output.src_path = None 573 args_output.cleanup = True 574 575 with self.assertRaises(subprocess.CalledProcessError) as err: 576 llvm_bisection.main(args_output) 577 578 self.assertEqual(err.exception.output, error_message) 579 580 mock_chromeos_root.assert_called_once() 581 582 mock_outside_chroot.assert_called_once() 583 584 mock_load_status_file.assert_called_once() 585 586 mock_get_range.assert_called_once() 587 588 589if __name__ == "__main__": 590 unittest.main() 591