1#!/usr/bin/env python3 2# Copyright 2019 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Tests for auto bisection of LLVM.""" 7 8import json 9import os 10import subprocess 11import time 12import traceback 13import unittest 14from unittest import mock 15 16import auto_llvm_bisection 17import chroot 18import llvm_bisection 19import test_helpers 20import update_tryjob_status 21 22 23class AutoLLVMBisectionTest(unittest.TestCase): 24 """Unittests for auto bisection of LLVM.""" 25 26 @mock.patch.object(chroot, "VerifyChromeOSRoot") 27 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 28 @mock.patch.object( 29 llvm_bisection, 30 "GetCommandLineArgs", 31 return_value=test_helpers.ArgsOutputTest(), 32 ) 33 @mock.patch.object(time, "sleep") 34 @mock.patch.object(traceback, "print_exc") 35 @mock.patch.object(llvm_bisection, "main") 36 @mock.patch.object(os.path, "isfile") 37 @mock.patch.object(auto_llvm_bisection, "open") 38 @mock.patch.object(json, "load") 39 @mock.patch.object(auto_llvm_bisection, "GetBuildResult") 40 @mock.patch.object(os, "rename") 41 def testAutoLLVMBisectionPassed( 42 self, 43 # pylint: disable=unused-argument 44 mock_rename, 45 mock_get_build_result, 46 mock_json_load, 47 # pylint: disable=unused-argument 48 mock_open, 49 mock_isfile, 50 mock_llvm_bisection, 51 mock_traceback, 52 mock_sleep, 53 mock_get_args, 54 mock_outside_chroot, 55 mock_chromeos_root, 56 ): 57 mock_isfile.side_effect = [False, False, True, True] 58 mock_llvm_bisection.side_effect = [ 59 0, 60 ValueError("Failed to launch more tryjobs."), 61 llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value, 62 ] 63 mock_json_load.return_value = { 64 "start": 369410, 65 "end": 369420, 66 "jobs": [ 67 { 68 "buildbucket_id": 12345, 69 "rev": 369411, 70 "status": update_tryjob_status.TryjobStatus.PENDING.value, 71 } 72 ], 73 } 74 mock_get_build_result.return_value = ( 75 update_tryjob_status.TryjobStatus.GOOD.value 76 ) 77 78 # Verify the excpetion is raised when successfully found the bad 79 # revision. Uses `sys.exit(0)` to indicate success. 80 with self.assertRaises(SystemExit) as err: 81 auto_llvm_bisection.main() 82 83 self.assertEqual(err.exception.code, 0) 84 85 mock_outside_chroot.assert_called_once() 86 mock_get_args.assert_called_once() 87 self.assertEqual(mock_isfile.call_count, 3) 88 self.assertEqual(mock_llvm_bisection.call_count, 3) 89 mock_traceback.assert_called_once() 90 mock_sleep.assert_called_once() 91 92 @mock.patch.object(chroot, "VerifyChromeOSRoot") 93 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 94 @mock.patch.object(time, "sleep") 95 @mock.patch.object(traceback, "print_exc") 96 @mock.patch.object(llvm_bisection, "main") 97 @mock.patch.object(os.path, "isfile") 98 @mock.patch.object( 99 llvm_bisection, 100 "GetCommandLineArgs", 101 return_value=test_helpers.ArgsOutputTest(), 102 ) 103 def testFailedToStartBisection( 104 self, 105 mock_get_args, 106 mock_isfile, 107 mock_llvm_bisection, 108 mock_traceback, 109 mock_sleep, 110 mock_outside_chroot, 111 mock_chromeos_root, 112 ): 113 mock_isfile.return_value = False 114 mock_llvm_bisection.side_effect = ValueError( 115 "Failed to launch more tryjobs." 116 ) 117 118 # Verify the exception is raised when the number of attempts to launched 119 # more tryjobs is exceeded, so unable to continue 120 # bisection. 121 with self.assertRaises(SystemExit) as err: 122 auto_llvm_bisection.main() 123 124 self.assertEqual(err.exception.code, "Unable to continue bisection.") 125 126 mock_chromeos_root.assert_called_once() 127 mock_outside_chroot.assert_called_once() 128 mock_get_args.assert_called_once() 129 self.assertEqual(mock_isfile.call_count, 2) 130 self.assertEqual(mock_llvm_bisection.call_count, 3) 131 self.assertEqual(mock_traceback.call_count, 3) 132 self.assertEqual(mock_sleep.call_count, 2) 133 134 @mock.patch.object(chroot, "VerifyChromeOSRoot") 135 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 136 @mock.patch.object( 137 llvm_bisection, 138 "GetCommandLineArgs", 139 return_value=test_helpers.ArgsOutputTest(), 140 ) 141 @mock.patch.object(time, "time") 142 @mock.patch.object(time, "sleep") 143 @mock.patch.object(os.path, "isfile") 144 @mock.patch.object(auto_llvm_bisection, "open") 145 @mock.patch.object(json, "load") 146 @mock.patch.object(auto_llvm_bisection, "GetBuildResult") 147 def testFailedToUpdatePendingTryJobs( 148 self, 149 mock_get_build_result, 150 mock_json_load, 151 # pylint: disable=unused-argument 152 mock_open, 153 mock_isfile, 154 mock_sleep, 155 mock_time, 156 mock_get_args, 157 mock_outside_chroot, 158 mock_chromeos_root, 159 ): 160 # Simulate behavior of `time.time()` for time passed. 161 @test_helpers.CallCountsToMockFunctions 162 def MockTimePassed(call_count): 163 if call_count < 3: 164 return call_count 165 166 assert False, "Called `time.time()` more than expected." 167 168 mock_isfile.return_value = True 169 mock_json_load.return_value = { 170 "start": 369410, 171 "end": 369420, 172 "jobs": [ 173 { 174 "buildbucket_id": 12345, 175 "rev": 369411, 176 "status": update_tryjob_status.TryjobStatus.PENDING.value, 177 } 178 ], 179 } 180 mock_get_build_result.return_value = None 181 mock_time.side_effect = MockTimePassed 182 # Reduce the polling limit for the test case to terminate faster. 183 auto_llvm_bisection.POLLING_LIMIT_SECS = 1 184 185 # Verify the exception is raised when unable to update tryjobs whose 186 # 'status' value is 'pending'. 187 with self.assertRaises(SystemExit) as err: 188 auto_llvm_bisection.main() 189 190 self.assertEqual( 191 err.exception.code, "Failed to update pending tryjobs." 192 ) 193 194 mock_outside_chroot.assert_called_once() 195 mock_get_args.assert_called_once() 196 self.assertEqual(mock_isfile.call_count, 2) 197 mock_sleep.assert_called_once() 198 self.assertEqual(mock_time.call_count, 3) 199 200 @mock.patch.object(subprocess, "check_output") 201 def testGetBuildResult(self, mock_chroot_command): 202 buildbucket_id = 192 203 status = auto_llvm_bisection.BuilderStatus.PASS.value 204 tryjob_contents = {buildbucket_id: {"status": status}} 205 mock_chroot_command.return_value = json.dumps(tryjob_contents) 206 chroot_path = "/some/path/to/chroot" 207 208 self.assertEqual( 209 auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id), 210 update_tryjob_status.TryjobStatus.GOOD.value, 211 ) 212 213 mock_chroot_command.assert_called_once_with( 214 [ 215 "cros", 216 "buildresult", 217 "--buildbucket-id", 218 str(buildbucket_id), 219 "--report", 220 "json", 221 ], 222 cwd="/some/path/to/chroot", 223 stderr=subprocess.STDOUT, 224 encoding="utf-8", 225 ) 226 227 @mock.patch.object(subprocess, "check_output") 228 def testGetBuildResultPassedWithUnstartedTryjob(self, mock_chroot_command): 229 buildbucket_id = 192 230 chroot_path = "/some/path/to/chroot" 231 mock_chroot_command.side_effect = subprocess.CalledProcessError( 232 returncode=1, cmd=[], output="No build found. Perhaps not started" 233 ) 234 auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id) 235 mock_chroot_command.assert_called_once_with( 236 [ 237 "cros", 238 "buildresult", 239 "--buildbucket-id", 240 "192", 241 "--report", 242 "json", 243 ], 244 cwd=chroot_path, 245 stderr=subprocess.STDOUT, 246 encoding="utf-8", 247 ) 248 249 @mock.patch.object(subprocess, "check_output") 250 def testGetBuildReusultFailedWithInvalidBuildStatus( 251 self, mock_chroot_command 252 ): 253 chroot_path = "/some/path/to/chroot" 254 buildbucket_id = 50 255 invalid_build_status = "querying" 256 tryjob_contents = {buildbucket_id: {"status": invalid_build_status}} 257 mock_chroot_command.return_value = json.dumps(tryjob_contents) 258 259 # Verify an exception is raised when the return value of `cros 260 # buildresult` is not in the `builder_status_mapping`. 261 with self.assertRaises(ValueError) as err: 262 auto_llvm_bisection.GetBuildResult(chroot_path, buildbucket_id) 263 264 self.assertEqual( 265 str(err.exception), 266 '"cros buildresult" return value is invalid: %s' 267 % invalid_build_status, 268 ) 269 270 mock_chroot_command.assert_called_once_with( 271 [ 272 "cros", 273 "buildresult", 274 "--buildbucket-id", 275 str(buildbucket_id), 276 "--report", 277 "json", 278 ], 279 cwd=chroot_path, 280 stderr=subprocess.STDOUT, 281 encoding="utf-8", 282 ) 283 284 285if __name__ == "__main__": 286 unittest.main() 287