1"""Tests for job_directories.""" 2 3from __future__ import absolute_import 4from __future__ import division 5from __future__ import print_function 6 7import contextlib 8import datetime 9import os 10import shutil 11import tempfile 12import unittest 13from unittest.mock import patch 14 15import common 16 17from autotest_lib.site_utils import job_directories 18from autotest_lib.client.common_lib import time_utils 19 20 21class SwarmingJobDirectoryTestCase(unittest.TestCase): 22 """Tests SwarmingJobDirectory.""" 23 24 def test_get_job_directories_legacy(self): 25 with _change_to_tempdir(): 26 os.makedirs("swarming-3e4391423c3a4311/b") 27 os.mkdir("not-a-swarming-dir") 28 results = job_directories.SwarmingJobDirectory.get_job_directories() 29 self.assertEqual(set(results), {"swarming-3e4391423c3a4311"}) 30 31 def test_get_job_directories(self): 32 with _change_to_tempdir(): 33 os.makedirs("swarming-3e4391423c3a4310/1") 34 os.makedirs("swarming-3e4391423c3a4310/0") 35 open("swarming-3e4391423c3a4310/1/.ready_for_offload", 36 'w+').close() 37 os.makedirs("swarming-3e4391423c3a4310/a") 38 open("swarming-3e4391423c3a4310/a/.ready_for_offload", 39 'w+').close() 40 os.makedirs("swarming-34391423c3a4310/1/test_id") 41 os.makedirs("swarming-34391423c3a4310/1/test_id2") 42 open("swarming-34391423c3a4310/1/test_id/.ready_for_offload", 43 'w+').close() 44 open("swarming-34391423c3a4310/1/test_id2/.ready_for_offload", 45 'w+').close() 46 os.mkdir("not-a-swarming-dir") 47 results = job_directories.SwarmingJobDirectory.get_job_directories() 48 self.assertEqual( 49 set(results), { 50 "swarming-3e4391423c3a4310/1", 51 "swarming-3e4391423c3a4310/a", 52 "swarming-34391423c3a4310/1/test_id", 53 "swarming-34391423c3a4310/1/test_id2" 54 }) 55 56 57class GetJobIDOrTaskID(unittest.TestCase): 58 """Tests get_job_id_or_task_id.""" 59 60 def test_legacy_swarming_path(self): 61 self.assertEqual( 62 "3e4391423c3a4311", 63 job_directories.get_job_id_or_task_id( 64 "/autotest/results/swarming-3e4391423c3a4311"), 65 ) 66 self.assertEqual( 67 "3e4391423c3a4311", 68 job_directories.get_job_id_or_task_id( 69 "swarming-3e4391423c3a4311"), 70 ) 71 72 def test_swarming_path(self): 73 self.assertEqual( 74 "3e4391423c3a4311", 75 job_directories.get_job_id_or_task_id( 76 "/autotest/results/swarming-3e4391423c3a4310/1"), 77 ) 78 self.assertEqual( 79 "3e4391423c3a431f", 80 job_directories.get_job_id_or_task_id( 81 "swarming-3e4391423c3a4310/f"), 82 ) 83 84 85class JobDirectorySubclassTests(unittest.TestCase): 86 """Test specific to RegularJobDirectory and SpecialJobDirectory. 87 88 This provides coverage for the implementation in both 89 RegularJobDirectory and SpecialJobDirectory. 90 91 """ 92 93 def setUp(self): 94 super(JobDirectorySubclassTests, self).setUp() 95 patcher = patch.object(job_directories, '_AFE') 96 self._mock = patcher.start() 97 self.addCleanup(patcher.stop) 98 99 def test_regular_job_fields(self): 100 """Test the constructor for `RegularJobDirectory`. 101 102 Construct a regular job, and assert that the `dirname` 103 and `_id` attributes are set as expected. 104 105 """ 106 resultsdir = '118-fubar' 107 job = job_directories.RegularJobDirectory(resultsdir) 108 self.assertEqual(job.dirname, resultsdir) 109 self.assertEqual(job._id, '118') 110 111 def test_special_job_fields(self): 112 """Test the constructor for `SpecialJobDirectory`. 113 114 Construct a special job, and assert that the `dirname` 115 and `_id` attributes are set as expected. 116 117 """ 118 destdir = 'hosts/host1' 119 resultsdir = destdir + '/118-reset' 120 job = job_directories.SpecialJobDirectory(resultsdir) 121 self.assertEqual(job.dirname, resultsdir) 122 self.assertEqual(job._id, '118') 123 124 def _check_finished_job(self, jobtime, hqetimes, expected): 125 """Mock and test behavior of a finished job. 126 127 Initialize the mocks for a call to 128 `get_timestamp_if_finished()`, then simulate one call. 129 Assert that the returned timestamp matches the passed 130 in expected value. 131 132 @param jobtime Time used to construct a _MockJob object. 133 @param hqetimes List of times used to construct 134 _MockHostQueueEntry objects. 135 @param expected Expected time to be returned by 136 get_timestamp_if_finished 137 138 """ 139 job = job_directories.RegularJobDirectory('118-fubar') 140 self._mock.get_jobs.return_value = [_MockJob(jobtime)] 141 142 self._mock.get_host_queue_entries.return_value = ([ 143 _MockHostQueueEntry(t) for t in hqetimes 144 ]) 145 146 self.assertEqual(expected, job.get_timestamp_if_finished()) 147 self._mock.get_jobs.assert_called_with(id=job._id, finished=True) 148 self._mock.get_host_queue_entries.assert_called_with( 149 finished_on__isnull=False, job_id=job._id) 150 151 def test_finished_regular_job(self): 152 """Test getting the timestamp for a finished regular job. 153 154 Tests the return value for 155 `RegularJobDirectory.get_timestamp_if_finished()` when 156 the AFE indicates the job is finished. 157 158 """ 159 created_timestamp = make_timestamp(1, True) 160 hqe_timestamp = make_timestamp(0, True) 161 self._check_finished_job(created_timestamp, 162 [hqe_timestamp], 163 hqe_timestamp) 164 165 def test_finished_regular_job_multiple_hqes(self): 166 """Test getting the timestamp for a regular job with multiple hqes. 167 168 Tests the return value for 169 `RegularJobDirectory.get_timestamp_if_finished()` when 170 the AFE indicates the job is finished and the job has multiple host 171 queue entries. 172 173 Tests that the returned timestamp is the latest timestamp in 174 the list of HQEs, regardless of the returned order. 175 176 """ 177 created_timestamp = make_timestamp(2, True) 178 older_hqe_timestamp = make_timestamp(1, True) 179 newer_hqe_timestamp = make_timestamp(0, True) 180 hqe_list = [older_hqe_timestamp, 181 newer_hqe_timestamp] 182 self._check_finished_job(created_timestamp, 183 hqe_list, 184 newer_hqe_timestamp) 185 hqe_list.reverse() 186 self._check_finished_job(created_timestamp, 187 hqe_list, 188 newer_hqe_timestamp) 189 190 def test_finished_regular_job_null_finished_times(self): 191 """Test getting the timestamp for an aborted regular job. 192 193 Tests the return value for 194 `RegularJobDirectory.get_timestamp_if_finished()` when 195 the AFE indicates the job is finished and the job has aborted host 196 queue entries. 197 198 """ 199 timestamp = make_timestamp(0, True) 200 self._check_finished_job(timestamp, [], timestamp) 201 202 def test_unfinished_regular_job(self): 203 """Test getting the timestamp for an unfinished regular job. 204 205 Tests the return value for 206 `RegularJobDirectory.get_timestamp_if_finished()` when 207 the AFE indicates the job is not finished. 208 209 """ 210 job = job_directories.RegularJobDirectory('118-fubar') 211 self._mock.get_jobs.return_value = [] 212 self.assertIsNone(job.get_timestamp_if_finished()) 213 self._mock.get_jobs.assert_called_with(id=job._id, finished=True) 214 215 def test_finished_special_job(self): 216 """Test getting the timestamp for a finished special job. 217 218 Tests the return value for 219 `SpecialJobDirectory.get_timestamp_if_finished()` when 220 the AFE indicates the job is finished. 221 222 """ 223 job = job_directories.SpecialJobDirectory( 224 'hosts/host1/118-reset') 225 timestamp = make_timestamp(0, True) 226 self._mock.get_special_tasks.return_value = ([ 227 _MockSpecialTask(timestamp) 228 ]) 229 self.assertEqual(timestamp, 230 job.get_timestamp_if_finished()) 231 self._mock.get_special_tasks.assert_called_with(id=job._id, 232 is_complete=True) 233 234 def test_unfinished_special_job(self): 235 """Test getting the timestamp for an unfinished special job. 236 237 Tests the return value for 238 `SpecialJobDirectory.get_timestamp_if_finished()` when 239 the AFE indicates the job is not finished. 240 241 """ 242 job = job_directories.SpecialJobDirectory( 243 'hosts/host1/118-reset') 244 self._mock.get_special_tasks.return_value = [] 245 self.assertIsNone(job.get_timestamp_if_finished()) 246 self._mock.get_special_tasks.assert_called_with(id=job._id, 247 is_complete=True) 248 249 250class JobExpirationTests(unittest.TestCase): 251 """Tests to exercise `job_directories.is_job_expired()`.""" 252 253 def test_expired(self): 254 """Test detection of an expired job.""" 255 timestamp = make_timestamp(_TEST_EXPIRATION_AGE, True) 256 self.assertTrue( 257 job_directories.is_job_expired( 258 _TEST_EXPIRATION_AGE, timestamp)) 259 260 def test_alive(self): 261 """Test detection of a job that's not expired.""" 262 # N.B. This test may fail if its run time exceeds more than 263 # about _MARGIN_SECS seconds. 264 timestamp = make_timestamp(_TEST_EXPIRATION_AGE, False) 265 self.assertFalse( 266 job_directories.is_job_expired( 267 _TEST_EXPIRATION_AGE, timestamp)) 268 269 270# When constructing sample time values for testing expiration, 271# allow this many seconds between the expiration time and the 272# current time. 273_MARGIN_SECS = 10.0 274# Test value to use for `days_old`, if nothing else is required. 275_TEST_EXPIRATION_AGE = 7 276 277 278class _MockJob(object): 279 """Class to mock the return value of `AFE.get_jobs()`.""" 280 def __init__(self, created): 281 self.created_on = created 282 283 284class _MockHostQueueEntry(object): 285 """Class to mock the return value of `AFE.get_host_queue_entries()`.""" 286 def __init__(self, finished): 287 self.finished_on = finished 288 289 290class _MockSpecialTask(object): 291 """Class to mock the return value of `AFE.get_special_tasks()`.""" 292 def __init__(self, finished): 293 self.time_finished = finished 294 295 296@contextlib.contextmanager 297def _change_to_tempdir(): 298 old_dir = os.getcwd() 299 tempdir = tempfile.mkdtemp('job_directories_unittest') 300 try: 301 os.chdir(tempdir) 302 yield 303 finally: 304 os.chdir(old_dir) 305 shutil.rmtree(tempdir) 306 307 308def make_timestamp(age_limit, is_expired): 309 """Create a timestamp for use by `job_directories.is_job_expired()`. 310 311 The timestamp will meet the syntactic requirements for 312 timestamps used as input to `is_job_expired()`. If 313 `is_expired` is true, the timestamp will be older than 314 `age_limit` days before the current time; otherwise, the 315 date will be younger. 316 317 @param age_limit The number of days before expiration of the 318 target timestamp. 319 @param is_expired Whether the timestamp should be expired 320 relative to `age_limit`. 321 322 """ 323 seconds = -_MARGIN_SECS 324 if is_expired: 325 seconds = -seconds 326 delta = datetime.timedelta(days=age_limit, seconds=seconds) 327 reference_time = datetime.datetime.now() - delta 328 return reference_time.strftime(time_utils.TIME_FMT) 329 330 331if __name__ == '__main__': 332 unittest.main() 333