1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright 2013 The ChromiumOS Authors 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Testing of benchmark_run.""" 9 10 11import inspect 12import unittest 13import unittest.mock as mock 14 15from benchmark import Benchmark 16import benchmark_run 17from cros_utils import logger 18from label import MockLabel 19from machine_manager import MachineManager 20from machine_manager import MockCrosMachine 21from machine_manager import MockMachineManager 22from results_cache import CacheConditions 23from results_cache import MockResultsCache 24from results_cache import Result 25from results_cache import ResultsCache 26from suite_runner import MockSuiteRunner 27from suite_runner import SuiteRunner 28 29 30class BenchmarkRunTest(unittest.TestCase): 31 """Unit tests for the BenchmarkRun class and all of its methods.""" 32 33 def setUp(self): 34 self.status = [] 35 self.called_ReadCache = None 36 self.log_error = [] 37 self.log_output = [] 38 self.err_msg = None 39 self.test_benchmark = Benchmark( 40 "page_cycler.netsim.top_10", # name 41 "page_cycler.netsim.top_10", # test_name 42 "", # test_args 43 1, # iterations 44 False, # rm_chroot_tmp 45 "", # perf_args 46 suite="telemetry_Crosperf", 47 ) # suite 48 49 self.test_label = MockLabel( 50 "test1", 51 "build", 52 "image1", 53 "autotest_dir", 54 "debug_dir", 55 "/tmp/test_benchmark_run", 56 "x86-alex", 57 "chromeos2-row1-rack4-host9.cros", 58 image_args="", 59 cache_dir="", 60 cache_only=False, 61 log_level="average", 62 compiler="gcc", 63 crosfleet=False, 64 ) 65 66 self.test_cache_conditions = [ 67 CacheConditions.CACHE_FILE_EXISTS, 68 CacheConditions.CHECKSUMS_MATCH, 69 ] 70 71 self.mock_logger = logger.GetLogger(log_dir="", mock=True) 72 73 self.mock_machine_manager = mock.Mock(spec=MachineManager) 74 75 def testDryRun(self): 76 my_label = MockLabel( 77 "test1", 78 "build", 79 "image1", 80 "autotest_dir", 81 "debug_dir", 82 "/tmp/test_benchmark_run", 83 "x86-alex", 84 "chromeos2-row1-rack4-host9.cros", 85 image_args="", 86 cache_dir="", 87 cache_only=False, 88 log_level="average", 89 compiler="gcc", 90 crosfleet=False, 91 ) 92 93 logging_level = "average" 94 m = MockMachineManager("/tmp/chromeos_root", 0, logging_level, "") 95 m.AddMachine("chromeos2-row1-rack4-host9.cros") 96 bench = Benchmark( 97 "page_cycler.netsim.top_10", # name 98 "page_cycler.netsim.top_10", # test_name 99 "", # test_args 100 1, # iterations 101 False, # rm_chroot_tmp 102 "", # perf_args 103 suite="telemetry_Crosperf", 104 ) # suite 105 dut_conf = { 106 "cooldown_time": 5, 107 "cooldown_temp": 45, 108 "governor": "powersave", 109 "cpu_usage": "big_only", 110 "cpu_freq_pct": 80, 111 } 112 b = benchmark_run.MockBenchmarkRun( 113 "test run", 114 bench, 115 my_label, 116 1, 117 [], 118 m, 119 logger.GetLogger(), 120 logging_level, 121 "", 122 dut_conf, 123 ) 124 b.cache = MockResultsCache() 125 b.suite_runner = MockSuiteRunner() 126 b.start() 127 128 # Make sure the arguments to BenchmarkRun.__init__ have not changed 129 # since the last time this test was updated: 130 args_list = [ 131 "self", 132 "name", 133 "benchmark", 134 "label", 135 "iteration", 136 "cache_conditions", 137 "machine_manager", 138 "logger_to_use", 139 "log_level", 140 "share_cache", 141 "dut_config", 142 ] 143 arg_spec = inspect.getfullargspec(benchmark_run.BenchmarkRun.__init__) 144 self.assertEqual(len(arg_spec.args), len(args_list)) 145 self.assertEqual(arg_spec.args, args_list) 146 147 def test_init(self): 148 # Nothing really worth testing here; just field assignments. 149 pass 150 151 def test_read_cache(self): 152 # Nothing really worth testing here, either. 153 pass 154 155 def test_run(self): 156 br = benchmark_run.BenchmarkRun( 157 "test_run", 158 self.test_benchmark, 159 self.test_label, 160 1, 161 self.test_cache_conditions, 162 self.mock_machine_manager, 163 self.mock_logger, 164 "average", 165 "", 166 {}, 167 ) 168 169 def MockLogOutput(msg, print_to_console=False): 170 """Helper function for test_run.""" 171 del print_to_console 172 self.log_output.append(msg) 173 174 def MockLogError(msg, print_to_console=False): 175 """Helper function for test_run.""" 176 del print_to_console 177 self.log_error.append(msg) 178 179 def MockRecordStatus(msg): 180 """Helper function for test_run.""" 181 self.status.append(msg) 182 183 def FakeReadCache(): 184 """Helper function for test_run.""" 185 br.cache = mock.Mock(spec=ResultsCache) 186 self.called_ReadCache = True 187 return 0 188 189 def FakeReadCacheSucceed(): 190 """Helper function for test_run.""" 191 br.cache = mock.Mock(spec=ResultsCache) 192 br.result = mock.Mock(spec=Result) 193 br.result.out = "result.out stuff" 194 br.result.err = "result.err stuff" 195 br.result.retval = 0 196 self.called_ReadCache = True 197 return 0 198 199 def FakeReadCacheException(): 200 """Helper function for test_run.""" 201 raise RuntimeError( 202 "This is an exception test; it is supposed to happen" 203 ) 204 205 def FakeAcquireMachine(): 206 """Helper function for test_run.""" 207 mock_machine = MockCrosMachine( 208 "chromeos1-row3-rack5-host7.cros", "chromeos", "average" 209 ) 210 return mock_machine 211 212 def FakeRunTest(_machine): 213 """Helper function for test_run.""" 214 mock_result = mock.Mock(spec=Result) 215 mock_result.retval = 0 216 return mock_result 217 218 def FakeRunTestFail(_machine): 219 """Helper function for test_run.""" 220 mock_result = mock.Mock(spec=Result) 221 mock_result.retval = 1 222 return mock_result 223 224 def ResetTestValues(): 225 """Helper function for test_run.""" 226 self.log_output = [] 227 self.log_error = [] 228 self.status = [] 229 br.result = None 230 self.called_ReadCache = False 231 232 # Assign all the fake functions to the appropriate objects. 233 br.logger().LogOutput = MockLogOutput 234 br.logger().LogError = MockLogError 235 br.timeline.Record = MockRecordStatus 236 br.ReadCache = FakeReadCache 237 br.RunTest = FakeRunTest 238 br.AcquireMachine = FakeAcquireMachine 239 240 # First test: No cache hit, all goes well. 241 ResetTestValues() 242 br.run() 243 self.assertTrue(self.called_ReadCache) 244 self.assertEqual( 245 self.log_output, 246 [ 247 "test_run: No cache hit.", 248 "Releasing machine: chromeos1-row3-rack5-host7.cros", 249 "Released machine: chromeos1-row3-rack5-host7.cros", 250 ], 251 ) 252 self.assertEqual(len(self.log_error), 0) 253 self.assertEqual(self.status, ["WAITING", "SUCCEEDED"]) 254 255 # Second test: No cached result found; test run was "terminated" for some 256 # reason. 257 ResetTestValues() 258 br.terminated = True 259 br.run() 260 self.assertTrue(self.called_ReadCache) 261 self.assertEqual( 262 self.log_output, 263 [ 264 "test_run: No cache hit.", 265 "Releasing machine: chromeos1-row3-rack5-host7.cros", 266 "Released machine: chromeos1-row3-rack5-host7.cros", 267 ], 268 ) 269 self.assertEqual(len(self.log_error), 0) 270 self.assertEqual(self.status, ["WAITING"]) 271 272 # Third test. No cached result found; RunTest failed for some reason. 273 ResetTestValues() 274 br.terminated = False 275 br.RunTest = FakeRunTestFail 276 br.run() 277 self.assertTrue(self.called_ReadCache) 278 self.assertEqual( 279 self.log_output, 280 [ 281 "test_run: No cache hit.", 282 "Releasing machine: chromeos1-row3-rack5-host7.cros", 283 "Released machine: chromeos1-row3-rack5-host7.cros", 284 ], 285 ) 286 self.assertEqual(len(self.log_error), 0) 287 self.assertEqual(self.status, ["WAITING", "FAILED"]) 288 289 # Fourth test: ReadCache found a cached result. 290 ResetTestValues() 291 br.RunTest = FakeRunTest 292 br.ReadCache = FakeReadCacheSucceed 293 br.run() 294 self.assertTrue(self.called_ReadCache) 295 self.assertEqual( 296 self.log_output, 297 [ 298 "test_run: Cache hit.", 299 "result.out stuff", 300 "Releasing machine: chromeos1-row3-rack5-host7.cros", 301 "Released machine: chromeos1-row3-rack5-host7.cros", 302 ], 303 ) 304 self.assertEqual(self.log_error, ["result.err stuff"]) 305 self.assertEqual(self.status, ["SUCCEEDED"]) 306 307 # Fifth test: ReadCache generates an exception; does the try/finally block 308 # work? 309 ResetTestValues() 310 br.ReadCache = FakeReadCacheException 311 br.machine = FakeAcquireMachine() 312 br.run() 313 self.assertEqual( 314 self.log_error, 315 [ 316 "Benchmark run: 'test_run' failed: This is an exception test; it is " 317 "supposed to happen" 318 ], 319 ) 320 self.assertEqual(self.status, ["FAILED"]) 321 322 def test_terminate_pass(self): 323 br = benchmark_run.BenchmarkRun( 324 "test_run", 325 self.test_benchmark, 326 self.test_label, 327 1, 328 self.test_cache_conditions, 329 self.mock_machine_manager, 330 self.mock_logger, 331 "average", 332 "", 333 {}, 334 ) 335 336 def GetLastEventPassed(): 337 """Helper function for test_terminate_pass""" 338 return benchmark_run.STATUS_SUCCEEDED 339 340 def RecordStub(status): 341 """Helper function for test_terminate_pass""" 342 self.status = status 343 344 self.status = benchmark_run.STATUS_SUCCEEDED 345 self.assertFalse(br.terminated) 346 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 347 348 br.timeline.GetLastEvent = GetLastEventPassed 349 br.timeline.Record = RecordStub 350 351 br.Terminate() 352 353 self.assertTrue(br.terminated) 354 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 355 self.assertEqual(self.status, benchmark_run.STATUS_FAILED) 356 357 def test_terminate_fail(self): 358 br = benchmark_run.BenchmarkRun( 359 "test_run", 360 self.test_benchmark, 361 self.test_label, 362 1, 363 self.test_cache_conditions, 364 self.mock_machine_manager, 365 self.mock_logger, 366 "average", 367 "", 368 {}, 369 ) 370 371 def GetLastEventFailed(): 372 """Helper function for test_terminate_fail""" 373 return benchmark_run.STATUS_FAILED 374 375 def RecordStub(status): 376 """Helper function for test_terminate_fail""" 377 self.status = status 378 379 self.status = benchmark_run.STATUS_SUCCEEDED 380 self.assertFalse(br.terminated) 381 self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated()) 382 383 br.timeline.GetLastEvent = GetLastEventFailed 384 br.timeline.Record = RecordStub 385 386 br.Terminate() 387 388 self.assertTrue(br.terminated) 389 self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated()) 390 self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED) 391 392 def test_acquire_machine(self): 393 br = benchmark_run.BenchmarkRun( 394 "test_run", 395 self.test_benchmark, 396 self.test_label, 397 1, 398 self.test_cache_conditions, 399 self.mock_machine_manager, 400 self.mock_logger, 401 "average", 402 "", 403 {}, 404 ) 405 406 br.terminated = True 407 self.assertRaises(Exception, br.AcquireMachine) 408 409 br.terminated = False 410 mock_machine = MockCrosMachine( 411 "chromeos1-row3-rack5-host7.cros", "chromeos", "average" 412 ) 413 self.mock_machine_manager.AcquireMachine.return_value = mock_machine 414 415 machine = br.AcquireMachine() 416 self.assertEqual(machine.name, "chromeos1-row3-rack5-host7.cros") 417 418 def test_get_extra_autotest_args(self): 419 br = benchmark_run.BenchmarkRun( 420 "test_run", 421 self.test_benchmark, 422 self.test_label, 423 1, 424 self.test_cache_conditions, 425 self.mock_machine_manager, 426 self.mock_logger, 427 "average", 428 "", 429 {}, 430 ) 431 432 def MockLogError(err_msg): 433 """Helper function for test_get_extra_autotest_args""" 434 self.err_msg = err_msg 435 436 self.mock_logger.LogError = MockLogError 437 438 result = br.GetExtraAutotestArgs() 439 self.assertEqual(result, "") 440 441 self.test_benchmark.perf_args = "record -e cycles" 442 result = br.GetExtraAutotestArgs() 443 self.assertEqual( 444 result, 445 "--profiler=custom_perf --profiler_args='perf_options=\"record -a -e " 446 "cycles\"'", 447 ) 448 449 self.test_benchmark.perf_args = "record -e cycles" 450 self.test_benchmark.suite = "test_that" 451 result = br.GetExtraAutotestArgs() 452 self.assertEqual(result, "") 453 self.assertEqual( 454 self.err_msg, "Non-telemetry benchmark does not support profiler." 455 ) 456 457 self.test_benchmark.perf_args = "junk args" 458 self.test_benchmark.suite = "telemetry_Crosperf" 459 self.assertRaises(Exception, br.GetExtraAutotestArgs) 460 461 @mock.patch.object(SuiteRunner, "Run") 462 @mock.patch.object(Result, "CreateFromRun") 463 def test_run_test(self, mock_result, mock_runner): 464 br = benchmark_run.BenchmarkRun( 465 "test_run", 466 self.test_benchmark, 467 self.test_label, 468 1, 469 self.test_cache_conditions, 470 self.mock_machine_manager, 471 self.mock_logger, 472 "average", 473 "", 474 {}, 475 ) 476 477 self.status = [] 478 479 def MockRecord(status): 480 self.status.append(status) 481 482 br.timeline.Record = MockRecord 483 mock_machine = MockCrosMachine( 484 "chromeos1-row3-rack5-host7.cros", "chromeos", "average" 485 ) 486 mock_runner.return_value = [0, "{'Score':100}", ""] 487 488 br.RunTest(mock_machine) 489 490 self.assertTrue(br.run_completed) 491 self.assertEqual( 492 self.status, 493 [benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING], 494 ) 495 496 self.assertEqual(br.machine_manager.ImageMachine.call_count, 1) 497 br.machine_manager.ImageMachine.assert_called_with( 498 mock_machine, self.test_label 499 ) 500 self.assertEqual(mock_runner.call_count, 1) 501 mock_runner.assert_called_with( 502 mock_machine, br.label, br.benchmark, "", br.profiler_args 503 ) 504 505 self.assertEqual(mock_result.call_count, 1) 506 mock_result.assert_called_with( 507 self.mock_logger, 508 "average", 509 self.test_label, 510 None, 511 "{'Score':100}", 512 "", 513 0, 514 "page_cycler.netsim.top_10", 515 "telemetry_Crosperf", 516 "", 517 ) 518 519 def test_set_cache_conditions(self): 520 br = benchmark_run.BenchmarkRun( 521 "test_run", 522 self.test_benchmark, 523 self.test_label, 524 1, 525 self.test_cache_conditions, 526 self.mock_machine_manager, 527 self.mock_logger, 528 "average", 529 "", 530 {}, 531 ) 532 533 phony_cache_conditions = [123, 456, True, False] 534 535 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 536 537 br.SetCacheConditions(phony_cache_conditions) 538 self.assertEqual(br.cache_conditions, phony_cache_conditions) 539 540 br.SetCacheConditions(self.test_cache_conditions) 541 self.assertEqual(br.cache_conditions, self.test_cache_conditions) 542 543 544if __name__ == "__main__": 545 unittest.main() 546