1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright 2016 The ChromiumOS Authors 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""Unittest for the results reporter.""" 9 10 11import collections 12import io 13import os 14import unittest 15import unittest.mock as mock 16 17from benchmark_run import MockBenchmarkRun 18from cros_utils import logger 19from experiment_factory import ExperimentFactory 20from experiment_file import ExperimentFile 21from machine_manager import MockCrosMachine 22from machine_manager import MockMachineManager 23from results_cache import MockResult 24from results_report import BenchmarkResults 25from results_report import HTMLResultsReport 26from results_report import JSONResultsReport 27from results_report import ParseChromeosImage 28from results_report import ParseStandardPerfReport 29from results_report import TextResultsReport 30import test_flag 31 32 33class FreeFunctionsTest(unittest.TestCase): 34 """Tests for any free functions in results_report.""" 35 36 def testParseChromeosImage(self): 37 # N.B. the cases with blank versions aren't explicitly supported by 38 # ParseChromeosImage. I'm not sure if they need to be supported, but the 39 # goal of this was to capture existing functionality as much as possible. 40 base_case = ( 41 "/my/chroot/src/build/images/x86-generic/R01-1.0.date-time" 42 "/chromiumos_test_image.bin" 43 ) 44 self.assertEqual(ParseChromeosImage(base_case), ("R01-1.0", base_case)) 45 46 dir_base_case = os.path.dirname(base_case) 47 self.assertEqual(ParseChromeosImage(dir_base_case), ("", dir_base_case)) 48 49 buildbot_case = ( 50 "/my/chroot/chroot/tmp/buildbot-build/R02-1.0.date-time" 51 "/chromiumos_test_image.bin" 52 ) 53 buildbot_img = buildbot_case.split("/chroot/tmp")[1] 54 55 self.assertEqual( 56 ParseChromeosImage(buildbot_case), ("R02-1.0", buildbot_img) 57 ) 58 self.assertEqual( 59 ParseChromeosImage(os.path.dirname(buildbot_case)), 60 ("", os.path.dirname(buildbot_img)), 61 ) 62 63 # Ensure we do something reasonable when giving paths that don't quite 64 # match the expected pattern. 65 fun_case = "/chromiumos_test_image.bin" 66 self.assertEqual(ParseChromeosImage(fun_case), ("", fun_case)) 67 68 fun_case2 = "chromiumos_test_image.bin" 69 self.assertEqual(ParseChromeosImage(fun_case2), ("", fun_case2)) 70 71 72# There are many ways for this to be done better, but the linter complains 73# about all of them (that I can think of, at least). 74_fake_path_number = [0] 75 76 77def FakePath(ext): 78 """Makes a unique path that shouldn't exist on the host system. 79 80 Each call returns a different path, so if said path finds its way into an 81 error message, it may be easier to track it to its source. 82 """ 83 _fake_path_number[0] += 1 84 prefix = "/tmp/should/not/exist/%d/" % (_fake_path_number[0],) 85 return os.path.join(prefix, ext) 86 87 88def MakeMockExperiment(compiler="gcc"): 89 """Mocks an experiment using the given compiler.""" 90 mock_experiment_file = io.StringIO( 91 """ 92 board: x86-alex 93 remote: 127.0.0.1 94 locks_dir: /tmp 95 perf_args: record -a -e cycles 96 benchmark: PageCycler { 97 iterations: 3 98 } 99 100 image1 { 101 chromeos_image: %s 102 } 103 104 image2 { 105 remote: 127.0.0.2 106 chromeos_image: %s 107 } 108 """ 109 % (FakePath("cros_image1.bin"), FakePath("cros_image2.bin")) 110 ) 111 efile = ExperimentFile(mock_experiment_file) 112 experiment = ExperimentFactory().GetExperiment( 113 efile, FakePath("working_directory"), FakePath("log_dir") 114 ) 115 for label in experiment.labels: 116 label.compiler = compiler 117 return experiment 118 119 120def _InjectSuccesses(experiment, how_many, keyvals, for_benchmark=0): 121 """Injects successful experiment runs (for each label) into the experiment.""" 122 # Defensive copy of keyvals, so if it's modified, we'll know. 123 keyvals = dict(keyvals) 124 num_configs = len(experiment.benchmarks) * len(experiment.labels) 125 num_runs = len(experiment.benchmark_runs) // num_configs 126 127 # TODO(gbiv): Centralize the mocking of these, maybe? (It's also done in 128 # benchmark_run_unittest) 129 bench = experiment.benchmarks[for_benchmark] 130 cache_conditions = [] 131 log_level = "average" 132 share_cache = "" 133 locks_dir = "" 134 log = logger.GetLogger() 135 machine_manager = MockMachineManager( 136 FakePath("chromeos_root"), 0, log_level, locks_dir 137 ) 138 machine_manager.AddMachine("testing_machine") 139 machine = next( 140 m for m in machine_manager.GetMachines() if m.name == "testing_machine" 141 ) 142 143 def MakeSuccessfulRun(n, label): 144 run = MockBenchmarkRun( 145 "mock_success%d" % (n,), 146 bench, 147 label, 148 1 + n + num_runs, 149 cache_conditions, 150 machine_manager, 151 log, 152 log_level, 153 share_cache, 154 {}, 155 ) 156 mock_result = MockResult(log, label, log_level, machine) 157 mock_result.keyvals = keyvals 158 run.result = mock_result 159 return run 160 161 for label in experiment.labels: 162 experiment.benchmark_runs.extend( 163 MakeSuccessfulRun(n, label) for n in range(how_many) 164 ) 165 return experiment 166 167 168class TextResultsReportTest(unittest.TestCase): 169 """Tests that the output of a text report contains the things we pass in. 170 171 At the moment, this doesn't care deeply about the format in which said 172 things are displayed. It just cares that they're present. 173 """ 174 175 def _checkReport(self, mock_getcooldown, email): 176 num_success = 2 177 success_keyvals = {"retval": 0, "machine": "some bot", "a_float": 3.96} 178 experiment = _InjectSuccesses( 179 MakeMockExperiment(), num_success, success_keyvals 180 ) 181 SECONDS_IN_MIN = 60 182 mock_getcooldown.return_value = { 183 experiment.remote[0]: 12 * SECONDS_IN_MIN, 184 experiment.remote[1]: 8 * SECONDS_IN_MIN, 185 } 186 187 text_report = TextResultsReport.FromExperiment( 188 experiment, email=email 189 ).GetReport() 190 self.assertIn(str(success_keyvals["a_float"]), text_report) 191 self.assertIn(success_keyvals["machine"], text_report) 192 self.assertIn(MockCrosMachine.CPUINFO_STRING, text_report) 193 self.assertIn("\nDuration\n", text_report) 194 self.assertIn("Total experiment time:\n", text_report) 195 self.assertIn("Cooldown wait time:\n", text_report) 196 self.assertIn( 197 "DUT %s: %d min" % (experiment.remote[0], 12), text_report 198 ) 199 self.assertIn("DUT %s: %d min" % (experiment.remote[1], 8), text_report) 200 return text_report 201 202 @mock.patch.object(TextResultsReport, "GetTotalWaitCooldownTime") 203 def testOutput(self, mock_getcooldown): 204 email_report = self._checkReport(mock_getcooldown, email=True) 205 text_report = self._checkReport(mock_getcooldown, email=False) 206 207 # Ensure that the reports somehow different. Otherwise, having the 208 # distinction is useless. 209 self.assertNotEqual(email_report, text_report) 210 211 def test_get_totalwait_cooldowntime(self): 212 experiment = MakeMockExperiment() 213 cros_machines = experiment.machine_manager.GetMachines() 214 cros_machines[0].AddCooldownWaitTime(120) 215 cros_machines[1].AddCooldownWaitTime(240) 216 text_results = TextResultsReport.FromExperiment(experiment, email=False) 217 total = text_results.GetTotalWaitCooldownTime() 218 self.assertEqual(total[experiment.remote[0]], 120) 219 self.assertEqual(total[experiment.remote[1]], 240) 220 221 222class HTMLResultsReportTest(unittest.TestCase): 223 """Tests that the output of a HTML report contains the things we pass in. 224 225 At the moment, this doesn't care deeply about the format in which said 226 things are displayed. It just cares that they're present. 227 """ 228 229 _TestOutput = collections.namedtuple( 230 "TestOutput", 231 [ 232 "summary_table", 233 "perf_html", 234 "chart_js", 235 "charts", 236 "full_table", 237 "experiment_file", 238 ], 239 ) 240 241 @staticmethod 242 def _GetTestOutput( 243 perf_table, 244 chart_js, 245 summary_table, 246 print_table, 247 chart_divs, 248 full_table, 249 experiment_file, 250 ): 251 # N.B. Currently we don't check chart_js; it's just passed through because 252 # cros lint complains otherwise. 253 summary_table = print_table(summary_table, "HTML") 254 perf_html = print_table(perf_table, "HTML") 255 full_table = print_table(full_table, "HTML") 256 return HTMLResultsReportTest._TestOutput( 257 summary_table=summary_table, 258 perf_html=perf_html, 259 chart_js=chart_js, 260 charts=chart_divs, 261 full_table=full_table, 262 experiment_file=experiment_file, 263 ) 264 265 def _GetOutput(self, experiment=None, benchmark_results=None): 266 with mock.patch("results_report_templates.GenerateHTMLPage") as standin: 267 if experiment is not None: 268 HTMLResultsReport.FromExperiment(experiment).GetReport() 269 else: 270 HTMLResultsReport(benchmark_results).GetReport() 271 mod_mock = standin 272 self.assertEqual(mod_mock.call_count, 1) 273 # call_args[0] is positional args, call_args[1] is kwargs. 274 self.assertEqual(mod_mock.call_args[0], tuple()) 275 fmt_args = mod_mock.call_args[1] 276 return self._GetTestOutput(**fmt_args) 277 278 def testNoSuccessOutput(self): 279 output = self._GetOutput(MakeMockExperiment()) 280 self.assertIn("no result", output.summary_table) 281 self.assertIn("no result", output.full_table) 282 self.assertEqual(output.charts, "") 283 self.assertNotEqual(output.experiment_file, "") 284 285 def testSuccessfulOutput(self): 286 num_success = 2 287 success_keyvals = {"retval": 0, "a_float": 3.96} 288 output = self._GetOutput( 289 _InjectSuccesses(MakeMockExperiment(), num_success, success_keyvals) 290 ) 291 292 self.assertNotIn("no result", output.summary_table) 293 # self.assertIn(success_keyvals['machine'], output.summary_table) 294 self.assertIn("a_float", output.summary_table) 295 self.assertIn(str(success_keyvals["a_float"]), output.summary_table) 296 self.assertIn("a_float", output.full_table) 297 # The _ in a_float is filtered out when we're generating HTML. 298 self.assertIn("afloat", output.charts) 299 # And make sure we have our experiment file... 300 self.assertNotEqual(output.experiment_file, "") 301 302 def testBenchmarkResultFailure(self): 303 labels = ["label1"] 304 benchmark_names_and_iterations = [("bench1", 1)] 305 benchmark_keyvals = {"bench1": [[]]} 306 results = BenchmarkResults( 307 labels, benchmark_names_and_iterations, benchmark_keyvals 308 ) 309 output = self._GetOutput(benchmark_results=results) 310 self.assertIn("no result", output.summary_table) 311 self.assertEqual(output.charts, "") 312 self.assertEqual(output.experiment_file, "") 313 314 def testBenchmarkResultSuccess(self): 315 labels = ["label1"] 316 benchmark_names_and_iterations = [("bench1", 1)] 317 benchmark_keyvals = {"bench1": [[{"retval": 1, "foo": 2.0}]]} 318 results = BenchmarkResults( 319 labels, benchmark_names_and_iterations, benchmark_keyvals 320 ) 321 output = self._GetOutput(benchmark_results=results) 322 self.assertNotIn("no result", output.summary_table) 323 self.assertIn("bench1", output.summary_table) 324 self.assertIn("bench1", output.full_table) 325 self.assertNotEqual(output.charts, "") 326 self.assertEqual(output.experiment_file, "") 327 328 329class JSONResultsReportTest(unittest.TestCase): 330 """Tests JSONResultsReport.""" 331 332 REQUIRED_REPORT_KEYS = ("date", "time", "label", "test_name", "pass") 333 EXPERIMENT_REPORT_KEYS = ( 334 "board", 335 "chromeos_image", 336 "chromeos_version", 337 "chrome_version", 338 "compiler", 339 ) 340 341 @staticmethod 342 def _GetRequiredKeys(is_experiment): 343 required_keys = JSONResultsReportTest.REQUIRED_REPORT_KEYS 344 if is_experiment: 345 required_keys += JSONResultsReportTest.EXPERIMENT_REPORT_KEYS 346 return required_keys 347 348 def _CheckRequiredKeys(self, test_output, is_experiment): 349 required_keys = self._GetRequiredKeys(is_experiment) 350 for output in test_output: 351 for key in required_keys: 352 self.assertIn(key, output) 353 354 def testAllFailedJSONReportOutput(self): 355 experiment = MakeMockExperiment() 356 results = JSONResultsReport.FromExperiment(experiment).GetReportObject() 357 self._CheckRequiredKeys(results, is_experiment=True) 358 # Nothing succeeded; we don't send anything more than what's required. 359 required_keys = self._GetRequiredKeys(is_experiment=True) 360 for result in results: 361 self.assertCountEqual(result.keys(), required_keys) 362 363 def testJSONReportOutputWithSuccesses(self): 364 success_keyvals = { 365 "retval": 0, 366 "a_float": "2.3", 367 "many_floats": [["1.0", "2.0"], ["3.0"]], 368 "machine": "i'm a pirate", 369 } 370 371 # 2 is arbitrary. 372 num_success = 2 373 experiment = _InjectSuccesses( 374 MakeMockExperiment(), num_success, success_keyvals 375 ) 376 results = JSONResultsReport.FromExperiment(experiment).GetReportObject() 377 self._CheckRequiredKeys(results, is_experiment=True) 378 379 num_passes = num_success * len(experiment.labels) 380 non_failures = [r for r in results if r["pass"]] 381 self.assertEqual(num_passes, len(non_failures)) 382 383 # TODO(gbiv): ...Is the 3.0 *actually* meant to be dropped? 384 expected_detailed = {"a_float": 2.3, "many_floats": [1.0, 2.0]} 385 for pass_ in non_failures: 386 self.assertIn("detailed_results", pass_) 387 self.assertDictEqual(expected_detailed, pass_["detailed_results"]) 388 self.assertIn("machine", pass_) 389 self.assertEqual(success_keyvals["machine"], pass_["machine"]) 390 391 def testFailedJSONReportOutputWithoutExperiment(self): 392 labels = ["label1"] 393 # yapf:disable 394 benchmark_names_and_iterations = [ 395 ("bench1", 1), 396 ("bench2", 2), 397 ("bench3", 1), 398 ("bench4", 0), 399 ] 400 # yapf:enable 401 402 benchmark_keyvals = { 403 "bench1": [[{"retval": 1, "foo": 2.0}]], 404 "bench2": [[{"retval": 1, "foo": 4.0}, {"retval": -1, "bar": 999}]], 405 # lack of retval is considered a failure. 406 "bench3": [[{}]], 407 "bench4": [[]], 408 } 409 bench_results = BenchmarkResults( 410 labels, benchmark_names_and_iterations, benchmark_keyvals 411 ) 412 results = JSONResultsReport(bench_results).GetReportObject() 413 self._CheckRequiredKeys(results, is_experiment=False) 414 self.assertFalse(any(r["pass"] for r in results)) 415 416 def testJSONGetReportObeysJSONSettings(self): 417 labels = ["label1"] 418 benchmark_names_and_iterations = [("bench1", 1)] 419 # These can be anything, really. So long as they're distinctive. 420 separators = (",\t\n\t", ":\t\n\t") 421 benchmark_keyvals = {"bench1": [[{"retval": 0, "foo": 2.0}]]} 422 bench_results = BenchmarkResults( 423 labels, benchmark_names_and_iterations, benchmark_keyvals 424 ) 425 reporter = JSONResultsReport( 426 bench_results, json_args={"separators": separators} 427 ) 428 result_str = reporter.GetReport() 429 self.assertIn(separators[0], result_str) 430 self.assertIn(separators[1], result_str) 431 432 def testSuccessfulJSONReportOutputWithoutExperiment(self): 433 labels = ["label1"] 434 benchmark_names_and_iterations = [("bench1", 1), ("bench2", 2)] 435 benchmark_keyvals = { 436 "bench1": [[{"retval": 0, "foo": 2.0}]], 437 "bench2": [[{"retval": 0, "foo": 4.0}, {"retval": 0, "bar": 999}]], 438 } 439 bench_results = BenchmarkResults( 440 labels, benchmark_names_and_iterations, benchmark_keyvals 441 ) 442 results = JSONResultsReport(bench_results).GetReportObject() 443 self._CheckRequiredKeys(results, is_experiment=False) 444 self.assertTrue(all(r["pass"] for r in results)) 445 # Enforce that the results have *some* deterministic order. 446 keyfn = lambda r: ( 447 r["test_name"], 448 r["detailed_results"].get("foo", 5.0), 449 ) 450 sorted_results = sorted(results, key=keyfn) 451 detailed_results = [r["detailed_results"] for r in sorted_results] 452 bench1, bench2_foo, bench2_bar = detailed_results 453 self.assertEqual(bench1["foo"], 2.0) 454 self.assertEqual(bench2_foo["foo"], 4.0) 455 self.assertEqual(bench2_bar["bar"], 999) 456 self.assertNotIn("bar", bench1) 457 self.assertNotIn("bar", bench2_foo) 458 self.assertNotIn("foo", bench2_bar) 459 460 461class PerfReportParserTest(unittest.TestCase): 462 """Tests for the perf report parser in results_report.""" 463 464 @staticmethod 465 def _ReadRealPerfReport(): 466 my_dir = os.path.dirname(os.path.realpath(__file__)) 467 with open(os.path.join(my_dir, "perf_files/perf.data.report.0")) as f: 468 return f.read() 469 470 def testParserParsesRealWorldPerfReport(self): 471 report = ParseStandardPerfReport(self._ReadRealPerfReport()) 472 self.assertCountEqual(["cycles", "instructions"], list(report.keys())) 473 474 # Arbitrarily selected known percentages from the perf report. 475 known_cycles_percentages = { 476 "0xffffffffa4a1f1c9": 0.66, 477 "0x0000115bb7ba9b54": 0.47, 478 "0x0000000000082e08": 0.00, 479 "0xffffffffa4a13e63": 0.00, 480 } 481 report_cycles = report["cycles"] 482 self.assertEqual(len(report_cycles), 214) 483 for k, v in known_cycles_percentages.items(): 484 self.assertIn(k, report_cycles) 485 self.assertEqual(v, report_cycles[k]) 486 487 known_instrunctions_percentages = { 488 "0x0000115bb6c35d7a": 1.65, 489 "0x0000115bb7ba9b54": 0.67, 490 "0x0000000000024f56": 0.00, 491 "0xffffffffa4a0ee03": 0.00, 492 } 493 report_instructions = report["instructions"] 494 self.assertEqual(len(report_instructions), 492) 495 for k, v in known_instrunctions_percentages.items(): 496 self.assertIn(k, report_instructions) 497 self.assertEqual(v, report_instructions[k]) 498 499 500if __name__ == "__main__": 501 test_flag.SetTestMode(True) 502 unittest.main() 503