1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright 2015 The ChromiumOS Authors 5# Use of this source code is governed by a BSD-style license that can be 6# found in the LICENSE file. 7 8"""This contains the unit tests for the new Crosperf task scheduler.""" 9 10 11import functools 12import io 13import unittest 14import unittest.mock as mock 15 16import benchmark_run 17from cros_utils.command_executer import CommandExecuter 18from experiment_factory import ExperimentFactory 19from experiment_file import ExperimentFile 20from experiment_runner_unittest import FakeLogger 21from schedv2 import Schedv2 22import test_flag 23 24 25EXPERIMENT_FILE_1 = """\ 26board: daisy 27remote: chromeos-daisy1.cros chromeos-daisy2.cros 28locks_dir: /tmp 29 30benchmark: kraken { 31 suite: telemetry_Crosperf 32 iterations: 3 33} 34 35image1 { 36 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 37 remote: chromeos-daisy3.cros 38} 39 40image2 { 41 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 42 remote: chromeos-daisy4.cros chromeos-daisy5.cros 43} 44""" 45 46EXPERIMENT_FILE_WITH_FORMAT = """\ 47board: daisy 48remote: chromeos-daisy1.cros chromeos-daisy2.cros 49locks_dir: /tmp 50 51benchmark: kraken {{ 52 suite: telemetry_Crosperf 53 iterations: {kraken_iterations} 54}} 55 56image1 {{ 57 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 58 remote: chromeos-daisy3.cros 59}} 60 61image2 {{ 62 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 63 remote: chromeos-daisy4.cros chromeos-daisy5.cros 64}} 65""" 66 67 68class Schedv2Test(unittest.TestCase): 69 """Class for setting up and running the unit tests.""" 70 71 def setUp(self): 72 self.exp = None 73 74 mock_logger = FakeLogger() 75 mock_cmd_exec = mock.Mock(spec=CommandExecuter) 76 77 @mock.patch( 78 "benchmark_run.BenchmarkRun", new=benchmark_run.MockBenchmarkRun 79 ) 80 def _make_fake_experiment(self, expstr): 81 """Create fake experiment from string. 82 83 Note - we mock out BenchmarkRun in this step. 84 """ 85 experiment_file = ExperimentFile(io.StringIO(expstr)) 86 experiment = ExperimentFactory().GetExperiment( 87 experiment_file, working_directory="", log_dir="" 88 ) 89 return experiment 90 91 def test_remote(self): 92 """Test that remotes in labels are aggregated into experiment.remote.""" 93 94 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 95 self.exp.log_level = "verbose" 96 my_schedv2 = Schedv2(self.exp) 97 self.assertFalse(my_schedv2.is_complete()) 98 self.assertIn("chromeos-daisy1.cros", self.exp.remote) 99 self.assertIn("chromeos-daisy2.cros", self.exp.remote) 100 self.assertIn("chromeos-daisy3.cros", self.exp.remote) 101 self.assertIn("chromeos-daisy4.cros", self.exp.remote) 102 self.assertIn("chromeos-daisy5.cros", self.exp.remote) 103 104 def test_unreachable_remote(self): 105 """Test unreachable remotes are removed from experiment and label.""" 106 107 def MockIsReachable(cm): 108 return ( 109 cm.name != "chromeos-daisy3.cros" 110 and cm.name != "chromeos-daisy5.cros" 111 ) 112 113 with mock.patch( 114 "machine_manager.MockCrosMachine.IsReachable", new=MockIsReachable 115 ): 116 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 117 self.assertIn("chromeos-daisy1.cros", self.exp.remote) 118 self.assertIn("chromeos-daisy2.cros", self.exp.remote) 119 self.assertNotIn("chromeos-daisy3.cros", self.exp.remote) 120 self.assertIn("chromeos-daisy4.cros", self.exp.remote) 121 self.assertNotIn("chromeos-daisy5.cros", self.exp.remote) 122 123 for l in self.exp.labels: 124 if l.name == "image2": 125 self.assertNotIn("chromeos-daisy5.cros", l.remote) 126 self.assertIn("chromeos-daisy4.cros", l.remote) 127 elif l.name == "image1": 128 self.assertNotIn("chromeos-daisy3.cros", l.remote) 129 130 @mock.patch("schedv2.BenchmarkRunCacheReader") 131 def test_BenchmarkRunCacheReader_1(self, reader): 132 """Test benchmarkrun set is split into 5 segments.""" 133 134 self.exp = self._make_fake_experiment( 135 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=9) 136 ) 137 my_schedv2 = Schedv2(self.exp) 138 self.assertFalse(my_schedv2.is_complete()) 139 # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4, 140 # 4, 2 brs respectively. 141 # Assert that BenchmarkRunCacheReader() is called 5 times. 142 self.assertEqual(reader.call_count, 5) 143 # reader.call_args_list[n] - nth call. 144 # reader.call_args_list[n][0] - positioned args in nth call. 145 # reader.call_args_list[n][0][1] - the 2nd arg in nth call, 146 # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'. 147 self.assertEqual(len(reader.call_args_list[0][0][1]), 4) 148 self.assertEqual(len(reader.call_args_list[1][0][1]), 4) 149 self.assertEqual(len(reader.call_args_list[2][0][1]), 4) 150 self.assertEqual(len(reader.call_args_list[3][0][1]), 4) 151 self.assertEqual(len(reader.call_args_list[4][0][1]), 2) 152 153 @mock.patch("schedv2.BenchmarkRunCacheReader") 154 def test_BenchmarkRunCacheReader_2(self, reader): 155 """Test benchmarkrun set is split into 4 segments.""" 156 157 self.exp = self._make_fake_experiment( 158 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=8) 159 ) 160 my_schedv2 = Schedv2(self.exp) 161 self.assertFalse(my_schedv2.is_complete()) 162 # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs. 163 self.assertEqual(reader.call_count, 4) 164 self.assertEqual(len(reader.call_args_list[0][0][1]), 4) 165 self.assertEqual(len(reader.call_args_list[1][0][1]), 4) 166 self.assertEqual(len(reader.call_args_list[2][0][1]), 4) 167 self.assertEqual(len(reader.call_args_list[3][0][1]), 4) 168 169 @mock.patch("schedv2.BenchmarkRunCacheReader") 170 def test_BenchmarkRunCacheReader_3(self, reader): 171 """Test benchmarkrun set is split into 2 segments.""" 172 173 self.exp = self._make_fake_experiment( 174 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=3) 175 ) 176 my_schedv2 = Schedv2(self.exp) 177 self.assertFalse(my_schedv2.is_complete()) 178 # We have 3 * 2 == 6 brs, we use 2 threads. 179 self.assertEqual(reader.call_count, 2) 180 self.assertEqual(len(reader.call_args_list[0][0][1]), 3) 181 self.assertEqual(len(reader.call_args_list[1][0][1]), 3) 182 183 @mock.patch("schedv2.BenchmarkRunCacheReader") 184 def test_BenchmarkRunCacheReader_4(self, reader): 185 """Test benchmarkrun set is not splitted.""" 186 187 self.exp = self._make_fake_experiment( 188 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=1) 189 ) 190 my_schedv2 = Schedv2(self.exp) 191 self.assertFalse(my_schedv2.is_complete()) 192 # We have 1 * 2 == 2 br, so only 1 instance. 193 self.assertEqual(reader.call_count, 1) 194 self.assertEqual(len(reader.call_args_list[0][0][1]), 2) 195 196 def test_cachehit(self): 197 """Test cache-hit and none-cache-hit brs are properly organized.""" 198 199 def MockReadCache(br): 200 br.cache_hit = br.label.name == "image2" 201 202 with mock.patch( 203 "benchmark_run.MockBenchmarkRun.ReadCache", new=MockReadCache 204 ): 205 # We have 2 * 30 brs, half of which are put into _cached_br_list. 206 self.exp = self._make_fake_experiment( 207 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30) 208 ) 209 my_schedv2 = Schedv2(self.exp) 210 self.assertEqual(len(my_schedv2.get_cached_run_list()), 30) 211 # The non-cache-hit brs are put into Schedv2._label_brl_map. 212 self.assertEqual( 213 functools.reduce( 214 lambda a, x: a + len(x[1]), 215 my_schedv2.get_label_map().items(), 216 0, 217 ), 218 30, 219 ) 220 221 def test_nocachehit(self): 222 """Test no cache-hit.""" 223 224 def MockReadCache(br): 225 br.cache_hit = False 226 227 with mock.patch( 228 "benchmark_run.MockBenchmarkRun.ReadCache", new=MockReadCache 229 ): 230 # We have 2 * 30 brs, none of which are put into _cached_br_list. 231 self.exp = self._make_fake_experiment( 232 EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30) 233 ) 234 my_schedv2 = Schedv2(self.exp) 235 self.assertEqual(len(my_schedv2.get_cached_run_list()), 0) 236 # The non-cache-hit brs are put into Schedv2._label_brl_map. 237 self.assertEqual( 238 functools.reduce( 239 lambda a, x: a + len(x[1]), 240 my_schedv2.get_label_map().items(), 241 0, 242 ), 243 60, 244 ) 245 246 247if __name__ == "__main__": 248 test_flag.SetTestMode(True) 249 unittest.main() 250