1# Lint as: python2, python3 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""This is a client side WebGL aquarium test. 7 8Description of some of the test result output: 9 - interframe time: The time elapsed between two frames. It is the elapsed 10 time between two consecutive calls to the render() function. 11 - render time: The time it takes in Javascript to construct a frame and 12 submit all the GL commands. It is the time it takes for a render() 13 function call to complete. 14""" 15 16from __future__ import print_function 17 18import functools 19import logging 20import math 21import os 22import sampler 23import system_sampler 24import threading 25import time 26 27from autotest_lib.client.bin import fps_meter 28from autotest_lib.client.bin import utils 29from autotest_lib.client.common_lib import error 30from autotest_lib.client.common_lib.cros import chrome 31from autotest_lib.client.common_lib.cros import memory_eater 32from autotest_lib.client.cros.graphics import graphics_utils 33from autotest_lib.client.cros import perf 34from autotest_lib.client.cros import service_stopper 35from autotest_lib.client.cros.power import power_rapl, power_status, power_utils 36 37# Minimum battery charge percentage to run the test 38BATTERY_INITIAL_CHARGED_MIN = 10 39 40# Measurement duration in seconds. 41MEASUREMENT_DURATION = 30 42 43POWER_DESCRIPTION = 'avg_energy_rate_1000_fishes' 44 45# Time to exclude from calculation after playing a webgl demo [seconds]. 46STABILIZATION_DURATION = 10 47 48 49class graphics_WebGLAquarium(graphics_utils.GraphicsTest): 50 """WebGL aquarium graphics test.""" 51 version = 1 52 53 _backlight = None 54 _power_status = None 55 _service_stopper = None 56 _test_power = False 57 active_tab = None 58 flip_stats = {} 59 kernel_sampler = None 60 perf_keyval = {} 61 sampler_lock = None 62 test_duration_secs = 30 63 test_setting_num_fishes = 50 64 test_settings = { 65 50: ('setSetting2', 2), 66 1000: ('setSetting6', 6), 67 } 68 69 def setup(self): 70 """Testcase setup.""" 71 tarball_path = os.path.join(self.bindir, 72 'webgl_aquarium_static.tar.bz2') 73 utils.extract_tarball_to_dir(tarball_path, self.srcdir) 74 75 def initialize(self): 76 """Testcase initialization.""" 77 super(graphics_WebGLAquarium, self).initialize() 78 self.sampler_lock = threading.Lock() 79 # TODO: Create samplers for other platforms (e.g. x86). 80 if utils.get_board().lower() in ['daisy', 'daisy_spring']: 81 # Enable ExynosSampler on Exynos platforms. The sampler looks for 82 # exynos-drm page flip states: 'wait_kds', 'rendered', 'prepared', 83 # and 'flipped' in kernel debugfs. 84 85 # Sample 3-second durtaion for every 5 seconds. 86 self.kernel_sampler = sampler.ExynosSampler(period=5, duration=3) 87 self.kernel_sampler.sampler_callback = self.exynos_sampler_callback 88 self.kernel_sampler.output_flip_stats = ( 89 self.exynos_output_flip_stats) 90 91 def cleanup(self): 92 """Testcase cleanup.""" 93 if self._backlight: 94 self._backlight.restore() 95 if self._service_stopper: 96 self._service_stopper.restore_services() 97 super(graphics_WebGLAquarium, self).cleanup() 98 99 def setup_webpage(self, browser, test_url, num_fishes): 100 """Open fish tank in a new tab. 101 102 @param browser: The Browser object to run the test with. 103 @param test_url: The URL to the aquarium test site. 104 @param num_fishes: The number of fishes to run the test with. 105 """ 106 # Create tab and load page. Set the number of fishes when page is fully 107 # loaded. 108 tab = browser.tabs.New() 109 tab.Navigate(test_url) 110 tab.Activate() 111 self.active_tab = tab 112 tab.WaitForDocumentReadyStateToBeComplete() 113 114 # Set the number of fishes when document finishes loading. Also reset 115 # our own FPS counter and start recording FPS and rendering time. 116 utils.wait_for_value( 117 lambda: tab.EvaluateJavaScript( 118 'if (document.readyState === "complete") {' 119 ' setSetting(document.getElementById("%s"), %d);' 120 ' g_crosFpsCounter.reset();' 121 ' true;' 122 '} else {' 123 ' false;' 124 '}' % self.test_settings[num_fishes] 125 ), 126 expected_value=True, 127 timeout_sec=30) 128 129 return tab 130 131 def tear_down_webpage(self): 132 """Close the tab containing testing webpage.""" 133 # Do not close the tab when the sampler_callback is 134 # doing its work. 135 with self.sampler_lock: 136 self.active_tab.Close() 137 self.active_tab = None 138 139 def run_fish_test(self, browser, test_url, num_fishes, perf_log=True): 140 """Run the test with the given number of fishes. 141 142 @param browser: The Browser object to run the test with. 143 @param test_url: The URL to the aquarium test site. 144 @param num_fishes: The number of fishes to run the test with. 145 @param perf_log: Report perf data only if it's set to True. 146 """ 147 148 tab = self.setup_webpage(browser, test_url, num_fishes) 149 150 if self.kernel_sampler: 151 self.kernel_sampler.start_sampling_thread() 152 time.sleep(self.test_duration_secs) 153 if self.kernel_sampler: 154 self.kernel_sampler.stop_sampling_thread() 155 self.kernel_sampler.output_flip_stats('flip_stats_%d' % num_fishes) 156 self.flip_stats = {} 157 158 # Get average FPS and rendering time, then close the tab. 159 avg_fps = tab.EvaluateJavaScript('g_crosFpsCounter.getAvgFps();') 160 if math.isnan(float(avg_fps)): 161 raise error.TestFail('Failed: Could not get FPS count.') 162 163 avg_interframe_time = tab.EvaluateJavaScript( 164 'g_crosFpsCounter.getAvgInterFrameTime();') 165 avg_render_time = tab.EvaluateJavaScript( 166 'g_crosFpsCounter.getAvgRenderTime();') 167 std_interframe_time = tab.EvaluateJavaScript( 168 'g_crosFpsCounter.getStdInterFrameTime();') 169 std_render_time = tab.EvaluateJavaScript( 170 'g_crosFpsCounter.getStdRenderTime();') 171 self.perf_keyval['avg_fps_%04d_fishes' % num_fishes] = avg_fps 172 self.perf_keyval['avg_interframe_time_%04d_fishes' % num_fishes] = ( 173 avg_interframe_time) 174 self.perf_keyval['avg_render_time_%04d_fishes' % num_fishes] = ( 175 avg_render_time) 176 self.perf_keyval['std_interframe_time_%04d_fishes' % num_fishes] = ( 177 std_interframe_time) 178 self.perf_keyval['std_render_time_%04d_fishes' % num_fishes] = ( 179 std_render_time) 180 logging.info('%d fish(es): Average FPS = %f, ' 181 'average render time = %f', num_fishes, avg_fps, 182 avg_render_time) 183 184 if perf_log: 185 # Report frames per second to chromeperf/ dashboard. 186 self.output_perf_value( 187 description='avg_fps_%04d_fishes' % num_fishes, 188 value=avg_fps, 189 units='fps', 190 higher_is_better=True) 191 192 # Intel only: Record the power consumption for the next few seconds. 193 rapl_rate = power_rapl.get_rapl_measurement( 194 'rapl_%04d_fishes' % num_fishes) 195 # Remove entries that we don't care about. 196 rapl_rate = {key: rapl_rate[key] 197 for key in list(rapl_rate.keys()) if key.endswith('pwr')} 198 # Report to chromeperf/ dashboard. 199 for key, values in list(rapl_rate.items()): 200 self.output_perf_value( 201 description=key, 202 value=values, 203 units='W', 204 higher_is_better=False, 205 graph='rapl_power_consumption' 206 ) 207 208 def run_power_test(self, browser, test_url, ac_ok): 209 """Runs the webgl power consumption test and reports the perf results. 210 211 @param browser: The Browser object to run the test with. 212 @param test_url: The URL to the aquarium test site. 213 @param ac_ok: Boolean on whether its ok to have AC power supplied. 214 """ 215 216 self._backlight = power_utils.Backlight() 217 self._backlight.set_default() 218 219 self._service_stopper = service_stopper.ServiceStopper( 220 service_stopper.ServiceStopper.POWER_DRAW_SERVICES) 221 self._service_stopper.stop_services() 222 223 if not ac_ok: 224 self._power_status = power_status.get_status() 225 # Verify that we are running on battery and the battery is 226 # sufficiently charged. 227 self._power_status.assert_battery_state(BATTERY_INITIAL_CHARGED_MIN) 228 229 measurements = [ 230 power_status.SystemPower(self._power_status.battery_path) 231 ] 232 233 def get_power(): 234 power_logger = power_status.PowerLogger(measurements) 235 power_logger.start() 236 time.sleep(STABILIZATION_DURATION) 237 start_time = time.time() 238 time.sleep(MEASUREMENT_DURATION) 239 power_logger.checkpoint('result', start_time) 240 keyval = power_logger.calc() 241 logging.info('Power output %s', keyval) 242 return keyval['result_' + measurements[0].domain + '_pwr'] 243 244 self.run_fish_test(browser, test_url, 1000, perf_log=False) 245 if not ac_ok: 246 energy_rate = get_power() 247 # This is a power specific test so we are not capturing 248 # avg_fps and avg_render_time in this test. 249 self.perf_keyval[POWER_DESCRIPTION] = energy_rate 250 self.output_perf_value( 251 description=POWER_DESCRIPTION, 252 value=energy_rate, 253 units='W', 254 higher_is_better=False) 255 256 def exynos_sampler_callback(self, sampler_obj): 257 """Sampler callback function for ExynosSampler. 258 259 @param sampler_obj: The ExynosSampler object that invokes this callback 260 function. 261 """ 262 if sampler_obj.stopped: 263 return 264 265 with self.sampler_lock: 266 now = time.time() 267 results = {} 268 info_str = ['\nfb_id wait_kds flipped'] 269 for value in list(sampler_obj.frame_buffers.values()): 270 results[value.fb] = {} 271 for state, stats in list(value.states.items()): 272 results[value.fb][state] = (stats.avg, stats.stdev) 273 info_str.append('%s: %s %s' % (value.fb, 274 results[value.fb]['wait_kds'][0], 275 results[value.fb]['flipped'][0])) 276 results['avg_fps'] = self.active_tab.EvaluateJavaScript( 277 'g_crosFpsCounter.getAvgFps();') 278 results['avg_render_time'] = self.active_tab.EvaluateJavaScript( 279 'g_crosFpsCounter.getAvgRenderTime();') 280 self.active_tab.ExecuteJavaScript('g_crosFpsCounter.reset();') 281 info_str.append('avg_fps: %s, avg_render_time: %s' % 282 (results['avg_fps'], results['avg_render_time'])) 283 self.flip_stats[now] = results 284 logging.info('\n'.join(info_str)) 285 286 def exynos_output_flip_stats(self, file_name): 287 """Pageflip statistics output function for ExynosSampler. 288 289 @param file_name: The output file name. 290 """ 291 # output format: 292 # time fb_id avg_rendered avg_prepared avg_wait_kds avg_flipped 293 # std_rendered std_prepared std_wait_kds std_flipped 294 with open(file_name, 'w') as f: 295 for t in sorted(self.flip_stats.keys()): 296 if ('avg_fps' in self.flip_stats[t] and 297 'avg_render_time' in self.flip_stats[t]): 298 f.write('%s %s %s\n' % 299 (t, self.flip_stats[t]['avg_fps'], 300 self.flip_stats[t]['avg_render_time'])) 301 for fb, stats in list(self.flip_stats[t].items()): 302 if not isinstance(fb, int): 303 continue 304 f.write('%s %s ' % (t, fb)) 305 f.write('%s %s %s %s ' % (stats['rendered'][0], 306 stats['prepared'][0], 307 stats['wait_kds'][0], 308 stats['flipped'][0])) 309 f.write('%s %s %s %s\n' % (stats['rendered'][1], 310 stats['prepared'][1], 311 stats['wait_kds'][1], 312 stats['flipped'][1])) 313 314 def write_samples(self, samples, filename): 315 """Writes all samples to result dir with the file name "samples'. 316 317 @param samples: A list of all collected samples. 318 @param filename: The file name to save under result directory. 319 """ 320 out_file = os.path.join(self.resultsdir, filename) 321 with open(out_file, 'w') as f: 322 for sample in samples: 323 print(sample, file=f) 324 325 def run_fish_test_with_memory_pressure( 326 self, browser, test_url, num_fishes, memory_pressure): 327 """Measure fps under memory pressure. 328 329 It measure FPS of WebGL aquarium while adding memory pressure. It runs 330 in 2 phases: 331 1. Allocate non-swappable memory until |memory_to_reserve_mb| is 332 remained. The memory is not accessed after allocated. 333 2. Run "active" memory consumer in the background. After allocated, 334 Its content is accessed sequentially by page and looped around 335 infinitely. 336 The second phase is opeared in two possible modes: 337 1. "single" mode, which means only one "active" memory consumer. After 338 running a single memory consumer with a given memory size, it waits 339 for a while to see if system can afford current memory pressure 340 (definition here is FPS > 5). If it does, kill current consumer and 341 launch another consumer with a larger memory size. The process keeps 342 going until system couldn't afford the load. 343 2. "multiple"mode. It simply launch memory consumers with a given size 344 one by one until system couldn't afford the load (e.g., FPS < 5). 345 In "single" mode, CPU load is lighter so we expect swap in/swap out 346 rate to be correlated to FPS better. In "multiple" mode, since there 347 are multiple busy loop processes, CPU pressure is another significant 348 cause of frame drop. Frame drop can happen easily due to busy CPU 349 instead of memory pressure. 350 351 @param browser: The Browser object to run the test with. 352 @param test_url: The URL to the aquarium test site. 353 @param num_fishes: The number of fishes to run the test with. 354 @param memory_pressure: Memory pressure parameters. 355 """ 356 consumer_mode = memory_pressure.get('consumer_mode', 'single') 357 memory_to_reserve_mb = memory_pressure.get('memory_to_reserve_mb', 500) 358 # Empirical number to quickly produce memory pressure. 359 if consumer_mode == 'single': 360 default_consumer_size_mb = memory_to_reserve_mb + 100 361 else: 362 default_consumer_size_mb = memory_to_reserve_mb / 2 363 consumer_size_mb = memory_pressure.get( 364 'consumer_size_mb', default_consumer_size_mb) 365 366 # Setup fish tank. 367 self.setup_webpage(browser, test_url, num_fishes) 368 369 # Drop all file caches. 370 utils.drop_caches() 371 372 def fps_near_zero(fps_sampler): 373 """Returns whether recent fps goes down to near 0. 374 375 @param fps_sampler: A system_sampler.Sampler object. 376 """ 377 last_fps = fps_sampler.get_last_avg_fps(6) 378 if last_fps: 379 logging.info('last fps %f', last_fps) 380 if last_fps <= 5: 381 return True 382 return False 383 384 max_allocated_mb = 0 385 # Consume free memory and release them by the end. 386 with memory_eater.consume_free_memory(memory_to_reserve_mb): 387 fps_sampler = system_sampler.SystemSampler( 388 memory_eater.MemoryEater.get_active_consumer_pids) 389 end_condition = functools.partial(fps_near_zero, fps_sampler) 390 with fps_meter.FPSMeter(fps_sampler.sample): 391 # Collects some samples before running memory pressure. 392 time.sleep(5) 393 try: 394 if consumer_mode == 'single': 395 # A single run couldn't generate samples representative 396 # enough. 397 # First runs squeeze more inactive anonymous memory into 398 # zram so in later runs we have a more stable memory 399 # stat. 400 max_allocated_mb = max( 401 memory_eater.run_single_memory_pressure( 402 consumer_size_mb, 100, end_condition, 10, 3, 403 900), 404 memory_eater.run_single_memory_pressure( 405 consumer_size_mb, 20, end_condition, 10, 3, 406 900), 407 memory_eater.run_single_memory_pressure( 408 consumer_size_mb, 10, end_condition, 10, 3, 409 900)) 410 elif consumer_mode == 'multiple': 411 max_allocated_mb = ( 412 memory_eater.run_multi_memory_pressure( 413 consumer_size_mb, end_condition, 10, 900)) 414 else: 415 raise error.TestFail( 416 'Failed: Unsupported consumer mode.') 417 except memory_eater.TimeoutException as e: 418 raise error.TestFail(e) 419 420 samples = fps_sampler.get_samples() 421 self.write_samples(samples, 'memory_pressure_fps_samples.txt') 422 423 self.perf_keyval['num_samples'] = len(samples) 424 self.perf_keyval['max_allocated_mb'] = max_allocated_mb 425 426 logging.info(self.perf_keyval) 427 428 self.output_perf_value( 429 description='max_allocated_mb_%d_fishes_reserved_%d_mb' % ( 430 num_fishes, memory_to_reserve_mb), 431 value=max_allocated_mb, 432 units='MB', 433 higher_is_better=True) 434 435 436 @graphics_utils.GraphicsTest.failure_report_decorator('graphics_WebGLAquarium') 437 def run_once(self, 438 test_duration_secs=30, 439 test_setting_num_fishes=(50, 1000), 440 power_test=False, 441 ac_ok=False, 442 memory_pressure=None): 443 """Find a browser with telemetry, and run the test. 444 445 @param test_duration_secs: The duration in seconds to run each scenario 446 for. 447 @param test_setting_num_fishes: A list of the numbers of fishes to 448 enable in the test. 449 @param power_test: Boolean on whether to run power_test 450 @param ac_ok: Boolean on whether its ok to have AC power supplied. 451 @param memory_pressure: A dictionay which specifies memory pressure 452 parameters: 453 'consumer_mode': 'single' or 'multiple' to have one or moultiple 454 concurrent memory consumers. 455 'consumer_size_mb': Amount of memory to allocate. In 'single' 456 mode, a single memory consumer would allocate memory by the 457 specific size. It then gradually allocates more memory until 458 FPS down to near 0. In 'multiple' mode, memory consumers of 459 this size would be spawn one by one until FPS down to near 0. 460 'memory_to_reserve_mb': Amount of memory to reserve before 461 running memory consumer. In practical we allocate mlocked 462 memory (i.e., not swappable) to consume free memory until this 463 amount of free memory remained. 464 """ 465 self.test_duration_secs = test_duration_secs 466 self.test_setting_num_fishes = test_setting_num_fishes 467 pc_error_reason = None 468 469 with chrome.Chrome(logged_in=False, init_network_controller=True) as cr: 470 cr.browser.platform.SetHTTPServerDirectories(self.srcdir) 471 test_url = cr.browser.platform.http_server.UrlOf( 472 os.path.join(self.srcdir, 'aquarium.html')) 473 474 utils.report_temperature(self, 'temperature_1_start') 475 # Wrap the test run inside of a PerfControl instance to make machine 476 # behavior more consistent. 477 with perf.PerfControl() as pc: 478 if not pc.verify_is_valid(): 479 raise error.TestFail('Failed: %s' % pc.get_error_reason()) 480 utils.report_temperature(self, 'temperature_2_before_test') 481 482 if memory_pressure: 483 self.run_fish_test_with_memory_pressure( 484 cr.browser, test_url, num_fishes=1000, 485 memory_pressure=memory_pressure) 486 self.tear_down_webpage() 487 elif power_test: 488 self._test_power = True 489 self.run_power_test(cr.browser, test_url, ac_ok) 490 self.tear_down_webpage() 491 else: 492 for n in self.test_setting_num_fishes: 493 self.run_fish_test(cr.browser, test_url, n) 494 self.tear_down_webpage() 495 496 if not pc.verify_is_valid(): 497 # Defer error handling until after perf report. 498 pc_error_reason = pc.get_error_reason() 499 500 utils.report_temperature(self, 'temperature_3_after_test') 501 self.write_perf_keyval(self.perf_keyval) 502 503 if pc_error_reason: 504 raise error.TestWarn('Warning: %s' % pc_error_reason) 505