1# Copyright 2013 The ChromiumOS Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Define a type that wraps a Benchmark instance.""" 6 7 8import math 9import statistics 10from typing import Any 11 12import numpy as np 13 14 15# See crbug.com/673558 for how these are estimated. 16_estimated_stddev = { 17 "octane": 0.015, 18 "kraken": 0.019, 19 "speedometer": 0.007, 20 "speedometer2": 0.006, 21 "dromaeo.domcoreattr": 0.023, 22 "dromaeo.domcoremodify": 0.011, 23 "graphics_WebGLAquarium": 0.008, 24 "page_cycler_v2.typical_25": 0.021, 25 "loading.desktop": 0.021, # Copied from page_cycler initially 26} 27 28# Numpy makes it hard to know the real type of some inputs 29# and outputs, so this type alias is just for docs. 30FloatLike = Any 31 32 33def isf(x: FloatLike, mu=0.0, sigma=1.0, pitch=0.01) -> FloatLike: 34 """Compute the inverse survival function for value x. 35 36 In the abscence of using scipy.stats.norm's isf(), this function 37 attempts to re-implement the inverse survival function by calculating 38 the numerical inverse of the survival function, interpolating between 39 table values. See bug b/284489250 for details. 40 41 Survival function as defined by: 42 https://en.wikipedia.org/wiki/Survival_function 43 44 Examples: 45 >>> -2.0e-16 < isf(0.5) < 2.0e-16 46 True 47 48 Args: 49 x: float or numpy array-like to compute the ISF for. 50 mu: Center of the underlying normal distribution. 51 sigma: Spread of the underlying normal distribution. 52 pitch: Absolute spacing between y-value interpolation points. 53 54 Returns: 55 float or numpy array-like representing the ISF of `x`. 56 """ 57 norm = statistics.NormalDist(mu, sigma) 58 # np.interp requires a monotonically increasing x table. 59 # Because the survival table is monotonically decreasing, we have to 60 # reverse the y_vals too. 61 y_vals = np.flip(np.arange(-4.0, 4.0, pitch)) 62 survival_table = np.fromiter( 63 (1.0 - norm.cdf(y) for y in y_vals), y_vals.dtype 64 ) 65 return np.interp(x, survival_table, y_vals) 66 67 68# Get #samples needed to guarantee a given confidence interval, assuming the 69# samples follow normal distribution. 70def _samples(b: str) -> int: 71 # TODO: Make this an option 72 # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%. 73 p = 0.9 74 e = 0.02 75 if b not in _estimated_stddev: 76 return 1 77 d = _estimated_stddev[b] 78 # Get at least 2 samples so as to calculate standard deviation, which is 79 # needed in T-test for p-value. 80 n = int(math.ceil((isf((1 - p) / 2) * d / e) ** 2)) 81 return n if n > 1 else 2 82 83 84class Benchmark(object): 85 """Class representing a benchmark to be run. 86 87 Contains details of the benchmark suite, arguments to pass to the suite, 88 iterations to run the benchmark suite and so on. Note that the benchmark name 89 can be different to the test suite name. For example, you may want to have 90 two different benchmarks which run the same test_name with different 91 arguments. 92 """ 93 94 def __init__( 95 self, 96 name, 97 test_name, 98 test_args, 99 iterations, 100 rm_chroot_tmp, 101 perf_args, 102 suite="", 103 show_all_results=False, 104 retries=0, 105 run_local=False, 106 cwp_dso="", 107 weight=0, 108 ): 109 self.name = name 110 # For telemetry, this is the benchmark name. 111 self.test_name = test_name 112 # For telemetry, this is the data. 113 self.test_args = test_args 114 self.iterations = iterations if iterations > 0 else _samples(name) 115 self.perf_args = perf_args 116 self.rm_chroot_tmp = rm_chroot_tmp 117 self.iteration_adjusted = False 118 self.suite = suite 119 self.show_all_results = show_all_results 120 self.retries = retries 121 if self.suite == "telemetry": 122 self.show_all_results = True 123 if run_local and self.suite != "telemetry_Crosperf": 124 raise RuntimeError( 125 "run_local is only supported by telemetry_Crosperf." 126 ) 127 self.run_local = run_local 128 self.cwp_dso = cwp_dso 129 self.weight = weight 130