1# -*- coding: utf-8 -*- 2# Copyright 2013 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Parse data from benchmark_runs for tabulator.""" 7 8 9import errno 10import json 11import os 12import re 13import sys 14 15from cros_utils import misc 16 17 18_TELEMETRY_RESULT_DEFAULTS_FILE = "default-telemetry-results.json" 19_DUP_KEY_REGEX = re.compile(r"(\w+)\{(\d+)\}") 20 21 22def _AdjustIteration(benchmarks, max_dup, bench): 23 """Adjust the interation numbers if they have keys like ABCD{i}.""" 24 for benchmark in benchmarks: 25 if benchmark.name != bench or benchmark.iteration_adjusted: 26 continue 27 benchmark.iteration_adjusted = True 28 benchmark.iterations *= max_dup + 1 29 30 31def _GetMaxDup(data): 32 """Find the maximum i inside ABCD{i}. 33 34 data should be a [[[Key]]], where Key is a string that may look like 35 ABCD{i}. 36 """ 37 max_dup = 0 38 for label in data: 39 for run in label: 40 for key in run: 41 match = _DUP_KEY_REGEX.match(key) 42 if match: 43 max_dup = max(max_dup, int(match.group(2))) 44 return max_dup 45 46 47def _Repeat(func, times): 48 """Returns the result of running func() n times.""" 49 return [func() for _ in range(times)] 50 51 52def _DictWithReturnValues(retval, pass_fail): 53 """Create a new dictionary pre-populated with success/fail values.""" 54 new_dict = {} 55 # Note: 0 is a valid retval; test to make sure it's not None. 56 if retval is not None: 57 new_dict["retval"] = retval 58 if pass_fail: 59 new_dict[""] = pass_fail 60 return new_dict 61 62 63def _GetNonDupLabel(max_dup, runs): 64 """Create new list for the runs of the same label. 65 66 Specifically, this will split out keys like foo{0}, foo{1} from one run into 67 their own runs. For example, given a run like: 68 {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5} 69 70 You'll get: 71 [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}] 72 73 Hands back the lists of transformed runs, all concatenated together. 74 """ 75 new_runs = [] 76 for run in runs: 77 run_retval = run.get("retval", None) 78 run_pass_fail = run.get("", None) 79 new_run = {} 80 # pylint: disable=cell-var-from-loop 81 added_runs = _Repeat( 82 lambda: _DictWithReturnValues(run_retval, run_pass_fail), max_dup 83 ) 84 for key, value in run.items(): 85 match = _DUP_KEY_REGEX.match(key) 86 if not match: 87 new_run[key] = value 88 else: 89 new_key, index_str = match.groups() 90 added_runs[int(index_str) - 1][new_key] = str(value) 91 new_runs.append(new_run) 92 new_runs += added_runs 93 return new_runs 94 95 96def _DuplicatePass(result, benchmarks): 97 """Properly expands keys like `foo{1}` in `result`.""" 98 for bench, data in result.items(): 99 max_dup = _GetMaxDup(data) 100 # If there's nothing to expand, there's nothing to do. 101 if not max_dup: 102 continue 103 for i, runs in enumerate(data): 104 data[i] = _GetNonDupLabel(max_dup, runs) 105 _AdjustIteration(benchmarks, max_dup, bench) 106 107 108def _ReadSummaryFile(filename): 109 """Reads the summary file at filename.""" 110 dirname, _ = misc.GetRoot(filename) 111 fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE) 112 try: 113 # Slurp the summary file into a dictionary. The keys in the dictionary are 114 # the benchmark names. The value for a key is a list containing the names 115 # of all the result fields that should be returned in a 'default' report. 116 with open(fullname) as in_file: 117 return json.load(in_file) 118 except IOError as e: 119 # ENOENT means "no such file or directory" 120 if e.errno == errno.ENOENT: 121 return {} 122 raise 123 124 125def _MakeOrganizeResultOutline(benchmark_runs, labels): 126 """Creates the "outline" of the OrganizeResults result for a set of runs. 127 128 Report generation returns lists of different sizes, depending on the input 129 data. Depending on the order in which we iterate through said input data, we 130 may populate the Nth index of a list, then the N-1st, then the N+Mth, ... 131 132 It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't 133 have to worry about resizing while computing results. 134 """ 135 # Count how many iterations exist for each benchmark run. 136 # We can't simply count up, since we may be given an incomplete set of 137 # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3]) 138 iteration_count = {} 139 for run in benchmark_runs: 140 name = run.benchmark.name 141 old_iterations = iteration_count.get(name, -1) 142 # N.B. run.iteration starts at 1, not 0. 143 iteration_count[name] = max(old_iterations, run.iteration) 144 145 # Result structure: {benchmark_name: [[{key: val}]]} 146 result = {} 147 for run in benchmark_runs: 148 name = run.benchmark.name 149 num_iterations = iteration_count[name] 150 # default param makes cros lint be quiet about defining num_iterations in a 151 # loop. 152 make_dicts = lambda n=num_iterations: _Repeat(dict, n) 153 result[name] = _Repeat(make_dicts, len(labels)) 154 return result 155 156 157def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): 158 """Create a dict from benchmark_runs. 159 160 The structure of the output dict is as follows: 161 {"benchmark_1":[ 162 [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}] 163 #one label 164 [] 165 #the other label 166 ] 167 "benchmark_2": 168 [ 169 ]}. 170 """ 171 result = _MakeOrganizeResultOutline(benchmark_runs, labels) 172 label_names = [label.name for label in labels] 173 label_indices = {name: i for i, name in enumerate(label_names)} 174 summary_file = _ReadSummaryFile(sys.argv[0]) 175 176 if benchmarks is None: 177 benchmarks = [] 178 179 for benchmark_run in benchmark_runs: 180 if not benchmark_run.result: 181 continue 182 benchmark = benchmark_run.benchmark 183 label_index = label_indices[benchmark_run.label.name] 184 cur_label_list = result[benchmark.name][label_index] 185 cur_dict = cur_label_list[benchmark_run.iteration - 1] 186 187 show_all_results = json_report or benchmark.show_all_results 188 if not show_all_results: 189 summary_list = summary_file.get(benchmark.name) 190 if summary_list: 191 for key in benchmark_run.result.keyvals.keys(): 192 if any( 193 key.startswith(added_key) 194 for added_key in ["retval", "cpufreq", "cputemp"] 195 ): 196 summary_list.append(key) 197 else: 198 # Did not find test_name in json file; show everything. 199 show_all_results = True 200 if benchmark_run.result.cwp_dso: 201 # If we are in cwp approximation mode, we only care about samples 202 if "samples" in benchmark_run.result.keyvals: 203 cur_dict["samples"] = benchmark_run.result.keyvals["samples"] 204 cur_dict["retval"] = benchmark_run.result.keyvals["retval"] 205 for key, value in benchmark_run.result.keyvals.items(): 206 if any( 207 key.startswith(cpustat_keyword) 208 for cpustat_keyword in ["cpufreq", "cputemp"] 209 ): 210 cur_dict[key] = value 211 else: 212 for test_key in benchmark_run.result.keyvals: 213 if show_all_results or test_key in summary_list: 214 cur_dict[test_key] = benchmark_run.result.keyvals[test_key] 215 # Occasionally Telemetry tests will not fail but they will not return a 216 # result, either. Look for those cases, and force them to be a fail. 217 # (This can happen if, for example, the test has been disabled.) 218 if len(cur_dict) == 1 and cur_dict["retval"] == 0: 219 cur_dict["retval"] = 1 220 benchmark_run.result.keyvals["retval"] = 1 221 # TODO: This output should be sent via logger. 222 print( 223 "WARNING: Test '%s' appears to have succeeded but returned" 224 " no results." % benchmark.name, 225 file=sys.stderr, 226 ) 227 if json_report and benchmark_run.machine: 228 cur_dict["machine"] = benchmark_run.machine.name 229 cur_dict["machine_checksum"] = benchmark_run.machine.checksum 230 cur_dict["machine_string"] = benchmark_run.machine.checksum_string 231 _DuplicatePass(result, benchmarks) 232 return result 233