1#!/usr/bin/env python3 2# Copyright 2017 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Utilities for manipulating JSON data that represents microbenchmark results. 17 18import os 19 20# template arguments and dynamic arguments of individual benchmark types 21# Example benchmark name: "BM_UnaryPingPong<TCP, NoOpMutator, NoOpMutator>/0/0" 22_BM_SPECS = { 23 "BM_UnaryPingPong": { 24 "tpl": ["fixture", "client_mutator", "server_mutator"], 25 "dyn": ["request_size", "response_size"], 26 }, 27 "BM_PumpStreamClientToServer": { 28 "tpl": ["fixture"], 29 "dyn": ["request_size"], 30 }, 31 "BM_PumpStreamServerToClient": { 32 "tpl": ["fixture"], 33 "dyn": ["request_size"], 34 }, 35 "BM_StreamingPingPong": { 36 "tpl": ["fixture", "client_mutator", "server_mutator"], 37 "dyn": ["request_size", "request_count"], 38 }, 39 "BM_StreamingPingPongMsgs": { 40 "tpl": ["fixture", "client_mutator", "server_mutator"], 41 "dyn": ["request_size"], 42 }, 43 "BM_PumpStreamServerToClient_Trickle": { 44 "tpl": [], 45 "dyn": ["request_size", "bandwidth_kilobits"], 46 }, 47 "BM_PumpUnbalancedUnary_Trickle": { 48 "tpl": [], 49 "dyn": ["cli_req_size", "svr_req_size", "bandwidth_kilobits"], 50 }, 51 "BM_ErrorStringOnNewError": { 52 "tpl": ["fixture"], 53 "dyn": [], 54 }, 55 "BM_ErrorStringRepeatedly": { 56 "tpl": ["fixture"], 57 "dyn": [], 58 }, 59 "BM_ErrorGetStatus": { 60 "tpl": ["fixture"], 61 "dyn": [], 62 }, 63 "BM_ErrorGetStatusCode": { 64 "tpl": ["fixture"], 65 "dyn": [], 66 }, 67 "BM_ErrorHttpError": { 68 "tpl": ["fixture"], 69 "dyn": [], 70 }, 71 "BM_HasClearGrpcStatus": { 72 "tpl": ["fixture"], 73 "dyn": [], 74 }, 75 "BM_IsolatedFilter": { 76 "tpl": ["fixture", "client_mutator"], 77 "dyn": [], 78 }, 79 "BM_HpackEncoderEncodeHeader": { 80 "tpl": ["fixture"], 81 "dyn": ["end_of_stream", "request_size"], 82 }, 83 "BM_HpackParserParseHeader": { 84 "tpl": ["fixture"], 85 "dyn": [], 86 }, 87 "BM_CallCreateDestroy": { 88 "tpl": ["fixture"], 89 "dyn": [], 90 }, 91 "BM_Zalloc": { 92 "tpl": [], 93 "dyn": ["request_size"], 94 }, 95 "BM_PollEmptyPollset_SpeedOfLight": { 96 "tpl": [], 97 "dyn": ["request_size", "request_count"], 98 }, 99 "BM_StreamCreateSendInitialMetadataDestroy": { 100 "tpl": ["fixture"], 101 "dyn": [], 102 }, 103 "BM_TransportStreamSend": { 104 "tpl": [], 105 "dyn": ["request_size"], 106 }, 107 "BM_TransportStreamRecv": { 108 "tpl": [], 109 "dyn": ["request_size"], 110 }, 111 "BM_StreamingPingPongWithCoalescingApi": { 112 "tpl": ["fixture", "client_mutator", "server_mutator"], 113 "dyn": ["request_size", "request_count", "end_of_stream"], 114 }, 115 "BM_Base16SomeStuff": { 116 "tpl": [], 117 "dyn": ["request_size"], 118 }, 119} 120 121 122def numericalize(s): 123 """Convert abbreviations like '100M' or '10k' to a number.""" 124 if not s: 125 return "" 126 if s[-1] == "k": 127 return float(s[:-1]) * 1024 128 if s[-1] == "M": 129 return float(s[:-1]) * 1024 * 1024 130 if 0 <= (ord(s[-1]) - ord("0")) <= 9: 131 return float(s) 132 assert "not a number: %s" % s 133 134 135def parse_name(name): 136 cpp_name = name 137 if "<" not in name and "/" not in name and name not in _BM_SPECS: 138 return {"name": name, "cpp_name": name} 139 rest = name 140 out = {} 141 tpl_args = [] 142 dyn_args = [] 143 if "<" in rest: 144 tpl_bit = rest[rest.find("<") + 1 : rest.rfind(">")] 145 arg = "" 146 nesting = 0 147 for c in tpl_bit: 148 if c == "<": 149 nesting += 1 150 arg += c 151 elif c == ">": 152 nesting -= 1 153 arg += c 154 elif c == ",": 155 if nesting == 0: 156 tpl_args.append(arg.strip()) 157 arg = "" 158 else: 159 arg += c 160 else: 161 arg += c 162 tpl_args.append(arg.strip()) 163 rest = rest[: rest.find("<")] + rest[rest.rfind(">") + 1 :] 164 if "/" in rest: 165 s = rest.split("/") 166 rest = s[0] 167 dyn_args = s[1:] 168 name = rest 169 assert name in _BM_SPECS, "_BM_SPECS needs to be expanded for %s" % name 170 assert len(dyn_args) == len(_BM_SPECS[name]["dyn"]) 171 assert len(tpl_args) == len(_BM_SPECS[name]["tpl"]) 172 out["name"] = name 173 out["cpp_name"] = cpp_name 174 out.update( 175 dict( 176 (k, numericalize(v)) 177 for k, v in zip(_BM_SPECS[name]["dyn"], dyn_args) 178 ) 179 ) 180 out.update(dict(zip(_BM_SPECS[name]["tpl"], tpl_args))) 181 return out 182 183 184def expand_json(js): 185 if not js: 186 raise StopIteration() 187 for bm in js["benchmarks"]: 188 if bm["name"].endswith("_stddev") or bm["name"].endswith("_mean"): 189 continue 190 context = js["context"] 191 if "label" in bm: 192 labels_list = [ 193 s.split(":") 194 for s in bm["label"].strip().split(" ") 195 if len(s) and s[0] != "#" 196 ] 197 for el in labels_list: 198 el[0] = el[0].replace("/iter", "_per_iteration") 199 labels = dict(labels_list) 200 else: 201 labels = {} 202 # TODO(jtattermusch): grabbing kokoro env values shouldn't be buried 203 # deep in the JSON conversion logic. 204 # Link the data to a kokoro job run by adding 205 # well known kokoro env variables as metadata for each row 206 row = { 207 "jenkins_build": os.environ.get("KOKORO_BUILD_NUMBER", ""), 208 "jenkins_job": os.environ.get("KOKORO_JOB_NAME", ""), 209 } 210 row.update(context) 211 row.update(bm) 212 row.update(parse_name(row["name"])) 213 row.update(labels) 214 yield row 215