1*cc02d7e2SAndroid Build Coastguard Worker#!/usr/bin/env python3 2*cc02d7e2SAndroid Build Coastguard Worker# Copyright 2017 gRPC authors. 3*cc02d7e2SAndroid Build Coastguard Worker# 4*cc02d7e2SAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License"); 5*cc02d7e2SAndroid Build Coastguard Worker# you may not use this file except in compliance with the License. 6*cc02d7e2SAndroid Build Coastguard Worker# You may obtain a copy of the License at 7*cc02d7e2SAndroid Build Coastguard Worker# 8*cc02d7e2SAndroid Build Coastguard Worker# http://www.apache.org/licenses/LICENSE-2.0 9*cc02d7e2SAndroid Build Coastguard Worker# 10*cc02d7e2SAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software 11*cc02d7e2SAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS, 12*cc02d7e2SAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13*cc02d7e2SAndroid Build Coastguard Worker# See the License for the specific language governing permissions and 14*cc02d7e2SAndroid Build Coastguard Worker# limitations under the License. 15*cc02d7e2SAndroid Build Coastguard Worker 16*cc02d7e2SAndroid Build Coastguard Workerimport argparse 17*cc02d7e2SAndroid Build Coastguard Workerimport html 18*cc02d7e2SAndroid Build Coastguard Workerimport multiprocessing 19*cc02d7e2SAndroid Build Coastguard Workerimport os 20*cc02d7e2SAndroid Build Coastguard Workerimport subprocess 21*cc02d7e2SAndroid Build Coastguard Workerimport sys 22*cc02d7e2SAndroid Build Coastguard Worker 23*cc02d7e2SAndroid Build Coastguard Workerimport python_utils.jobset as jobset 24*cc02d7e2SAndroid Build Coastguard Workerimport python_utils.start_port_server as start_port_server 25*cc02d7e2SAndroid Build Coastguard Worker 26*cc02d7e2SAndroid Build Coastguard Workersys.path.append( 27*cc02d7e2SAndroid Build Coastguard Worker os.path.join( 28*cc02d7e2SAndroid Build Coastguard Worker os.path.dirname(sys.argv[0]), 29*cc02d7e2SAndroid Build Coastguard Worker "..", 30*cc02d7e2SAndroid Build Coastguard Worker "profiling", 31*cc02d7e2SAndroid Build Coastguard Worker "microbenchmarks", 32*cc02d7e2SAndroid Build Coastguard Worker "bm_diff", 33*cc02d7e2SAndroid Build Coastguard Worker ) 34*cc02d7e2SAndroid Build Coastguard Worker) 35*cc02d7e2SAndroid Build Coastguard Workerimport bm_constants 36*cc02d7e2SAndroid Build Coastguard Worker 37*cc02d7e2SAndroid Build Coastguard Workerflamegraph_dir = os.path.join(os.path.expanduser("~"), "FlameGraph") 38*cc02d7e2SAndroid Build Coastguard Worker 39*cc02d7e2SAndroid Build Coastguard Workeros.chdir(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 40*cc02d7e2SAndroid Build Coastguard Workerif not os.path.exists("reports"): 41*cc02d7e2SAndroid Build Coastguard Worker os.makedirs("reports") 42*cc02d7e2SAndroid Build Coastguard Worker 43*cc02d7e2SAndroid Build Coastguard Workerstart_port_server.start_port_server() 44*cc02d7e2SAndroid Build Coastguard Worker 45*cc02d7e2SAndroid Build Coastguard Worker 46*cc02d7e2SAndroid Build Coastguard Workerdef fnize(s): 47*cc02d7e2SAndroid Build Coastguard Worker out = "" 48*cc02d7e2SAndroid Build Coastguard Worker for c in s: 49*cc02d7e2SAndroid Build Coastguard Worker if c in "<>, /": 50*cc02d7e2SAndroid Build Coastguard Worker if len(out) and out[-1] == "_": 51*cc02d7e2SAndroid Build Coastguard Worker continue 52*cc02d7e2SAndroid Build Coastguard Worker out += "_" 53*cc02d7e2SAndroid Build Coastguard Worker else: 54*cc02d7e2SAndroid Build Coastguard Worker out += c 55*cc02d7e2SAndroid Build Coastguard Worker return out 56*cc02d7e2SAndroid Build Coastguard Worker 57*cc02d7e2SAndroid Build Coastguard Worker 58*cc02d7e2SAndroid Build Coastguard Worker# index html 59*cc02d7e2SAndroid Build Coastguard Workerindex_html = """ 60*cc02d7e2SAndroid Build Coastguard Worker<html> 61*cc02d7e2SAndroid Build Coastguard Worker<head> 62*cc02d7e2SAndroid Build Coastguard Worker<title>Microbenchmark Results</title> 63*cc02d7e2SAndroid Build Coastguard Worker</head> 64*cc02d7e2SAndroid Build Coastguard Worker<body> 65*cc02d7e2SAndroid Build Coastguard Worker""" 66*cc02d7e2SAndroid Build Coastguard Worker 67*cc02d7e2SAndroid Build Coastguard Worker 68*cc02d7e2SAndroid Build Coastguard Workerdef heading(name): 69*cc02d7e2SAndroid Build Coastguard Worker global index_html 70*cc02d7e2SAndroid Build Coastguard Worker index_html += "<h1>%s</h1>\n" % name 71*cc02d7e2SAndroid Build Coastguard Worker 72*cc02d7e2SAndroid Build Coastguard Worker 73*cc02d7e2SAndroid Build Coastguard Workerdef link(txt, tgt): 74*cc02d7e2SAndroid Build Coastguard Worker global index_html 75*cc02d7e2SAndroid Build Coastguard Worker index_html += '<p><a href="%s">%s</a></p>\n' % ( 76*cc02d7e2SAndroid Build Coastguard Worker html.escape(tgt, quote=True), 77*cc02d7e2SAndroid Build Coastguard Worker html.escape(txt), 78*cc02d7e2SAndroid Build Coastguard Worker ) 79*cc02d7e2SAndroid Build Coastguard Worker 80*cc02d7e2SAndroid Build Coastguard Worker 81*cc02d7e2SAndroid Build Coastguard Workerdef text(txt): 82*cc02d7e2SAndroid Build Coastguard Worker global index_html 83*cc02d7e2SAndroid Build Coastguard Worker index_html += "<p><pre>%s</pre></p>\n" % html.escape(txt) 84*cc02d7e2SAndroid Build Coastguard Worker 85*cc02d7e2SAndroid Build Coastguard Worker 86*cc02d7e2SAndroid Build Coastguard Workerdef _bazel_build_benchmark(bm_name, cfg): 87*cc02d7e2SAndroid Build Coastguard Worker """Build given benchmark with bazel""" 88*cc02d7e2SAndroid Build Coastguard Worker subprocess.check_call( 89*cc02d7e2SAndroid Build Coastguard Worker [ 90*cc02d7e2SAndroid Build Coastguard Worker "tools/bazel", 91*cc02d7e2SAndroid Build Coastguard Worker "build", 92*cc02d7e2SAndroid Build Coastguard Worker "--config=%s" % cfg, 93*cc02d7e2SAndroid Build Coastguard Worker "//test/cpp/microbenchmarks:%s" % bm_name, 94*cc02d7e2SAndroid Build Coastguard Worker ] 95*cc02d7e2SAndroid Build Coastguard Worker ) 96*cc02d7e2SAndroid Build Coastguard Worker 97*cc02d7e2SAndroid Build Coastguard Worker 98*cc02d7e2SAndroid Build Coastguard Workerdef run_summary(bm_name, cfg, base_json_name): 99*cc02d7e2SAndroid Build Coastguard Worker _bazel_build_benchmark(bm_name, cfg) 100*cc02d7e2SAndroid Build Coastguard Worker cmd = [ 101*cc02d7e2SAndroid Build Coastguard Worker "bazel-bin/test/cpp/microbenchmarks/%s" % bm_name, 102*cc02d7e2SAndroid Build Coastguard Worker "--benchmark_out=%s.%s.json" % (base_json_name, cfg), 103*cc02d7e2SAndroid Build Coastguard Worker "--benchmark_out_format=json", 104*cc02d7e2SAndroid Build Coastguard Worker ] 105*cc02d7e2SAndroid Build Coastguard Worker if args.summary_time is not None: 106*cc02d7e2SAndroid Build Coastguard Worker cmd += ["--benchmark_min_time=%d" % args.summary_time] 107*cc02d7e2SAndroid Build Coastguard Worker return subprocess.check_output(cmd).decode("UTF-8") 108*cc02d7e2SAndroid Build Coastguard Worker 109*cc02d7e2SAndroid Build Coastguard Worker 110*cc02d7e2SAndroid Build Coastguard Workerdef collect_summary(bm_name, args): 111*cc02d7e2SAndroid Build Coastguard Worker # no counters, run microbenchmark and add summary 112*cc02d7e2SAndroid Build Coastguard Worker # both to HTML report and to console. 113*cc02d7e2SAndroid Build Coastguard Worker nocounters_heading = "Summary: %s" % bm_name 114*cc02d7e2SAndroid Build Coastguard Worker nocounters_summary = run_summary(bm_name, "opt", bm_name) 115*cc02d7e2SAndroid Build Coastguard Worker heading(nocounters_heading) 116*cc02d7e2SAndroid Build Coastguard Worker text(nocounters_summary) 117*cc02d7e2SAndroid Build Coastguard Worker print(nocounters_heading) 118*cc02d7e2SAndroid Build Coastguard Worker print(nocounters_summary) 119*cc02d7e2SAndroid Build Coastguard Worker 120*cc02d7e2SAndroid Build Coastguard Worker 121*cc02d7e2SAndroid Build Coastguard Workercollectors = { 122*cc02d7e2SAndroid Build Coastguard Worker "summary": collect_summary, 123*cc02d7e2SAndroid Build Coastguard Worker} 124*cc02d7e2SAndroid Build Coastguard Worker 125*cc02d7e2SAndroid Build Coastguard Workerargp = argparse.ArgumentParser(description="Collect data from microbenchmarks") 126*cc02d7e2SAndroid Build Coastguard Workerargp.add_argument( 127*cc02d7e2SAndroid Build Coastguard Worker "-c", 128*cc02d7e2SAndroid Build Coastguard Worker "--collect", 129*cc02d7e2SAndroid Build Coastguard Worker choices=sorted(collectors.keys()), 130*cc02d7e2SAndroid Build Coastguard Worker nargs="*", 131*cc02d7e2SAndroid Build Coastguard Worker default=sorted(collectors.keys()), 132*cc02d7e2SAndroid Build Coastguard Worker help="Which collectors should be run against each benchmark", 133*cc02d7e2SAndroid Build Coastguard Worker) 134*cc02d7e2SAndroid Build Coastguard Workerargp.add_argument( 135*cc02d7e2SAndroid Build Coastguard Worker "-b", 136*cc02d7e2SAndroid Build Coastguard Worker "--benchmarks", 137*cc02d7e2SAndroid Build Coastguard Worker choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, 138*cc02d7e2SAndroid Build Coastguard Worker default=bm_constants._AVAILABLE_BENCHMARK_TESTS, 139*cc02d7e2SAndroid Build Coastguard Worker nargs="+", 140*cc02d7e2SAndroid Build Coastguard Worker type=str, 141*cc02d7e2SAndroid Build Coastguard Worker help="Which microbenchmarks should be run", 142*cc02d7e2SAndroid Build Coastguard Worker) 143*cc02d7e2SAndroid Build Coastguard Workerargp.add_argument( 144*cc02d7e2SAndroid Build Coastguard Worker "--bq_result_table", 145*cc02d7e2SAndroid Build Coastguard Worker default="", 146*cc02d7e2SAndroid Build Coastguard Worker type=str, 147*cc02d7e2SAndroid Build Coastguard Worker help=( 148*cc02d7e2SAndroid Build Coastguard Worker "Upload results from summary collection to a specified bigquery table." 149*cc02d7e2SAndroid Build Coastguard Worker ), 150*cc02d7e2SAndroid Build Coastguard Worker) 151*cc02d7e2SAndroid Build Coastguard Workerargp.add_argument( 152*cc02d7e2SAndroid Build Coastguard Worker "--summary_time", 153*cc02d7e2SAndroid Build Coastguard Worker default=None, 154*cc02d7e2SAndroid Build Coastguard Worker type=int, 155*cc02d7e2SAndroid Build Coastguard Worker help="Minimum time to run benchmarks for the summary collection", 156*cc02d7e2SAndroid Build Coastguard Worker) 157*cc02d7e2SAndroid Build Coastguard Workerargs = argp.parse_args() 158*cc02d7e2SAndroid Build Coastguard Worker 159*cc02d7e2SAndroid Build Coastguard Workertry: 160*cc02d7e2SAndroid Build Coastguard Worker for collect in args.collect: 161*cc02d7e2SAndroid Build Coastguard Worker for bm_name in args.benchmarks: 162*cc02d7e2SAndroid Build Coastguard Worker collectors[collect](bm_name, args) 163*cc02d7e2SAndroid Build Coastguard Workerfinally: 164*cc02d7e2SAndroid Build Coastguard Worker if not os.path.exists("reports"): 165*cc02d7e2SAndroid Build Coastguard Worker os.makedirs("reports") 166*cc02d7e2SAndroid Build Coastguard Worker index_html += "</body>\n</html>\n" 167*cc02d7e2SAndroid Build Coastguard Worker with open("reports/index.html", "w") as f: 168*cc02d7e2SAndroid Build Coastguard Worker f.write(index_html) 169