1#!/usr/bin/python3
2#
3# Copyright (c) 2009-2021, Google LLC
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above copyright
11#       notice, this list of conditions and the following disclaimer in the
12#       documentation and/or other materials provided with the distribution.
13#     * Neither the name of Google LLC nor the
14#       names of its contributors may be used to endorse or promote products
15#       derived from this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
21# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28"""Benchmarks the current working directory against a given baseline.
29
30This script benchmarks both size and speed. Sample output:
31"""
32
33import contextlib
34import json
35import os
36import re
37import subprocess
38import sys
39import tempfile
40
41@contextlib.contextmanager
42def GitWorktree(commit):
43  tmpdir = tempfile.mkdtemp()
44  subprocess.run(['git', 'worktree', 'add', '-q', '-d', tmpdir, commit], check=True)
45  cwd = os.getcwd()
46  os.chdir(tmpdir)
47  try:
48    yield tmpdir
49  finally:
50    os.chdir(cwd)
51    subprocess.run(['git', 'worktree', 'remove', tmpdir], check=True)
52
53def Run(cmd):
54  subprocess.check_call(cmd, shell=True)
55
56def Benchmark(outbase, bench_cpu=True, runs=12, fasttable=False):
57  tmpfile = "/tmp/bench-output.json"
58  Run("rm -rf {}".format(tmpfile))
59  #Run("CC=clang bazel test ...")
60  if fasttable:
61    extra_args = " --//:fasttable_enabled=true"
62  else:
63    extra_args = ""
64
65  if bench_cpu:
66    Run("CC=clang bazel build -c opt --copt=-march=native benchmarks:benchmark" + extra_args)
67    Run("./bazel-bin/benchmarks/benchmark --benchmark_out_format=json --benchmark_out={} --benchmark_repetitions={} --benchmark_min_time=0.05 --benchmark_enable_random_interleaving=true".format(tmpfile, runs))
68    with open(tmpfile) as f:
69      bench_json = json.load(f)
70
71    # Translate into the format expected by benchstat.
72    txt_filename = outbase + ".txt"
73    with open(txt_filename, "w") as f:
74      for run in bench_json["benchmarks"]:
75        if run["run_type"] == "aggregate":
76          continue
77        name = run["name"]
78        name = name.replace(" ", "")
79        name = re.sub(r'^BM_', 'Benchmark', name)
80        values = (name, run["iterations"], run["cpu_time"])
81        print("{} {} {} ns/op".format(*values), file=f)
82    Run("sort {} -o {} ".format(txt_filename, txt_filename))
83
84  Run("CC=clang bazel build -c opt --copt=-g --copt=-march=native :conformance_upb"
85      + extra_args)
86  Run("cp -f bazel-bin/conformance_upb {}.bin".format(outbase))
87
88
89baseline = "main"
90bench_cpu = True
91fasttable = False
92
93if len(sys.argv) > 1:
94  baseline = sys.argv[1]
95
96  # Quickly verify that the baseline exists.
97  with GitWorktree(baseline):
98    pass
99
100# Benchmark our current directory first, since it's more likely to be broken.
101Benchmark("/tmp/new", bench_cpu, fasttable=fasttable)
102
103# Benchmark the baseline.
104with GitWorktree(baseline):
105  Benchmark("/tmp/old", bench_cpu, fasttable=fasttable)
106
107print()
108print()
109
110if bench_cpu:
111  Run("~/go/bin/benchstat /tmp/old.txt /tmp/new.txt")
112
113print()
114print()
115
116Run("objcopy --strip-debug /tmp/old.bin /tmp/old.bin.stripped")
117Run("objcopy --strip-debug /tmp/new.bin /tmp/new.bin.stripped")
118Run("~/code/bloaty/bloaty /tmp/new.bin.stripped -- /tmp/old.bin.stripped --debug-file=/tmp/old.bin --debug-file=/tmp/new.bin -d compileunits,symbols")
119