1#!/usr/bin/env python3
2#
3# Copyright 2017 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16""" Computes the diff between two qps runs and outputs significant results """
17
18import argparse
19import json
20import multiprocessing
21import os
22import shutil
23import subprocess
24import sys
25
26import qps_scenarios
27import tabulate
28
29sys.path.append(
30    os.path.join(os.path.dirname(sys.argv[0]), '..', 'microbenchmarks',
31                 'bm_diff'))
32import bm_speedup
33
34sys.path.append(
35    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
36                 'python_utils'))
37import check_on_pr
38
39
40def _args():
41    argp = argparse.ArgumentParser(description='Perform diff on QPS Driver')
42    argp.add_argument('-d',
43                      '--diff_base',
44                      type=str,
45                      help='Commit or branch to compare the current one to')
46    argp.add_argument(
47        '-l',
48        '--loops',
49        type=int,
50        default=4,
51        help='Number of loops for each benchmark. More loops cuts down on noise'
52    )
53    argp.add_argument('-j',
54                      '--jobs',
55                      type=int,
56                      default=multiprocessing.cpu_count(),
57                      help='Number of CPUs to use')
58    args = argp.parse_args()
59    assert args.diff_base, "diff_base must be set"
60    return args
61
62
63def _make_cmd(jobs):
64    return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
65
66
67def build(name, jobs):
68    shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
69    subprocess.check_call(['git', 'submodule', 'update'])
70    try:
71        subprocess.check_call(_make_cmd(jobs))
72    except subprocess.CalledProcessError as e:
73        subprocess.check_call(['make', 'clean'])
74        subprocess.check_call(_make_cmd(jobs))
75    os.rename('bins', 'qps_diff_%s' % name)
76
77
78def _run_cmd(name, scenario, fname):
79    return [
80        'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario,
81        '--json_file_out', fname
82    ]
83
84
85def run(name, scenarios, loops):
86    for sn in scenarios:
87        for i in range(0, loops):
88            fname = "%s.%s.%d.json" % (sn, name, i)
89            subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
90
91
92def _load_qps(fname):
93    try:
94        with open(fname) as f:
95            return json.loads(f.read())['qps']
96    except IOError as e:
97        print(("IOError occurred reading file: %s" % fname))
98        return None
99    except ValueError as e:
100        print(("ValueError occurred reading file: %s" % fname))
101        return None
102
103
104def _median(ary):
105    assert (len(ary))
106    ary = sorted(ary)
107    n = len(ary)
108    if n % 2 == 0:
109        return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
110    else:
111        return ary[n / 2]
112
113
114def diff(scenarios, loops, old, new):
115    old_data = {}
116    new_data = {}
117
118    # collect data
119    for sn in scenarios:
120        old_data[sn] = []
121        new_data[sn] = []
122        for i in range(loops):
123            old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
124            new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
125
126    # crunch data
127    headers = ['Benchmark', 'qps']
128    rows = []
129    for sn in scenarios:
130        mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
131        print(('%s: %s=%r %s=%r mdn_diff=%r' %
132               (sn, new, new_data[sn], old, old_data[sn], mdn_diff)))
133        s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
134        if abs(s) > 3 and mdn_diff > 0.5:
135            rows.append([sn, '%+d%%' % s])
136
137    if rows:
138        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
139    else:
140        return None
141
142
143def main(args):
144    build('new', args.jobs)
145
146    if args.diff_base:
147        where_am_i = subprocess.check_output(
148            ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode().strip()
149        subprocess.check_call(['git', 'checkout', args.diff_base])
150        try:
151            build('old', args.jobs)
152        finally:
153            subprocess.check_call(['git', 'checkout', where_am_i])
154            subprocess.check_call(['git', 'submodule', 'update'])
155
156    run('new', qps_scenarios._SCENARIOS, args.loops)
157    run('old', qps_scenarios._SCENARIOS, args.loops)
158
159    diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
160
161    if diff_output:
162        text = '[qps] Performance differences noted:\n%s' % diff_output
163    else:
164        text = '[qps] No significant performance differences'
165    print(('%s' % text))
166    check_on_pr.check_on_pr('QPS', '```\n%s\n```' % text)
167
168
169if __name__ == '__main__':
170    args = _args()
171    main(args)
172