1#!/usr/bin/env python3 2 3# Copyright 2021 The gRPC Authors 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19import json 20import os 21import sys 22 23import yaml 24 25run_tests_root = os.path.abspath( 26 os.path.join(os.path.dirname(sys.argv[0]), "../../../tools/run_tests") 27) 28sys.path.append(run_tests_root) 29 30import performance.scenario_config as scenario_config 31 32_COPYRIGHT = """# Copyright 2021 The gRPC Authors 33# 34# Licensed under the Apache License, Version 2.0 (the "License"); 35# you may not use this file except in compliance with the License. 36# You may obtain a copy of the License at 37# 38# http://www.apache.org/licenses/LICENSE-2.0 39# 40# Unless required by applicable law or agreed to in writing, software 41# distributed under the License is distributed on an "AS IS" BASIS, 42# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 43# See the License for the specific language governing permissions and 44# limitations under the License. 45""" 46 47 48def _mutate_scenario(scenario_json): 49 """Modifies vanilla benchmark scenario config to make it more suitable for running as a unit test.""" 50 # tweak parameters to get fast test times 51 scenario_json = dict(scenario_json) 52 scenario_json["warmup_seconds"] = 0 53 scenario_json["benchmark_seconds"] = 1 54 outstanding_rpcs_divisor = 1 55 if ( 56 scenario_json["client_config"]["client_type"] == "SYNC_CLIENT" 57 or scenario_json["server_config"]["server_type"] == "SYNC_SERVER" 58 ): 59 # reduce the number of threads needed for scenarios that use synchronous API 60 outstanding_rpcs_divisor = 10 61 scenario_json["client_config"]["outstanding_rpcs_per_channel"] = max( 62 1, 63 scenario_json["client_config"]["outstanding_rpcs_per_channel"] 64 // outstanding_rpcs_divisor, 65 ) 66 # Some scenarios use high channel count since when actually 67 # benchmarking, we want to saturate the machine that runs the benchmark. 68 # For unit test, this is an overkill. 69 max_client_channels = 16 70 if scenario_json["client_config"]["rpc_type"] == "STREAMING_FROM_SERVER": 71 # streaming from server scenarios tend to have trouble shutting down 72 # quickly if there are too many channels. 73 max_client_channels = 4 74 75 scenario_json["client_config"]["client_channels"] = min( 76 max_client_channels, scenario_json["client_config"]["client_channels"] 77 ) 78 79 return scenario_config.remove_nonproto_fields(scenario_json) 80 81 82def generate_json_run_localhost_scenarios(): 83 return [ 84 _mutate_scenario(scenario_json) 85 for scenario_json in scenario_config.CXXLanguage().scenarios() 86 if "scalable" in scenario_json.get("CATEGORIES", []) 87 ] 88 89 90def generate_qps_json_driver_scenarios(): 91 return [ 92 _mutate_scenario(scenario_json) 93 for scenario_json in scenario_config.CXXLanguage().scenarios() 94 if "inproc" in scenario_json.get("CATEGORIES", []) 95 ] 96 97 98def generate_scenarios_bzl(json_scenarios, bzl_filename, bzl_variablename): 99 """Generate .bzl file that defines a variable with JSON scenario configs.""" 100 all_scenarios = [] 101 for scenario in json_scenarios: 102 scenario_name = scenario["name"] 103 # argument will be passed as "--scenarios_json" to the test binary 104 # the string needs to be quoted in \' to ensure it gets passed as a single argument in shell 105 scenarios_json_arg_str = "\\'%s\\'" % json.dumps( 106 {"scenarios": [scenario]} 107 ) 108 all_scenarios.append((scenario_name, scenarios_json_arg_str)) 109 110 with open(bzl_filename, "w") as f: 111 f.write(_COPYRIGHT) 112 f.write( 113 '"""AUTOGENERATED: configuration of benchmark scenarios to be run' 114 ' as bazel test"""\n\n' 115 ) 116 f.write("%s = {\n" % bzl_variablename) 117 for scenario in all_scenarios: 118 f.write(" \"%s\": '%s',\n" % (scenario[0], scenario[1])) 119 f.write("}\n") 120