1# Copyright 2016 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# performance scenario configuration for various languages 16 17import math 18 19WARMUP_SECONDS = 5 20JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in. 21CXX_WARMUP_SECONDS = 30 # Core needs more warmup time for the thread pool to scale appropriately. 22BENCHMARK_SECONDS = 30 23 24SMOKETEST = "smoketest" 25SCALABLE = "scalable" 26INPROC = "inproc" 27SWEEP = "sweep" 28PSM = "psm" 29# A small superset of the benchmarks required to produce 30# https://grafana-dot-grpc-testing.appspot.com/ 31DASHBOARD = "dashboard" 32DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST) 33 34SECURE_SECARGS = { 35 "use_test_ca": True, 36 "server_host_override": "foo.test.google.fr", 37} 38 39HISTOGRAM_PARAMS = { 40 "resolution": 0.01, 41 "max_possible": 60e9, 42} 43 44# target number of RPCs outstanding on across all client channels in 45# non-ping-pong tests (since we can only specify per-channel numbers, the 46# actual target will be slightly higher) 47OUTSTANDING_REQUESTS = { 48 "async": 6400, 49 "async-limited": 800, 50 "sync": 1000, 51 "callback": 6400, 52} 53 54# wide is the number of client channels in multi-channel tests (1 otherwise) 55WIDE = 64 56 57 58def _get_secargs(is_secure): 59 if is_secure: 60 return SECURE_SECARGS 61 else: 62 return None 63 64 65def remove_nonproto_fields(scenario): 66 """Removes special-purpose fields that don't belong in the protobuf. 67 68 This function removes additional information about the scenario that is not 69 included in the ScenarioConfig protobuf message. 70 """ 71 scenario.pop("CATEGORIES", None) 72 scenario.pop("CLIENT_LANGUAGE", None) 73 scenario.pop("SERVER_LANGUAGE", None) 74 scenario.pop("EXCLUDED_POLL_ENGINES", None) 75 return scenario 76 77 78def geometric_progression(start, stop, step): 79 n = start 80 while n < stop: 81 yield int(round(n)) 82 n *= step 83 84 85def _payload_type(use_generic_payload, req_size, resp_size): 86 r = {} 87 sizes = { 88 "req_size": req_size, 89 "resp_size": resp_size, 90 } 91 if use_generic_payload: 92 r["bytebuf_params"] = sizes 93 else: 94 r["simple_params"] = sizes 95 return r 96 97 98def _load_params(offered_load): 99 r = {} 100 if offered_load is None: 101 r["closed_loop"] = {} 102 else: 103 load = {} 104 load["offered_load"] = offered_load 105 r["poisson"] = load 106 return r 107 108 109def _add_channel_arg(config, key, value): 110 if "channel_args" in config: 111 channel_args = config["channel_args"] 112 else: 113 channel_args = [] 114 config["channel_args"] = channel_args 115 arg = {"name": key} 116 if isinstance(value, int): 117 arg["int_value"] = value 118 else: 119 arg["str_value"] = value 120 channel_args.append(arg) 121 122 123def _ping_pong_scenario( 124 name, 125 rpc_type, 126 client_type, 127 server_type, 128 secure=True, 129 use_generic_payload=False, 130 req_size=0, 131 resp_size=0, 132 unconstrained_client=None, 133 client_language=None, 134 server_language=None, 135 async_server_threads=0, 136 client_processes=0, 137 server_processes=0, 138 server_threads_per_cq=0, 139 client_threads_per_cq=0, 140 warmup_seconds=WARMUP_SECONDS, 141 categories=None, 142 channels=None, 143 outstanding=None, 144 num_clients=None, 145 resource_quota_size=None, 146 messages_per_stream=None, 147 excluded_poll_engines=None, 148 minimal_stack=False, 149 offered_load=None, 150): 151 """Creates a basic ping pong scenario.""" 152 scenario = { 153 "name": name, 154 "num_servers": 1, 155 "num_clients": 1, 156 "client_config": { 157 "client_type": client_type, 158 "security_params": _get_secargs(secure), 159 "outstanding_rpcs_per_channel": 1, 160 "client_channels": 1, 161 "async_client_threads": 1, 162 "client_processes": client_processes, 163 "threads_per_cq": client_threads_per_cq, 164 "rpc_type": rpc_type, 165 "histogram_params": HISTOGRAM_PARAMS, 166 "channel_args": [], 167 }, 168 "server_config": { 169 "server_type": server_type, 170 "security_params": _get_secargs(secure), 171 "async_server_threads": async_server_threads, 172 "server_processes": server_processes, 173 "threads_per_cq": server_threads_per_cq, 174 "channel_args": [], 175 }, 176 "warmup_seconds": warmup_seconds, 177 "benchmark_seconds": BENCHMARK_SECONDS, 178 "CATEGORIES": list(DEFAULT_CATEGORIES), 179 "EXCLUDED_POLL_ENGINES": [], 180 } 181 if resource_quota_size: 182 scenario["server_config"]["resource_quota_size"] = resource_quota_size 183 if use_generic_payload: 184 if server_type != "ASYNC_GENERIC_SERVER": 185 raise Exception("Use ASYNC_GENERIC_SERVER for generic payload.") 186 scenario["server_config"]["payload_config"] = _payload_type( 187 use_generic_payload, req_size, resp_size 188 ) 189 190 scenario["client_config"]["payload_config"] = _payload_type( 191 use_generic_payload, req_size, resp_size 192 ) 193 194 # Optimization target of 'throughput' does not work well with epoll1 polling 195 # engine. Use the default value of 'blend' 196 optimization_target = "throughput" 197 198 if unconstrained_client: 199 outstanding_calls = ( 200 outstanding 201 if outstanding is not None 202 else OUTSTANDING_REQUESTS[unconstrained_client] 203 ) 204 # clamp buffer usage to something reasonable (16 gig for now) 205 MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 206 if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: 207 outstanding_calls = max( 208 1, MAX_MEMORY_USE / max(req_size, resp_size) 209 ) 210 wide = channels if channels is not None else WIDE 211 deep = int(math.ceil(1.0 * outstanding_calls / wide)) 212 213 scenario["num_clients"] = ( 214 num_clients if num_clients is not None else 0 215 ) # use as many clients as available. 216 scenario["client_config"]["outstanding_rpcs_per_channel"] = deep 217 scenario["client_config"]["client_channels"] = wide 218 scenario["client_config"]["async_client_threads"] = 0 219 if offered_load is not None: 220 optimization_target = "latency" 221 else: 222 scenario["client_config"]["outstanding_rpcs_per_channel"] = 1 223 scenario["client_config"]["client_channels"] = 1 224 scenario["client_config"]["async_client_threads"] = 1 225 optimization_target = "latency" 226 227 scenario["client_config"]["load_params"] = _load_params(offered_load) 228 229 optimization_channel_arg = { 230 "name": "grpc.optimization_target", 231 "str_value": optimization_target, 232 } 233 scenario["client_config"]["channel_args"].append(optimization_channel_arg) 234 scenario["server_config"]["channel_args"].append(optimization_channel_arg) 235 236 if minimal_stack: 237 _add_channel_arg(scenario["client_config"], "grpc.minimal_stack", 1) 238 _add_channel_arg(scenario["server_config"], "grpc.minimal_stack", 1) 239 240 if messages_per_stream: 241 scenario["client_config"]["messages_per_stream"] = messages_per_stream 242 if client_language: 243 # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py 244 scenario["CLIENT_LANGUAGE"] = client_language 245 if server_language: 246 # the SERVER_LANGUAGE field is recognized by run_performance_tests.py 247 scenario["SERVER_LANGUAGE"] = server_language 248 if categories: 249 scenario["CATEGORIES"] = categories 250 if excluded_poll_engines: 251 # The polling engines for which this scenario is excluded 252 scenario["EXCLUDED_POLL_ENGINES"] = excluded_poll_engines 253 return scenario 254 255 256class Language(object): 257 @property 258 def safename(self): 259 return str(self) 260 261 262class CXXLanguage(Language): 263 @property 264 def safename(self): 265 return "cxx" 266 267 def worker_cmdline(self): 268 return ["cmake/build/qps_worker"] 269 270 def worker_port_offset(self): 271 return 0 272 273 def scenarios(self): 274 yield _ping_pong_scenario( 275 "cpp_protobuf_async_unary_5000rpcs_1KB_psm", 276 rpc_type="UNARY", 277 client_type="ASYNC_CLIENT", 278 server_type="ASYNC_SERVER", 279 req_size=1024, 280 resp_size=1024, 281 outstanding=5000, 282 channels=1, 283 num_clients=1, 284 secure=False, 285 async_server_threads=1, 286 categories=[PSM], 287 warmup_seconds=CXX_WARMUP_SECONDS, 288 ) 289 290 # TODO(ctiller): add 70% load latency test 291 yield _ping_pong_scenario( 292 "cpp_protobuf_async_unary_1channel_100rpcs_1MB", 293 rpc_type="UNARY", 294 client_type="ASYNC_CLIENT", 295 server_type="ASYNC_SERVER", 296 req_size=1024 * 1024, 297 resp_size=1024 * 1024, 298 unconstrained_client="async", 299 outstanding=100, 300 channels=1, 301 num_clients=1, 302 secure=False, 303 categories=[SWEEP], 304 warmup_seconds=CXX_WARMUP_SECONDS, 305 ) 306 307 yield _ping_pong_scenario( 308 "cpp_protobuf_async_streaming_from_client_1channel_1MB", 309 rpc_type="STREAMING_FROM_CLIENT", 310 client_type="ASYNC_CLIENT", 311 server_type="ASYNC_SERVER", 312 req_size=1024 * 1024, 313 resp_size=1024 * 1024, 314 unconstrained_client="async", 315 outstanding=1, 316 channels=1, 317 num_clients=1, 318 secure=False, 319 categories=[SWEEP], 320 warmup_seconds=CXX_WARMUP_SECONDS, 321 ) 322 323 # Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear 324 # (beyond excercising some params that other scenarios don't) 325 yield _ping_pong_scenario( 326 "cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp", 327 rpc_type="UNARY", 328 client_type="ASYNC_CLIENT", 329 server_type="ASYNC_SERVER", 330 req_size=300, 331 resp_size=50, 332 unconstrained_client="async", 333 outstanding=30000, 334 channels=300, 335 offered_load=37500, 336 secure=False, 337 async_server_threads=16, 338 server_threads_per_cq=1, 339 categories=[SCALABLE], 340 warmup_seconds=CXX_WARMUP_SECONDS, 341 ) 342 343 for secure in [True, False]: 344 secstr = "secure" if secure else "insecure" 345 smoketest_categories = [SMOKETEST] if secure else [] 346 inproc_categories = [INPROC] if not secure else [] 347 348 yield _ping_pong_scenario( 349 "cpp_generic_async_streaming_ping_pong_%s" % secstr, 350 rpc_type="STREAMING", 351 client_type="ASYNC_CLIENT", 352 server_type="ASYNC_GENERIC_SERVER", 353 use_generic_payload=True, 354 async_server_threads=1, 355 secure=secure, 356 categories=smoketest_categories 357 + inproc_categories 358 + [SCALABLE], 359 warmup_seconds=CXX_WARMUP_SECONDS, 360 ) 361 362 yield _ping_pong_scenario( 363 "cpp_generic_async_streaming_qps_unconstrained_%s" % secstr, 364 rpc_type="STREAMING", 365 client_type="ASYNC_CLIENT", 366 server_type="ASYNC_GENERIC_SERVER", 367 unconstrained_client="async", 368 use_generic_payload=True, 369 secure=secure, 370 client_threads_per_cq=2, 371 server_threads_per_cq=2, 372 minimal_stack=not secure, 373 categories=smoketest_categories 374 + inproc_categories 375 + [SCALABLE], 376 warmup_seconds=CXX_WARMUP_SECONDS, 377 ) 378 379 for mps in geometric_progression(10, 20, 10): 380 yield _ping_pong_scenario( 381 "cpp_generic_async_streaming_qps_unconstrained_%smps_%s" 382 % (mps, secstr), 383 rpc_type="STREAMING", 384 client_type="ASYNC_CLIENT", 385 server_type="ASYNC_GENERIC_SERVER", 386 unconstrained_client="async", 387 use_generic_payload=True, 388 secure=secure, 389 messages_per_stream=mps, 390 minimal_stack=not secure, 391 categories=smoketest_categories 392 + inproc_categories 393 + [SCALABLE], 394 warmup_seconds=CXX_WARMUP_SECONDS, 395 ) 396 397 for mps in geometric_progression(1, 200, math.sqrt(10)): 398 yield _ping_pong_scenario( 399 "cpp_generic_async_streaming_qps_unconstrained_%smps_%s" 400 % (mps, secstr), 401 rpc_type="STREAMING", 402 client_type="ASYNC_CLIENT", 403 server_type="ASYNC_GENERIC_SERVER", 404 unconstrained_client="async", 405 use_generic_payload=True, 406 secure=secure, 407 messages_per_stream=mps, 408 minimal_stack=not secure, 409 categories=[SWEEP], 410 warmup_seconds=CXX_WARMUP_SECONDS, 411 ) 412 413 yield _ping_pong_scenario( 414 "cpp_generic_async_streaming_qps_1channel_1MBmsg_%s" % secstr, 415 rpc_type="STREAMING", 416 req_size=1024 * 1024, 417 resp_size=1024 * 1024, 418 client_type="ASYNC_CLIENT", 419 server_type="ASYNC_GENERIC_SERVER", 420 unconstrained_client="async", 421 use_generic_payload=True, 422 secure=secure, 423 minimal_stack=not secure, 424 categories=inproc_categories + [SCALABLE], 425 channels=1, 426 outstanding=100, 427 warmup_seconds=CXX_WARMUP_SECONDS, 428 ) 429 430 yield _ping_pong_scenario( 431 "cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s" 432 % secstr, 433 rpc_type="STREAMING", 434 req_size=64 * 1024, 435 resp_size=64 * 1024, 436 client_type="ASYNC_CLIENT", 437 server_type="ASYNC_GENERIC_SERVER", 438 unconstrained_client="async", 439 use_generic_payload=True, 440 secure=secure, 441 minimal_stack=not secure, 442 categories=inproc_categories + [SCALABLE], 443 warmup_seconds=CXX_WARMUP_SECONDS, 444 ) 445 446 yield _ping_pong_scenario( 447 "cpp_generic_async_streaming_qps_unconstrained_1cq_%s" % secstr, 448 rpc_type="STREAMING", 449 client_type="ASYNC_CLIENT", 450 server_type="ASYNC_GENERIC_SERVER", 451 unconstrained_client="async-limited", 452 use_generic_payload=True, 453 secure=secure, 454 client_threads_per_cq=1000000, 455 server_threads_per_cq=1000000, 456 categories=[SWEEP], 457 warmup_seconds=CXX_WARMUP_SECONDS, 458 ) 459 460 yield _ping_pong_scenario( 461 "cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s" 462 % secstr, 463 rpc_type="STREAMING", 464 client_type="ASYNC_CLIENT", 465 server_type="ASYNC_SERVER", 466 unconstrained_client="async-limited", 467 secure=secure, 468 client_threads_per_cq=1000000, 469 server_threads_per_cq=1000000, 470 categories=inproc_categories + [SCALABLE], 471 warmup_seconds=CXX_WARMUP_SECONDS, 472 ) 473 474 yield _ping_pong_scenario( 475 "cpp_protobuf_async_unary_qps_unconstrained_1cq_%s" % secstr, 476 rpc_type="UNARY", 477 client_type="ASYNC_CLIENT", 478 server_type="ASYNC_SERVER", 479 unconstrained_client="async-limited", 480 secure=secure, 481 client_threads_per_cq=1000000, 482 server_threads_per_cq=1000000, 483 categories=inproc_categories + [SCALABLE], 484 warmup_seconds=CXX_WARMUP_SECONDS, 485 ) 486 487 yield _ping_pong_scenario( 488 "cpp_generic_async_streaming_qps_one_server_core_%s" % secstr, 489 rpc_type="STREAMING", 490 client_type="ASYNC_CLIENT", 491 server_type="ASYNC_GENERIC_SERVER", 492 unconstrained_client="async-limited", 493 use_generic_payload=True, 494 async_server_threads=1, 495 minimal_stack=not secure, 496 secure=secure, 497 categories=[SWEEP], 498 warmup_seconds=CXX_WARMUP_SECONDS, 499 ) 500 501 yield _ping_pong_scenario( 502 "cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s" 503 % (secstr), 504 rpc_type="UNARY", 505 client_type="ASYNC_CLIENT", 506 server_type="SYNC_SERVER", 507 unconstrained_client="async", 508 secure=secure, 509 minimal_stack=not secure, 510 categories=smoketest_categories 511 + inproc_categories 512 + [SCALABLE], 513 warmup_seconds=CXX_WARMUP_SECONDS, 514 ) 515 516 yield _ping_pong_scenario( 517 "cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s" 518 % (secstr), 519 rpc_type="UNARY", 520 client_type="ASYNC_CLIENT", 521 server_type="ASYNC_SERVER", 522 channels=1, 523 outstanding=64, 524 req_size=128, 525 resp_size=8 * 1024 * 1024, 526 secure=secure, 527 minimal_stack=not secure, 528 categories=inproc_categories + [SCALABLE], 529 warmup_seconds=CXX_WARMUP_SECONDS, 530 ) 531 532 yield _ping_pong_scenario( 533 "cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s" 534 % secstr, 535 rpc_type="STREAMING", 536 client_type="ASYNC_CLIENT", 537 server_type="SYNC_SERVER", 538 unconstrained_client="async", 539 secure=secure, 540 minimal_stack=not secure, 541 categories=[SWEEP], 542 warmup_seconds=CXX_WARMUP_SECONDS, 543 ) 544 545 yield _ping_pong_scenario( 546 "cpp_protobuf_async_unary_ping_pong_%s_1MB" % secstr, 547 rpc_type="UNARY", 548 client_type="ASYNC_CLIENT", 549 server_type="ASYNC_SERVER", 550 req_size=1024 * 1024, 551 resp_size=1024 * 1024, 552 secure=secure, 553 minimal_stack=not secure, 554 categories=smoketest_categories 555 + inproc_categories 556 + [SCALABLE, DASHBOARD], 557 warmup_seconds=CXX_WARMUP_SECONDS, 558 ) 559 560 for rpc_type in [ 561 "unary", 562 "streaming", 563 "streaming_from_client", 564 "streaming_from_server", 565 ]: 566 maybe_dashboard = ( 567 [DASHBOARD] if rpc_type in ("unary", "streaming") else [] 568 ) 569 for synchronicity in ["sync", "async", "callback"]: 570 # TODO(hork): Re-enable when the callback scenarios support 571 # one-sided streaming. 572 if synchronicity == "callback" and rpc_type in ( 573 "streaming_from_client", 574 "streaming_from_server", 575 ): 576 continue 577 yield _ping_pong_scenario( 578 "cpp_protobuf_%s_%s_ping_pong_%s" 579 % (synchronicity, rpc_type, secstr), 580 rpc_type=rpc_type.upper(), 581 client_type="%s_CLIENT" % synchronicity.upper(), 582 server_type="%s_SERVER" % synchronicity.upper(), 583 async_server_threads=1, 584 minimal_stack=not secure, 585 secure=secure, 586 categories=list(DEFAULT_CATEGORIES) + maybe_dashboard, 587 warmup_seconds=CXX_WARMUP_SECONDS, 588 ) 589 for size in geometric_progression( 590 1, 1024 * 1024 * 1024 + 1, 8 591 ): 592 yield _ping_pong_scenario( 593 "cpp_protobuf_%s_%s_qps_unconstrained_%s_%db" 594 % (synchronicity, rpc_type, secstr, size), 595 rpc_type=rpc_type.upper(), 596 req_size=size, 597 resp_size=size, 598 client_type="%s_CLIENT" % synchronicity.upper(), 599 server_type="%s_SERVER" % synchronicity.upper(), 600 unconstrained_client=synchronicity, 601 secure=secure, 602 minimal_stack=not secure, 603 categories=[SWEEP], 604 warmup_seconds=CXX_WARMUP_SECONDS, 605 ) 606 607 maybe_scalable = [SCALABLE] 608 if ( 609 rpc_type == "streaming_from_server" 610 and synchronicity == "async" 611 and secure 612 ): 613 # protobuf_async_streaming_from_server_qps_unconstrained_secure is very flaky 614 # and has extremely high variance so running it isn't really useful. 615 # see b/198275705 616 maybe_scalable = [SWEEP] 617 618 maybe_dashboard = ( 619 [DASHBOARD] 620 if rpc_type in ("unary", "streaming") 621 else [] 622 ) 623 yield _ping_pong_scenario( 624 "cpp_protobuf_%s_%s_qps_unconstrained_%s" 625 % (synchronicity, rpc_type, secstr), 626 rpc_type=rpc_type.upper(), 627 client_type="%s_CLIENT" % synchronicity.upper(), 628 server_type="%s_SERVER" % synchronicity.upper(), 629 unconstrained_client=synchronicity, 630 secure=secure, 631 minimal_stack=not secure, 632 server_threads_per_cq=2, 633 client_threads_per_cq=2, 634 categories=inproc_categories 635 + maybe_scalable 636 + maybe_dashboard, 637 warmup_seconds=CXX_WARMUP_SECONDS, 638 ) 639 640 # TODO(vjpai): Re-enable this test. It has a lot of timeouts 641 # and hasn't yet been conclusively identified as a test failure 642 # or race in the library 643 # yield _ping_pong_scenario( 644 # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), 645 # rpc_type=rpc_type.upper(), 646 # client_type='%s_CLIENT' % synchronicity.upper(), 647 # server_type='%s_SERVER' % synchronicity.upper(), 648 # unconstrained_client=synchronicity, 649 # secure=secure, 650 # categories=smoketest_categories+[SCALABLE], 651 # warmup_seconds=CXX_WARMUP_SECONDS, 652 # resource_quota_size=500*1024) 653 654 if rpc_type == "streaming": 655 for mps in geometric_progression(10, 20, 10): 656 yield _ping_pong_scenario( 657 "cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s" 658 % (synchronicity, rpc_type, mps, secstr), 659 rpc_type=rpc_type.upper(), 660 client_type="%s_CLIENT" % synchronicity.upper(), 661 server_type="%s_SERVER" % synchronicity.upper(), 662 unconstrained_client=synchronicity, 663 secure=secure, 664 messages_per_stream=mps, 665 minimal_stack=not secure, 666 categories=inproc_categories + [SCALABLE], 667 warmup_seconds=CXX_WARMUP_SECONDS, 668 ) 669 670 for mps in geometric_progression(1, 200, math.sqrt(10)): 671 yield _ping_pong_scenario( 672 "cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s" 673 % (synchronicity, rpc_type, mps, secstr), 674 rpc_type=rpc_type.upper(), 675 client_type="%s_CLIENT" % synchronicity.upper(), 676 server_type="%s_SERVER" % synchronicity.upper(), 677 unconstrained_client=synchronicity, 678 secure=secure, 679 messages_per_stream=mps, 680 minimal_stack=not secure, 681 categories=[SWEEP], 682 warmup_seconds=CXX_WARMUP_SECONDS, 683 ) 684 685 for channels in geometric_progression( 686 1, 20000, math.sqrt(10) 687 ): 688 for outstanding in geometric_progression( 689 1, 200000, math.sqrt(10) 690 ): 691 if synchronicity == "sync" and outstanding > 1200: 692 continue 693 if outstanding < channels: 694 continue 695 yield _ping_pong_scenario( 696 "cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding" 697 % ( 698 synchronicity, 699 rpc_type, 700 secstr, 701 channels, 702 outstanding, 703 ), 704 rpc_type=rpc_type.upper(), 705 client_type="%s_CLIENT" % synchronicity.upper(), 706 server_type="%s_SERVER" % synchronicity.upper(), 707 unconstrained_client=synchronicity, 708 secure=secure, 709 minimal_stack=not secure, 710 categories=[SWEEP], 711 channels=channels, 712 outstanding=outstanding, 713 warmup_seconds=CXX_WARMUP_SECONDS, 714 ) 715 716 def __str__(self): 717 return "c++" 718 719 720class CSharpLanguage(Language): 721 """The legacy Grpc.Core implementation from grpc/grpc.""" 722 723 def worker_cmdline(self): 724 return ["tools/run_tests/performance/run_worker_csharp.sh"] 725 726 def worker_port_offset(self): 727 return 100 728 729 def scenarios(self): 730 yield _ping_pong_scenario( 731 "csharp_generic_async_streaming_ping_pong", 732 rpc_type="STREAMING", 733 client_type="ASYNC_CLIENT", 734 server_type="ASYNC_GENERIC_SERVER", 735 use_generic_payload=True, 736 categories=[SMOKETEST, SCALABLE], 737 ) 738 739 yield _ping_pong_scenario( 740 "csharp_generic_async_streaming_ping_pong_insecure_1MB", 741 rpc_type="STREAMING", 742 client_type="ASYNC_CLIENT", 743 server_type="ASYNC_GENERIC_SERVER", 744 req_size=1024 * 1024, 745 resp_size=1024 * 1024, 746 use_generic_payload=True, 747 secure=False, 748 categories=[SMOKETEST, SCALABLE], 749 ) 750 751 yield _ping_pong_scenario( 752 "csharp_generic_async_streaming_qps_unconstrained_insecure", 753 rpc_type="STREAMING", 754 client_type="ASYNC_CLIENT", 755 server_type="ASYNC_GENERIC_SERVER", 756 unconstrained_client="async", 757 use_generic_payload=True, 758 secure=False, 759 categories=[SMOKETEST, SCALABLE], 760 ) 761 762 yield _ping_pong_scenario( 763 "csharp_protobuf_async_streaming_ping_pong", 764 rpc_type="STREAMING", 765 client_type="ASYNC_CLIENT", 766 server_type="ASYNC_SERVER", 767 ) 768 769 yield _ping_pong_scenario( 770 "csharp_protobuf_async_unary_ping_pong", 771 rpc_type="UNARY", 772 client_type="ASYNC_CLIENT", 773 server_type="ASYNC_SERVER", 774 categories=[SMOKETEST, SCALABLE], 775 ) 776 777 yield _ping_pong_scenario( 778 "csharp_protobuf_sync_to_async_unary_ping_pong", 779 rpc_type="UNARY", 780 client_type="SYNC_CLIENT", 781 server_type="ASYNC_SERVER", 782 ) 783 784 yield _ping_pong_scenario( 785 "csharp_protobuf_async_unary_qps_unconstrained", 786 rpc_type="UNARY", 787 client_type="ASYNC_CLIENT", 788 server_type="ASYNC_SERVER", 789 unconstrained_client="async", 790 categories=[SMOKETEST, SCALABLE], 791 ) 792 793 yield _ping_pong_scenario( 794 "csharp_protobuf_async_streaming_qps_unconstrained", 795 rpc_type="STREAMING", 796 client_type="ASYNC_CLIENT", 797 server_type="ASYNC_SERVER", 798 unconstrained_client="async", 799 categories=[SCALABLE], 800 ) 801 802 yield _ping_pong_scenario( 803 "csharp_to_cpp_protobuf_sync_unary_ping_pong", 804 rpc_type="UNARY", 805 client_type="SYNC_CLIENT", 806 server_type="SYNC_SERVER", 807 server_language="c++", 808 async_server_threads=1, 809 categories=[SMOKETEST, SCALABLE], 810 ) 811 812 yield _ping_pong_scenario( 813 "csharp_to_cpp_protobuf_async_streaming_ping_pong", 814 rpc_type="STREAMING", 815 client_type="ASYNC_CLIENT", 816 server_type="ASYNC_SERVER", 817 server_language="c++", 818 async_server_threads=1, 819 ) 820 821 yield _ping_pong_scenario( 822 "csharp_to_cpp_protobuf_async_unary_qps_unconstrained", 823 rpc_type="UNARY", 824 client_type="ASYNC_CLIENT", 825 server_type="ASYNC_SERVER", 826 unconstrained_client="async", 827 server_language="c++", 828 categories=[SCALABLE], 829 ) 830 831 yield _ping_pong_scenario( 832 "csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained", 833 rpc_type="UNARY", 834 client_type="SYNC_CLIENT", 835 server_type="ASYNC_SERVER", 836 unconstrained_client="sync", 837 server_language="c++", 838 categories=[SCALABLE], 839 ) 840 841 yield _ping_pong_scenario( 842 "cpp_to_csharp_protobuf_async_unary_qps_unconstrained", 843 rpc_type="UNARY", 844 client_type="ASYNC_CLIENT", 845 server_type="ASYNC_SERVER", 846 unconstrained_client="async", 847 client_language="c++", 848 categories=[SCALABLE], 849 ) 850 851 yield _ping_pong_scenario( 852 "csharp_protobuf_async_unary_ping_pong_1MB", 853 rpc_type="UNARY", 854 client_type="ASYNC_CLIENT", 855 server_type="ASYNC_SERVER", 856 req_size=1024 * 1024, 857 resp_size=1024 * 1024, 858 categories=[SMOKETEST, SCALABLE], 859 ) 860 861 def __str__(self): 862 return "csharp" 863 864 865class DotnetLanguage(Language): 866 """The pure C# implementation from grpc/grpc-dotnet.""" 867 868 def worker_cmdline(self): 869 # grpc-dotnet worker is only supported by the new GKE based OSS benchmark 870 # framework, and the worker_cmdline() is only used by run_performance_tests.py 871 return ["grpc_dotnet_not_supported_by_legacy_performance_runner.sh"] 872 873 def worker_port_offset(self): 874 return 1100 875 876 def scenarios(self): 877 yield _ping_pong_scenario( 878 "dotnet_generic_async_streaming_ping_pong", 879 rpc_type="STREAMING", 880 client_type="ASYNC_CLIENT", 881 server_type="ASYNC_GENERIC_SERVER", 882 use_generic_payload=True, 883 categories=[SMOKETEST, SCALABLE], 884 ) 885 886 yield _ping_pong_scenario( 887 "dotnet_generic_async_streaming_ping_pong_insecure_1MB", 888 rpc_type="STREAMING", 889 client_type="ASYNC_CLIENT", 890 server_type="ASYNC_GENERIC_SERVER", 891 req_size=1024 * 1024, 892 resp_size=1024 * 1024, 893 use_generic_payload=True, 894 secure=False, 895 categories=[SMOKETEST, SCALABLE], 896 ) 897 898 yield _ping_pong_scenario( 899 "dotnet_generic_async_streaming_qps_unconstrained_insecure", 900 rpc_type="STREAMING", 901 client_type="ASYNC_CLIENT", 902 server_type="ASYNC_GENERIC_SERVER", 903 unconstrained_client="async", 904 use_generic_payload=True, 905 secure=False, 906 categories=[SMOKETEST, SCALABLE], 907 ) 908 909 yield _ping_pong_scenario( 910 "dotnet_protobuf_async_streaming_ping_pong", 911 rpc_type="STREAMING", 912 client_type="ASYNC_CLIENT", 913 server_type="ASYNC_SERVER", 914 ) 915 916 yield _ping_pong_scenario( 917 "dotnet_protobuf_async_unary_ping_pong", 918 rpc_type="UNARY", 919 client_type="ASYNC_CLIENT", 920 server_type="ASYNC_SERVER", 921 categories=[SMOKETEST, SCALABLE], 922 ) 923 924 yield _ping_pong_scenario( 925 "dotnet_protobuf_sync_to_async_unary_ping_pong", 926 rpc_type="UNARY", 927 client_type="SYNC_CLIENT", 928 server_type="ASYNC_SERVER", 929 ) 930 931 yield _ping_pong_scenario( 932 "dotnet_protobuf_async_unary_qps_unconstrained", 933 rpc_type="UNARY", 934 client_type="ASYNC_CLIENT", 935 server_type="ASYNC_SERVER", 936 unconstrained_client="async", 937 categories=[SMOKETEST, SCALABLE], 938 ) 939 940 yield _ping_pong_scenario( 941 "dotnet_protobuf_async_streaming_qps_unconstrained", 942 rpc_type="STREAMING", 943 client_type="ASYNC_CLIENT", 944 server_type="ASYNC_SERVER", 945 unconstrained_client="async", 946 categories=[SCALABLE], 947 ) 948 949 yield _ping_pong_scenario( 950 "dotnet_to_cpp_protobuf_sync_unary_ping_pong", 951 rpc_type="UNARY", 952 client_type="SYNC_CLIENT", 953 server_type="SYNC_SERVER", 954 server_language="c++", 955 async_server_threads=1, 956 categories=[SMOKETEST, SCALABLE], 957 ) 958 959 yield _ping_pong_scenario( 960 "dotnet_to_cpp_protobuf_async_streaming_ping_pong", 961 rpc_type="STREAMING", 962 client_type="ASYNC_CLIENT", 963 server_type="ASYNC_SERVER", 964 server_language="c++", 965 async_server_threads=1, 966 ) 967 968 yield _ping_pong_scenario( 969 "dotnet_to_cpp_protobuf_async_unary_qps_unconstrained", 970 rpc_type="UNARY", 971 client_type="ASYNC_CLIENT", 972 server_type="ASYNC_SERVER", 973 unconstrained_client="async", 974 server_language="c++", 975 categories=[SCALABLE], 976 ) 977 978 yield _ping_pong_scenario( 979 "dotnet_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained", 980 rpc_type="UNARY", 981 client_type="SYNC_CLIENT", 982 server_type="ASYNC_SERVER", 983 unconstrained_client="sync", 984 server_language="c++", 985 categories=[SCALABLE], 986 ) 987 988 yield _ping_pong_scenario( 989 "cpp_to_dotnet_protobuf_async_unary_qps_unconstrained", 990 rpc_type="UNARY", 991 client_type="ASYNC_CLIENT", 992 server_type="ASYNC_SERVER", 993 unconstrained_client="async", 994 client_language="c++", 995 categories=[SCALABLE], 996 ) 997 998 yield _ping_pong_scenario( 999 "dotnet_protobuf_async_unary_ping_pong_1MB", 1000 rpc_type="UNARY", 1001 client_type="ASYNC_CLIENT", 1002 server_type="ASYNC_SERVER", 1003 req_size=1024 * 1024, 1004 resp_size=1024 * 1024, 1005 categories=[SMOKETEST, SCALABLE], 1006 ) 1007 1008 def __str__(self): 1009 return "dotnet" 1010 1011 1012class PythonLanguage(Language): 1013 def worker_cmdline(self): 1014 return ["tools/run_tests/performance/run_worker_python.sh"] 1015 1016 def worker_port_offset(self): 1017 return 500 1018 1019 def scenarios(self): 1020 yield _ping_pong_scenario( 1021 "python_protobuf_async_unary_5000rpcs_1KB_psm", 1022 rpc_type="UNARY", 1023 client_type="ASYNC_CLIENT", 1024 server_type="ASYNC_SERVER", 1025 req_size=1024, 1026 resp_size=1024, 1027 outstanding=5000, 1028 channels=1, 1029 num_clients=1, 1030 secure=False, 1031 async_server_threads=1, 1032 categories=[PSM], 1033 ) 1034 1035 yield _ping_pong_scenario( 1036 "python_generic_sync_streaming_ping_pong", 1037 rpc_type="STREAMING", 1038 client_type="SYNC_CLIENT", 1039 server_type="ASYNC_GENERIC_SERVER", 1040 use_generic_payload=True, 1041 categories=[SMOKETEST, SCALABLE], 1042 ) 1043 1044 yield _ping_pong_scenario( 1045 "python_protobuf_sync_streaming_ping_pong", 1046 rpc_type="STREAMING", 1047 client_type="SYNC_CLIENT", 1048 server_type="ASYNC_SERVER", 1049 ) 1050 1051 yield _ping_pong_scenario( 1052 "python_protobuf_async_unary_ping_pong", 1053 rpc_type="UNARY", 1054 client_type="ASYNC_CLIENT", 1055 server_type="ASYNC_SERVER", 1056 ) 1057 1058 yield _ping_pong_scenario( 1059 "python_protobuf_sync_unary_ping_pong", 1060 rpc_type="UNARY", 1061 client_type="SYNC_CLIENT", 1062 server_type="ASYNC_SERVER", 1063 categories=[SMOKETEST, SCALABLE], 1064 ) 1065 1066 yield _ping_pong_scenario( 1067 "python_protobuf_sync_unary_qps_unconstrained", 1068 rpc_type="UNARY", 1069 client_type="SYNC_CLIENT", 1070 server_type="ASYNC_SERVER", 1071 unconstrained_client="sync", 1072 ) 1073 1074 yield _ping_pong_scenario( 1075 "python_protobuf_sync_streaming_qps_unconstrained", 1076 rpc_type="STREAMING", 1077 client_type="SYNC_CLIENT", 1078 server_type="ASYNC_SERVER", 1079 unconstrained_client="sync", 1080 ) 1081 1082 yield _ping_pong_scenario( 1083 "python_to_cpp_protobuf_sync_unary_ping_pong", 1084 rpc_type="UNARY", 1085 client_type="SYNC_CLIENT", 1086 server_type="ASYNC_SERVER", 1087 server_language="c++", 1088 async_server_threads=0, 1089 categories=[SMOKETEST, SCALABLE], 1090 ) 1091 1092 yield _ping_pong_scenario( 1093 "python_to_cpp_protobuf_sync_streaming_ping_pong", 1094 rpc_type="STREAMING", 1095 client_type="SYNC_CLIENT", 1096 server_type="ASYNC_SERVER", 1097 server_language="c++", 1098 async_server_threads=1, 1099 ) 1100 1101 yield _ping_pong_scenario( 1102 "python_protobuf_sync_unary_ping_pong_1MB", 1103 rpc_type="UNARY", 1104 client_type="SYNC_CLIENT", 1105 server_type="ASYNC_SERVER", 1106 req_size=1024 * 1024, 1107 resp_size=1024 * 1024, 1108 categories=[SMOKETEST, SCALABLE], 1109 ) 1110 1111 def __str__(self): 1112 return "python" 1113 1114 1115class PythonAsyncIOLanguage(Language): 1116 def worker_cmdline(self): 1117 return ["tools/run_tests/performance/run_worker_python_asyncio.sh"] 1118 1119 def worker_port_offset(self): 1120 return 1200 1121 1122 def scenarios(self): 1123 yield _ping_pong_scenario( 1124 "python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm", 1125 rpc_type="UNARY", 1126 client_type="ASYNC_CLIENT", 1127 server_type="ASYNC_SERVER", 1128 req_size=1024, 1129 resp_size=1024, 1130 outstanding=5000, 1131 channels=1, 1132 num_clients=1, 1133 secure=False, 1134 async_server_threads=1, 1135 categories=[PSM], 1136 ) 1137 1138 for outstanding in [64, 128, 256, 512]: 1139 for channels in [1, 4]: 1140 yield _ping_pong_scenario( 1141 "python_asyncio_protobuf_async_unary_ping_pong_%dx%d_max" 1142 % ( 1143 outstanding, 1144 channels, 1145 ), 1146 rpc_type="UNARY", 1147 client_type="ASYNC_CLIENT", 1148 server_type="ASYNC_SERVER", 1149 outstanding=outstanding * channels, 1150 channels=channels, 1151 client_processes=0, 1152 server_processes=0, 1153 unconstrained_client="async", 1154 categories=[SCALABLE], 1155 ) 1156 1157 yield _ping_pong_scenario( 1158 "python_asyncio_protobuf_async_unary_ping_pong_%d_1thread" 1159 % outstanding, 1160 rpc_type="UNARY", 1161 client_type="ASYNC_CLIENT", 1162 server_type="ASYNC_SERVER", 1163 outstanding=outstanding, 1164 channels=1, 1165 client_processes=1, 1166 server_processes=1, 1167 unconstrained_client="async", 1168 categories=[SCALABLE], 1169 ) 1170 1171 yield _ping_pong_scenario( 1172 "python_asyncio_generic_async_streaming_ping_pong", 1173 rpc_type="STREAMING", 1174 client_type="ASYNC_CLIENT", 1175 server_type="ASYNC_GENERIC_SERVER", 1176 channels=1, 1177 client_processes=1, 1178 server_processes=1, 1179 use_generic_payload=True, 1180 categories=[SMOKETEST, SCALABLE], 1181 ) 1182 1183 yield _ping_pong_scenario( 1184 "python_asyncio_protobuf_async_streaming_ping_pong", 1185 rpc_type="STREAMING", 1186 client_type="ASYNC_CLIENT", 1187 server_type="ASYNC_SERVER", 1188 channels=1, 1189 client_processes=1, 1190 server_processes=1, 1191 categories=[SMOKETEST, SCALABLE], 1192 ) 1193 1194 yield _ping_pong_scenario( 1195 "python_asyncio_protobuf_async_unary_ping_pong", 1196 rpc_type="UNARY", 1197 client_type="ASYNC_CLIENT", 1198 server_type="ASYNC_SERVER", 1199 client_processes=1, 1200 server_processes=1, 1201 categories=[SMOKETEST, SCALABLE], 1202 ) 1203 1204 yield _ping_pong_scenario( 1205 "python_asyncio_protobuf_async_unary_ping_pong", 1206 rpc_type="UNARY", 1207 client_type="ASYNC_CLIENT", 1208 server_type="ASYNC_SERVER", 1209 channels=1, 1210 client_processes=1, 1211 server_processes=1, 1212 categories=[SMOKETEST, SCALABLE], 1213 ) 1214 1215 yield _ping_pong_scenario( 1216 "python_asyncio_protobuf_async_unary_qps_unconstrained", 1217 rpc_type="UNARY", 1218 client_type="ASYNC_CLIENT", 1219 server_type="ASYNC_SERVER", 1220 channels=1, 1221 unconstrained_client="async", 1222 ) 1223 1224 yield _ping_pong_scenario( 1225 "python_asyncio_protobuf_async_streaming_qps_unconstrained", 1226 rpc_type="STREAMING", 1227 client_type="ASYNC_CLIENT", 1228 server_type="ASYNC_SERVER", 1229 channels=1, 1230 unconstrained_client="async", 1231 ) 1232 1233 yield _ping_pong_scenario( 1234 "python_asyncio_to_cpp_protobuf_async_unary_ping_pong_1thread", 1235 rpc_type="UNARY", 1236 client_type="ASYNC_CLIENT", 1237 server_type="ASYNC_SERVER", 1238 server_language="c++", 1239 channels=1, 1240 client_processes=1, 1241 unconstrained_client="async", 1242 categories=[SMOKETEST, SCALABLE], 1243 ) 1244 1245 yield _ping_pong_scenario( 1246 "python_asyncio_to_cpp_protobuf_async_unary_ping_pong_max", 1247 rpc_type="UNARY", 1248 client_type="ASYNC_CLIENT", 1249 server_type="ASYNC_SERVER", 1250 unconstrained_client="async", 1251 channels=1, 1252 client_processes=0, 1253 server_language="c++", 1254 categories=[SMOKETEST, SCALABLE], 1255 ) 1256 1257 yield _ping_pong_scenario( 1258 "python_asyncio_to_cpp_protobuf_sync_streaming_ping_pong_1thread", 1259 rpc_type="STREAMING", 1260 client_type="ASYNC_CLIENT", 1261 server_type="ASYNC_SERVER", 1262 channels=1, 1263 client_processes=1, 1264 server_processes=1, 1265 unconstrained_client="async", 1266 server_language="c++", 1267 ) 1268 1269 yield _ping_pong_scenario( 1270 "python_asyncio_protobuf_async_unary_ping_pong_1MB", 1271 rpc_type="UNARY", 1272 client_type="ASYNC_CLIENT", 1273 server_type="ASYNC_SERVER", 1274 req_size=1024 * 1024, 1275 resp_size=1024 * 1024, 1276 channels=1, 1277 client_processes=1, 1278 server_processes=1, 1279 categories=[SMOKETEST, SCALABLE], 1280 ) 1281 1282 def __str__(self): 1283 return "python_asyncio" 1284 1285 1286class RubyLanguage(Language): 1287 def worker_cmdline(self): 1288 return ["tools/run_tests/performance/run_worker_ruby.sh"] 1289 1290 def worker_port_offset(self): 1291 return 300 1292 1293 def scenarios(self): 1294 yield _ping_pong_scenario( 1295 "ruby_protobuf_sync_streaming_ping_pong", 1296 rpc_type="STREAMING", 1297 client_type="SYNC_CLIENT", 1298 server_type="SYNC_SERVER", 1299 categories=[SMOKETEST, SCALABLE], 1300 ) 1301 1302 yield _ping_pong_scenario( 1303 "ruby_protobuf_unary_ping_pong", 1304 rpc_type="UNARY", 1305 client_type="SYNC_CLIENT", 1306 server_type="SYNC_SERVER", 1307 categories=[SMOKETEST, SCALABLE], 1308 ) 1309 1310 yield _ping_pong_scenario( 1311 "ruby_protobuf_sync_unary_qps_unconstrained", 1312 rpc_type="UNARY", 1313 client_type="SYNC_CLIENT", 1314 server_type="SYNC_SERVER", 1315 unconstrained_client="sync", 1316 ) 1317 1318 yield _ping_pong_scenario( 1319 "ruby_protobuf_sync_streaming_qps_unconstrained", 1320 rpc_type="STREAMING", 1321 client_type="SYNC_CLIENT", 1322 server_type="SYNC_SERVER", 1323 unconstrained_client="sync", 1324 ) 1325 1326 yield _ping_pong_scenario( 1327 "ruby_to_cpp_protobuf_sync_unary_ping_pong", 1328 rpc_type="UNARY", 1329 client_type="SYNC_CLIENT", 1330 server_type="SYNC_SERVER", 1331 server_language="c++", 1332 async_server_threads=1, 1333 ) 1334 1335 yield _ping_pong_scenario( 1336 "ruby_to_cpp_protobuf_sync_streaming_ping_pong", 1337 rpc_type="STREAMING", 1338 client_type="SYNC_CLIENT", 1339 server_type="SYNC_SERVER", 1340 server_language="c++", 1341 async_server_threads=1, 1342 ) 1343 1344 yield _ping_pong_scenario( 1345 "ruby_protobuf_unary_ping_pong_1MB", 1346 rpc_type="UNARY", 1347 client_type="SYNC_CLIENT", 1348 server_type="SYNC_SERVER", 1349 req_size=1024 * 1024, 1350 resp_size=1024 * 1024, 1351 categories=[SMOKETEST, SCALABLE], 1352 ) 1353 1354 def __str__(self): 1355 return "ruby" 1356 1357 1358class Php7Language(Language): 1359 def __init__(self, php7_protobuf_c=False): 1360 super().__init__() 1361 self.php7_protobuf_c = php7_protobuf_c 1362 1363 def worker_cmdline(self): 1364 if self.php7_protobuf_c: 1365 return [ 1366 "tools/run_tests/performance/run_worker_php.sh", 1367 "--use_protobuf_c_extension", 1368 ] 1369 return ["tools/run_tests/performance/run_worker_php.sh"] 1370 1371 def worker_port_offset(self): 1372 if self.php7_protobuf_c: 1373 return 900 1374 return 800 1375 1376 def scenarios(self): 1377 php7_extension_mode = "php7_protobuf_php_extension" 1378 if self.php7_protobuf_c: 1379 php7_extension_mode = "php7_protobuf_c_extension" 1380 1381 yield _ping_pong_scenario( 1382 "%s_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm" 1383 % php7_extension_mode, 1384 rpc_type="UNARY", 1385 client_type="ASYNC_CLIENT", 1386 server_type="ASYNC_SERVER", 1387 server_language="c++", 1388 req_size=1024, 1389 resp_size=1024, 1390 outstanding=5000, 1391 channels=1, 1392 num_clients=1, 1393 secure=False, 1394 async_server_threads=1, 1395 categories=[PSM], 1396 ) 1397 1398 yield _ping_pong_scenario( 1399 "%s_to_cpp_protobuf_sync_unary_ping_pong" % php7_extension_mode, 1400 rpc_type="UNARY", 1401 client_type="SYNC_CLIENT", 1402 server_type="SYNC_SERVER", 1403 server_language="c++", 1404 async_server_threads=1, 1405 ) 1406 1407 yield _ping_pong_scenario( 1408 "%s_to_cpp_protobuf_sync_streaming_ping_pong" % php7_extension_mode, 1409 rpc_type="STREAMING", 1410 client_type="SYNC_CLIENT", 1411 server_type="SYNC_SERVER", 1412 server_language="c++", 1413 async_server_threads=1, 1414 ) 1415 1416 # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs 1417 # better than async_server_threads=0/CPU usage 490%. 1418 yield _ping_pong_scenario( 1419 "%s_to_cpp_protobuf_sync_unary_qps_unconstrained" 1420 % php7_extension_mode, 1421 rpc_type="UNARY", 1422 client_type="SYNC_CLIENT", 1423 server_type="ASYNC_SERVER", 1424 server_language="c++", 1425 outstanding=1, 1426 async_server_threads=1, 1427 unconstrained_client="sync", 1428 ) 1429 1430 yield _ping_pong_scenario( 1431 "%s_to_cpp_protobuf_sync_streaming_qps_unconstrained" 1432 % php7_extension_mode, 1433 rpc_type="STREAMING", 1434 client_type="SYNC_CLIENT", 1435 server_type="ASYNC_SERVER", 1436 server_language="c++", 1437 outstanding=1, 1438 async_server_threads=1, 1439 unconstrained_client="sync", 1440 ) 1441 1442 def __str__(self): 1443 if self.php7_protobuf_c: 1444 return "php7_protobuf_c" 1445 return "php7" 1446 1447 1448class JavaLanguage(Language): 1449 def worker_cmdline(self): 1450 return ["tools/run_tests/performance/run_worker_java.sh"] 1451 1452 def worker_port_offset(self): 1453 return 400 1454 1455 def scenarios(self): 1456 yield _ping_pong_scenario( 1457 "java_protobuf_async_unary_5000rpcs_1KB_psm", 1458 rpc_type="UNARY", 1459 client_type="ASYNC_CLIENT", 1460 server_type="ASYNC_SERVER", 1461 req_size=1024, 1462 resp_size=1024, 1463 outstanding=5000, 1464 channels=1, 1465 num_clients=1, 1466 secure=False, 1467 async_server_threads=1, 1468 warmup_seconds=JAVA_WARMUP_SECONDS, 1469 categories=[PSM], 1470 ) 1471 1472 for secure in [True, False]: 1473 secstr = "secure" if secure else "insecure" 1474 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1475 1476 yield _ping_pong_scenario( 1477 "java_generic_async_streaming_ping_pong_%s" % secstr, 1478 rpc_type="STREAMING", 1479 client_type="ASYNC_CLIENT", 1480 server_type="ASYNC_GENERIC_SERVER", 1481 use_generic_payload=True, 1482 async_server_threads=1, 1483 secure=secure, 1484 warmup_seconds=JAVA_WARMUP_SECONDS, 1485 categories=smoketest_categories, 1486 ) 1487 1488 yield _ping_pong_scenario( 1489 "java_protobuf_async_streaming_ping_pong_%s" % secstr, 1490 rpc_type="STREAMING", 1491 client_type="ASYNC_CLIENT", 1492 server_type="ASYNC_SERVER", 1493 async_server_threads=1, 1494 secure=secure, 1495 warmup_seconds=JAVA_WARMUP_SECONDS, 1496 ) 1497 1498 yield _ping_pong_scenario( 1499 "java_protobuf_async_unary_ping_pong_%s" % secstr, 1500 rpc_type="UNARY", 1501 client_type="ASYNC_CLIENT", 1502 server_type="ASYNC_SERVER", 1503 async_server_threads=1, 1504 secure=secure, 1505 warmup_seconds=JAVA_WARMUP_SECONDS, 1506 categories=smoketest_categories, 1507 ) 1508 1509 yield _ping_pong_scenario( 1510 "java_protobuf_unary_ping_pong_%s" % secstr, 1511 rpc_type="UNARY", 1512 client_type="SYNC_CLIENT", 1513 server_type="SYNC_SERVER", 1514 async_server_threads=1, 1515 secure=secure, 1516 warmup_seconds=JAVA_WARMUP_SECONDS, 1517 ) 1518 1519 yield _ping_pong_scenario( 1520 "java_protobuf_async_unary_qps_unconstrained_%s" % secstr, 1521 rpc_type="UNARY", 1522 client_type="ASYNC_CLIENT", 1523 server_type="ASYNC_SERVER", 1524 unconstrained_client="async", 1525 secure=secure, 1526 warmup_seconds=JAVA_WARMUP_SECONDS, 1527 categories=smoketest_categories + [SCALABLE], 1528 ) 1529 1530 yield _ping_pong_scenario( 1531 "java_protobuf_async_streaming_qps_unconstrained_%s" % secstr, 1532 rpc_type="STREAMING", 1533 client_type="ASYNC_CLIENT", 1534 server_type="ASYNC_SERVER", 1535 unconstrained_client="async", 1536 secure=secure, 1537 warmup_seconds=JAVA_WARMUP_SECONDS, 1538 categories=[SCALABLE], 1539 ) 1540 1541 yield _ping_pong_scenario( 1542 "java_generic_async_streaming_qps_unconstrained_%s" % secstr, 1543 rpc_type="STREAMING", 1544 client_type="ASYNC_CLIENT", 1545 server_type="ASYNC_GENERIC_SERVER", 1546 unconstrained_client="async", 1547 use_generic_payload=True, 1548 secure=secure, 1549 warmup_seconds=JAVA_WARMUP_SECONDS, 1550 categories=[SCALABLE], 1551 ) 1552 1553 yield _ping_pong_scenario( 1554 "java_generic_async_streaming_qps_one_server_core_%s" % secstr, 1555 rpc_type="STREAMING", 1556 client_type="ASYNC_CLIENT", 1557 server_type="ASYNC_GENERIC_SERVER", 1558 unconstrained_client="async-limited", 1559 use_generic_payload=True, 1560 async_server_threads=1, 1561 secure=secure, 1562 warmup_seconds=JAVA_WARMUP_SECONDS, 1563 ) 1564 1565 # TODO(jtattermusch): add scenarios java vs C++ 1566 1567 def __str__(self): 1568 return "java" 1569 1570 1571class GoLanguage(Language): 1572 def worker_cmdline(self): 1573 return ["tools/run_tests/performance/run_worker_go.sh"] 1574 1575 def worker_port_offset(self): 1576 return 600 1577 1578 def scenarios(self): 1579 yield _ping_pong_scenario( 1580 "go_protobuf_async_unary_5000rpcs_1KB_psm", 1581 rpc_type="UNARY", 1582 client_type="ASYNC_CLIENT", 1583 server_type="ASYNC_SERVER", 1584 req_size=1024, 1585 resp_size=1024, 1586 outstanding=5000, 1587 channels=1, 1588 num_clients=1, 1589 secure=False, 1590 async_server_threads=1, 1591 categories=[PSM], 1592 ) 1593 1594 for secure in [True, False]: 1595 secstr = "secure" if secure else "insecure" 1596 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1597 1598 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1599 # but that's mostly because of lack of better name of the enum value. 1600 yield _ping_pong_scenario( 1601 "go_generic_sync_streaming_ping_pong_%s" % secstr, 1602 rpc_type="STREAMING", 1603 client_type="SYNC_CLIENT", 1604 server_type="ASYNC_GENERIC_SERVER", 1605 use_generic_payload=True, 1606 async_server_threads=1, 1607 secure=secure, 1608 categories=smoketest_categories, 1609 ) 1610 1611 yield _ping_pong_scenario( 1612 "go_protobuf_sync_streaming_ping_pong_%s" % secstr, 1613 rpc_type="STREAMING", 1614 client_type="SYNC_CLIENT", 1615 server_type="SYNC_SERVER", 1616 async_server_threads=1, 1617 secure=secure, 1618 ) 1619 1620 yield _ping_pong_scenario( 1621 "go_protobuf_sync_unary_ping_pong_%s" % secstr, 1622 rpc_type="UNARY", 1623 client_type="SYNC_CLIENT", 1624 server_type="SYNC_SERVER", 1625 async_server_threads=1, 1626 secure=secure, 1627 categories=smoketest_categories, 1628 ) 1629 1630 # unconstrained_client='async' is intended (client uses goroutines) 1631 yield _ping_pong_scenario( 1632 "go_protobuf_sync_unary_qps_unconstrained_%s" % secstr, 1633 rpc_type="UNARY", 1634 client_type="SYNC_CLIENT", 1635 server_type="SYNC_SERVER", 1636 unconstrained_client="async", 1637 secure=secure, 1638 categories=smoketest_categories + [SCALABLE], 1639 ) 1640 1641 # unconstrained_client='async' is intended (client uses goroutines) 1642 yield _ping_pong_scenario( 1643 "go_protobuf_sync_streaming_qps_unconstrained_%s" % secstr, 1644 rpc_type="STREAMING", 1645 client_type="SYNC_CLIENT", 1646 server_type="SYNC_SERVER", 1647 unconstrained_client="async", 1648 secure=secure, 1649 categories=[SCALABLE], 1650 ) 1651 1652 # unconstrained_client='async' is intended (client uses goroutines) 1653 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1654 # but that's mostly because of lack of better name of the enum value. 1655 yield _ping_pong_scenario( 1656 "go_generic_sync_streaming_qps_unconstrained_%s" % secstr, 1657 rpc_type="STREAMING", 1658 client_type="SYNC_CLIENT", 1659 server_type="ASYNC_GENERIC_SERVER", 1660 unconstrained_client="async", 1661 use_generic_payload=True, 1662 secure=secure, 1663 categories=[SCALABLE], 1664 ) 1665 1666 # TODO(jtattermusch): add scenarios go vs C++ 1667 1668 def __str__(self): 1669 return "go" 1670 1671 1672class NodeLanguage(Language): 1673 def worker_cmdline(self): 1674 fixture = "js_js" 1675 return [ 1676 "tools/run_tests/performance/run_worker_node.sh", 1677 fixture, 1678 "--benchmark_impl=grpc", 1679 ] 1680 1681 def worker_port_offset(self): 1682 return 1100 1683 1684 def scenarios(self): 1685 yield _ping_pong_scenario( 1686 "node_to_node_protobuf_async_unary_5000rpcs_1KB_psm", 1687 rpc_type="UNARY", 1688 client_type="ASYNC_CLIENT", 1689 server_type="ASYNC_SERVER", 1690 req_size=1024, 1691 resp_size=1024, 1692 outstanding=5000, 1693 channels=1, 1694 num_clients=1, 1695 secure=False, 1696 async_server_threads=1, 1697 categories=[PSM], 1698 ) 1699 1700 for secure in [True, False]: 1701 secstr = "secure" if secure else "insecure" 1702 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1703 1704 yield _ping_pong_scenario( 1705 "node_to_node_generic_async_streaming_ping_pong_%s" % secstr, 1706 rpc_type="STREAMING", 1707 client_type="ASYNC_CLIENT", 1708 server_type="ASYNC_GENERIC_SERVER", 1709 use_generic_payload=True, 1710 async_server_threads=1, 1711 secure=secure, 1712 categories=smoketest_categories, 1713 ) 1714 1715 yield _ping_pong_scenario( 1716 "node_to_node_protobuf_async_streaming_ping_pong_%s" % secstr, 1717 rpc_type="STREAMING", 1718 client_type="ASYNC_CLIENT", 1719 server_type="ASYNC_SERVER", 1720 async_server_threads=1, 1721 secure=secure, 1722 ) 1723 1724 yield _ping_pong_scenario( 1725 "node_to_node_protobuf_async_unary_ping_pong_%s" % secstr, 1726 rpc_type="UNARY", 1727 client_type="ASYNC_CLIENT", 1728 server_type="ASYNC_SERVER", 1729 async_server_threads=1, 1730 secure=secure, 1731 categories=smoketest_categories, 1732 ) 1733 1734 yield _ping_pong_scenario( 1735 "node_to_node_protobuf_async_unary_qps_unconstrained_%s" 1736 % secstr, 1737 rpc_type="UNARY", 1738 client_type="ASYNC_CLIENT", 1739 server_type="ASYNC_SERVER", 1740 unconstrained_client="async", 1741 secure=secure, 1742 categories=smoketest_categories + [SCALABLE], 1743 ) 1744 1745 yield _ping_pong_scenario( 1746 "node_to_node_protobuf_async_streaming_qps_unconstrained_%s" 1747 % secstr, 1748 rpc_type="STREAMING", 1749 client_type="ASYNC_CLIENT", 1750 server_type="ASYNC_SERVER", 1751 unconstrained_client="async", 1752 secure=secure, 1753 categories=[SCALABLE], 1754 ) 1755 1756 yield _ping_pong_scenario( 1757 "node_to_node_generic_async_streaming_qps_unconstrained_%s" 1758 % secstr, 1759 rpc_type="STREAMING", 1760 client_type="ASYNC_CLIENT", 1761 server_type="ASYNC_GENERIC_SERVER", 1762 unconstrained_client="async", 1763 use_generic_payload=True, 1764 secure=secure, 1765 categories=[SCALABLE], 1766 ) 1767 1768 # TODO(murgatroid99): add scenarios node vs C++ 1769 1770 def __str__(self): 1771 return "node" 1772 1773 1774LANGUAGES = { 1775 "c++": CXXLanguage(), 1776 "csharp": CSharpLanguage(), 1777 "dotnet": DotnetLanguage(), 1778 "ruby": RubyLanguage(), 1779 "php7": Php7Language(), 1780 "php7_protobuf_c": Php7Language(php7_protobuf_c=True), 1781 "java": JavaLanguage(), 1782 "python": PythonLanguage(), 1783 "python_asyncio": PythonAsyncIOLanguage(), 1784 "go": GoLanguage(), 1785 "node": NodeLanguage(), # 'node' means 'node_purejs'. 1786} 1787