1# Copyright 2016 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15# performance scenario configuration for various languages 16 17import math 18 19WARMUP_SECONDS = 5 20JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in. 21BENCHMARK_SECONDS = 30 22 23SMOKETEST = 'smoketest' 24SCALABLE = 'scalable' 25INPROC = 'inproc' 26SWEEP = 'sweep' 27PSM = 'psm' 28DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST) 29 30SECURE_SECARGS = { 31 'use_test_ca': True, 32 'server_host_override': 'foo.test.google.fr' 33} 34 35HISTOGRAM_PARAMS = { 36 'resolution': 0.01, 37 'max_possible': 60e9, 38} 39 40# target number of RPCs outstanding on across all client channels in 41# non-ping-pong tests (since we can only specify per-channel numbers, the 42# actual target will be slightly higher) 43OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000} 44 45# wide is the number of client channels in multi-channel tests (1 otherwise) 46WIDE = 64 47 48 49def _get_secargs(is_secure): 50 if is_secure: 51 return SECURE_SECARGS 52 else: 53 return None 54 55 56def remove_nonproto_fields(scenario): 57 """Removes special-purpose fields that don't belong in the protobuf. 58 59 This function removes additional information about the scenario that is not 60 included in the ScenarioConfig protobuf message. 61 """ 62 scenario.pop('CATEGORIES', None) 63 scenario.pop('CLIENT_LANGUAGE', None) 64 scenario.pop('SERVER_LANGUAGE', None) 65 scenario.pop('EXCLUDED_POLL_ENGINES', None) 66 return scenario 67 68 69def geometric_progression(start, stop, step): 70 n = start 71 while n < stop: 72 yield int(round(n)) 73 n *= step 74 75 76def _payload_type(use_generic_payload, req_size, resp_size): 77 r = {} 78 sizes = { 79 'req_size': req_size, 80 'resp_size': resp_size, 81 } 82 if use_generic_payload: 83 r['bytebuf_params'] = sizes 84 else: 85 r['simple_params'] = sizes 86 return r 87 88 89def _load_params(offered_load): 90 r = {} 91 if offered_load is None: 92 r['closed_loop'] = {} 93 else: 94 load = {} 95 load['offered_load'] = offered_load 96 r['poisson'] = load 97 return r 98 99 100def _add_channel_arg(config, key, value): 101 if 'channel_args' in config: 102 channel_args = config['channel_args'] 103 else: 104 channel_args = [] 105 config['channel_args'] = channel_args 106 arg = {'name': key} 107 if isinstance(value, int): 108 arg['int_value'] = value 109 else: 110 arg['str_value'] = value 111 channel_args.append(arg) 112 113 114def _ping_pong_scenario(name, 115 rpc_type, 116 client_type, 117 server_type, 118 secure=True, 119 use_generic_payload=False, 120 req_size=0, 121 resp_size=0, 122 unconstrained_client=None, 123 client_language=None, 124 server_language=None, 125 async_server_threads=0, 126 client_processes=0, 127 server_processes=0, 128 server_threads_per_cq=0, 129 client_threads_per_cq=0, 130 warmup_seconds=WARMUP_SECONDS, 131 categories=None, 132 channels=None, 133 outstanding=None, 134 num_clients=None, 135 resource_quota_size=None, 136 messages_per_stream=None, 137 excluded_poll_engines=None, 138 minimal_stack=False, 139 offered_load=None): 140 """Creates a basic ping pong scenario.""" 141 scenario = { 142 'name': name, 143 'num_servers': 1, 144 'num_clients': 1, 145 'client_config': { 146 'client_type': client_type, 147 'security_params': _get_secargs(secure), 148 'outstanding_rpcs_per_channel': 1, 149 'client_channels': 1, 150 'async_client_threads': 1, 151 'client_processes': client_processes, 152 'threads_per_cq': client_threads_per_cq, 153 'rpc_type': rpc_type, 154 'histogram_params': HISTOGRAM_PARAMS, 155 'channel_args': [], 156 }, 157 'server_config': { 158 'server_type': server_type, 159 'security_params': _get_secargs(secure), 160 'async_server_threads': async_server_threads, 161 'server_processes': server_processes, 162 'threads_per_cq': server_threads_per_cq, 163 'channel_args': [], 164 }, 165 'warmup_seconds': warmup_seconds, 166 'benchmark_seconds': BENCHMARK_SECONDS, 167 'CATEGORIES': list(DEFAULT_CATEGORIES), 168 'EXCLUDED_POLL_ENGINES': [], 169 } 170 if resource_quota_size: 171 scenario['server_config']['resource_quota_size'] = resource_quota_size 172 if use_generic_payload: 173 if server_type != 'ASYNC_GENERIC_SERVER': 174 raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') 175 scenario['server_config']['payload_config'] = _payload_type( 176 use_generic_payload, req_size, resp_size) 177 178 scenario['client_config']['payload_config'] = _payload_type( 179 use_generic_payload, req_size, resp_size) 180 181 # Optimization target of 'throughput' does not work well with epoll1 polling 182 # engine. Use the default value of 'blend' 183 optimization_target = 'throughput' 184 185 if unconstrained_client: 186 outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[ 187 unconstrained_client] 188 # clamp buffer usage to something reasonable (16 gig for now) 189 MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 190 if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: 191 outstanding_calls = max(1, 192 MAX_MEMORY_USE / max(req_size, resp_size)) 193 wide = channels if channels is not None else WIDE 194 deep = int(math.ceil(1.0 * outstanding_calls / wide)) 195 196 scenario[ 197 'num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available. 198 scenario['client_config']['outstanding_rpcs_per_channel'] = deep 199 scenario['client_config']['client_channels'] = wide 200 scenario['client_config']['async_client_threads'] = 0 201 if offered_load is not None: 202 optimization_target = 'latency' 203 else: 204 scenario['client_config']['outstanding_rpcs_per_channel'] = 1 205 scenario['client_config']['client_channels'] = 1 206 scenario['client_config']['async_client_threads'] = 1 207 optimization_target = 'latency' 208 209 scenario['client_config']['load_params'] = _load_params(offered_load) 210 211 optimization_channel_arg = { 212 'name': 'grpc.optimization_target', 213 'str_value': optimization_target 214 } 215 scenario['client_config']['channel_args'].append(optimization_channel_arg) 216 scenario['server_config']['channel_args'].append(optimization_channel_arg) 217 218 if minimal_stack: 219 _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1) 220 _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1) 221 222 if messages_per_stream: 223 scenario['client_config']['messages_per_stream'] = messages_per_stream 224 if client_language: 225 # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py 226 scenario['CLIENT_LANGUAGE'] = client_language 227 if server_language: 228 # the SERVER_LANGUAGE field is recognized by run_performance_tests.py 229 scenario['SERVER_LANGUAGE'] = server_language 230 if categories: 231 scenario['CATEGORIES'] = categories 232 if excluded_poll_engines: 233 # The polling engines for which this scenario is excluded 234 scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines 235 return scenario 236 237 238class Language(object): 239 240 @property 241 def safename(self): 242 return str(self) 243 244 245class CXXLanguage(Language): 246 247 @property 248 def safename(self): 249 return 'cxx' 250 251 def worker_cmdline(self): 252 return ['cmake/build/qps_worker'] 253 254 def worker_port_offset(self): 255 return 0 256 257 def scenarios(self): 258 yield _ping_pong_scenario('cpp_protobuf_async_unary_5000rpcs_1KB_psm', 259 rpc_type='UNARY', 260 client_type='ASYNC_CLIENT', 261 server_type='ASYNC_SERVER', 262 req_size=1024, 263 resp_size=1024, 264 outstanding=5000, 265 channels=1, 266 num_clients=1, 267 secure=False, 268 async_server_threads=1, 269 categories=[PSM]) 270 271 # TODO(ctiller): add 70% load latency test 272 yield _ping_pong_scenario( 273 'cpp_protobuf_async_unary_1channel_100rpcs_1MB', 274 rpc_type='UNARY', 275 client_type='ASYNC_CLIENT', 276 server_type='ASYNC_SERVER', 277 req_size=1024 * 1024, 278 resp_size=1024 * 1024, 279 unconstrained_client='async', 280 outstanding=100, 281 channels=1, 282 num_clients=1, 283 secure=False, 284 categories=[SWEEP]) 285 286 yield _ping_pong_scenario( 287 'cpp_protobuf_async_streaming_from_client_1channel_1MB', 288 rpc_type='STREAMING_FROM_CLIENT', 289 client_type='ASYNC_CLIENT', 290 server_type='ASYNC_SERVER', 291 req_size=1024 * 1024, 292 resp_size=1024 * 1024, 293 unconstrained_client='async', 294 outstanding=1, 295 channels=1, 296 num_clients=1, 297 secure=False, 298 categories=[SWEEP]) 299 300 # Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear 301 # (beyond excercising some params that other scenarios don't) 302 yield _ping_pong_scenario( 303 'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp', 304 rpc_type='UNARY', 305 client_type='ASYNC_CLIENT', 306 server_type='ASYNC_SERVER', 307 req_size=300, 308 resp_size=50, 309 unconstrained_client='async', 310 outstanding=30000, 311 channels=300, 312 offered_load=37500, 313 secure=False, 314 async_server_threads=16, 315 server_threads_per_cq=1, 316 categories=[SCALABLE]) 317 318 for secure in [True, False]: 319 secstr = 'secure' if secure else 'insecure' 320 smoketest_categories = ([SMOKETEST] if secure else []) 321 inproc_categories = ([INPROC] if not secure else []) 322 323 yield _ping_pong_scenario( 324 'cpp_generic_async_streaming_ping_pong_%s' % secstr, 325 rpc_type='STREAMING', 326 client_type='ASYNC_CLIENT', 327 server_type='ASYNC_GENERIC_SERVER', 328 use_generic_payload=True, 329 async_server_threads=1, 330 secure=secure, 331 categories=smoketest_categories + inproc_categories + 332 [SCALABLE]) 333 334 yield _ping_pong_scenario( 335 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, 336 rpc_type='STREAMING', 337 client_type='ASYNC_CLIENT', 338 server_type='ASYNC_GENERIC_SERVER', 339 unconstrained_client='async', 340 use_generic_payload=True, 341 secure=secure, 342 client_threads_per_cq=2, 343 server_threads_per_cq=2, 344 minimal_stack=not secure, 345 categories=smoketest_categories + inproc_categories + 346 [SCALABLE]) 347 348 for mps in geometric_progression(10, 20, 10): 349 yield _ping_pong_scenario( 350 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % 351 (mps, secstr), 352 rpc_type='STREAMING', 353 client_type='ASYNC_CLIENT', 354 server_type='ASYNC_GENERIC_SERVER', 355 unconstrained_client='async', 356 use_generic_payload=True, 357 secure=secure, 358 messages_per_stream=mps, 359 minimal_stack=not secure, 360 categories=smoketest_categories + inproc_categories + 361 [SCALABLE]) 362 363 for mps in geometric_progression(1, 200, math.sqrt(10)): 364 yield _ping_pong_scenario( 365 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % 366 (mps, secstr), 367 rpc_type='STREAMING', 368 client_type='ASYNC_CLIENT', 369 server_type='ASYNC_GENERIC_SERVER', 370 unconstrained_client='async', 371 use_generic_payload=True, 372 secure=secure, 373 messages_per_stream=mps, 374 minimal_stack=not secure, 375 categories=[SWEEP]) 376 377 yield _ping_pong_scenario( 378 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, 379 rpc_type='STREAMING', 380 req_size=1024 * 1024, 381 resp_size=1024 * 1024, 382 client_type='ASYNC_CLIENT', 383 server_type='ASYNC_GENERIC_SERVER', 384 unconstrained_client='async', 385 use_generic_payload=True, 386 secure=secure, 387 minimal_stack=not secure, 388 categories=inproc_categories + [SCALABLE], 389 channels=1, 390 outstanding=100) 391 392 yield _ping_pong_scenario( 393 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % 394 secstr, 395 rpc_type='STREAMING', 396 req_size=64 * 1024, 397 resp_size=64 * 1024, 398 client_type='ASYNC_CLIENT', 399 server_type='ASYNC_GENERIC_SERVER', 400 unconstrained_client='async', 401 use_generic_payload=True, 402 secure=secure, 403 minimal_stack=not secure, 404 categories=inproc_categories + [SCALABLE]) 405 406 yield _ping_pong_scenario( 407 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, 408 rpc_type='STREAMING', 409 client_type='ASYNC_CLIENT', 410 server_type='ASYNC_GENERIC_SERVER', 411 unconstrained_client='async-limited', 412 use_generic_payload=True, 413 secure=secure, 414 client_threads_per_cq=1000000, 415 server_threads_per_cq=1000000, 416 categories=[SWEEP]) 417 418 yield _ping_pong_scenario( 419 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % 420 secstr, 421 rpc_type='STREAMING', 422 client_type='ASYNC_CLIENT', 423 server_type='ASYNC_SERVER', 424 unconstrained_client='async-limited', 425 secure=secure, 426 client_threads_per_cq=1000000, 427 server_threads_per_cq=1000000, 428 categories=inproc_categories + [SCALABLE]) 429 430 yield _ping_pong_scenario( 431 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, 432 rpc_type='UNARY', 433 client_type='ASYNC_CLIENT', 434 server_type='ASYNC_SERVER', 435 unconstrained_client='async-limited', 436 secure=secure, 437 client_threads_per_cq=1000000, 438 server_threads_per_cq=1000000, 439 categories=inproc_categories + [SCALABLE]) 440 441 yield _ping_pong_scenario( 442 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, 443 rpc_type='STREAMING', 444 client_type='ASYNC_CLIENT', 445 server_type='ASYNC_GENERIC_SERVER', 446 unconstrained_client='async-limited', 447 use_generic_payload=True, 448 async_server_threads=1, 449 minimal_stack=not secure, 450 secure=secure, 451 categories=[SWEEP]) 452 453 yield _ping_pong_scenario( 454 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' 455 % (secstr), 456 rpc_type='UNARY', 457 client_type='ASYNC_CLIENT', 458 server_type='SYNC_SERVER', 459 unconstrained_client='async', 460 secure=secure, 461 minimal_stack=not secure, 462 categories=smoketest_categories + inproc_categories + 463 [SCALABLE]) 464 465 yield _ping_pong_scenario( 466 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' 467 % (secstr), 468 rpc_type='UNARY', 469 client_type='ASYNC_CLIENT', 470 server_type='ASYNC_SERVER', 471 channels=1, 472 outstanding=64, 473 req_size=128, 474 resp_size=8 * 1024 * 1024, 475 secure=secure, 476 minimal_stack=not secure, 477 categories=inproc_categories + [SCALABLE]) 478 479 yield _ping_pong_scenario( 480 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' 481 % secstr, 482 rpc_type='STREAMING', 483 client_type='ASYNC_CLIENT', 484 server_type='SYNC_SERVER', 485 unconstrained_client='async', 486 secure=secure, 487 minimal_stack=not secure, 488 categories=[SWEEP]) 489 490 yield _ping_pong_scenario( 491 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, 492 rpc_type='UNARY', 493 client_type='ASYNC_CLIENT', 494 server_type='ASYNC_SERVER', 495 req_size=1024 * 1024, 496 resp_size=1024 * 1024, 497 secure=secure, 498 minimal_stack=not secure, 499 categories=smoketest_categories + inproc_categories + 500 [SCALABLE]) 501 502 for rpc_type in [ 503 'unary', 'streaming', 'streaming_from_client', 504 'streaming_from_server' 505 ]: 506 for synchronicity in ['sync', 'async']: 507 yield _ping_pong_scenario( 508 'cpp_protobuf_%s_%s_ping_pong_%s' % 509 (synchronicity, rpc_type, secstr), 510 rpc_type=rpc_type.upper(), 511 client_type='%s_CLIENT' % synchronicity.upper(), 512 server_type='%s_SERVER' % synchronicity.upper(), 513 async_server_threads=1, 514 minimal_stack=not secure, 515 secure=secure) 516 517 for size in geometric_progression(1, 1024 * 1024 * 1024 + 1, 518 8): 519 yield _ping_pong_scenario( 520 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % 521 (synchronicity, rpc_type, secstr, size), 522 rpc_type=rpc_type.upper(), 523 req_size=size, 524 resp_size=size, 525 client_type='%s_CLIENT' % synchronicity.upper(), 526 server_type='%s_SERVER' % synchronicity.upper(), 527 unconstrained_client=synchronicity, 528 secure=secure, 529 minimal_stack=not secure, 530 categories=[SWEEP]) 531 532 maybe_scalable = [SCALABLE] 533 if rpc_type == 'streaming_from_server' and synchronicity == 'async' and secure: 534 # protobuf_async_streaming_from_server_qps_unconstrained_secure is very flaky 535 # and has extremely high variance so running it isn't really useful. 536 # see b/198275705 537 maybe_scalable = [SWEEP] 538 539 yield _ping_pong_scenario( 540 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % 541 (synchronicity, rpc_type, secstr), 542 rpc_type=rpc_type.upper(), 543 client_type='%s_CLIENT' % synchronicity.upper(), 544 server_type='%s_SERVER' % synchronicity.upper(), 545 unconstrained_client=synchronicity, 546 secure=secure, 547 minimal_stack=not secure, 548 server_threads_per_cq=2, 549 client_threads_per_cq=2, 550 categories=inproc_categories + maybe_scalable) 551 552 # TODO(vjpai): Re-enable this test. It has a lot of timeouts 553 # and hasn't yet been conclusively identified as a test failure 554 # or race in the library 555 # yield _ping_pong_scenario( 556 # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), 557 # rpc_type=rpc_type.upper(), 558 # client_type='%s_CLIENT' % synchronicity.upper(), 559 # server_type='%s_SERVER' % synchronicity.upper(), 560 # unconstrained_client=synchronicity, 561 # secure=secure, 562 # categories=smoketest_categories+[SCALABLE], 563 # resource_quota_size=500*1024) 564 565 if rpc_type == 'streaming': 566 for mps in geometric_progression(10, 20, 10): 567 yield _ping_pong_scenario( 568 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' 569 % (synchronicity, rpc_type, mps, secstr), 570 rpc_type=rpc_type.upper(), 571 client_type='%s_CLIENT' % synchronicity.upper(), 572 server_type='%s_SERVER' % synchronicity.upper(), 573 unconstrained_client=synchronicity, 574 secure=secure, 575 messages_per_stream=mps, 576 minimal_stack=not secure, 577 categories=inproc_categories + [SCALABLE]) 578 579 for mps in geometric_progression(1, 200, math.sqrt(10)): 580 yield _ping_pong_scenario( 581 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' 582 % (synchronicity, rpc_type, mps, secstr), 583 rpc_type=rpc_type.upper(), 584 client_type='%s_CLIENT' % synchronicity.upper(), 585 server_type='%s_SERVER' % synchronicity.upper(), 586 unconstrained_client=synchronicity, 587 secure=secure, 588 messages_per_stream=mps, 589 minimal_stack=not secure, 590 categories=[SWEEP]) 591 592 for channels in geometric_progression( 593 1, 20000, math.sqrt(10)): 594 for outstanding in geometric_progression( 595 1, 200000, math.sqrt(10)): 596 if synchronicity == 'sync' and outstanding > 1200: 597 continue 598 if outstanding < channels: 599 continue 600 yield _ping_pong_scenario( 601 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' 602 % (synchronicity, rpc_type, secstr, channels, 603 outstanding), 604 rpc_type=rpc_type.upper(), 605 client_type='%s_CLIENT' % synchronicity.upper(), 606 server_type='%s_SERVER' % synchronicity.upper(), 607 unconstrained_client=synchronicity, 608 secure=secure, 609 minimal_stack=not secure, 610 categories=[SWEEP], 611 channels=channels, 612 outstanding=outstanding) 613 614 def __str__(self): 615 return 'c++' 616 617 618class CSharpLanguage(Language): 619 """The legacy Grpc.Core implementation from grpc/grpc.""" 620 621 def worker_cmdline(self): 622 return ['tools/run_tests/performance/run_worker_csharp.sh'] 623 624 def worker_port_offset(self): 625 return 100 626 627 def scenarios(self): 628 yield _ping_pong_scenario('csharp_generic_async_streaming_ping_pong', 629 rpc_type='STREAMING', 630 client_type='ASYNC_CLIENT', 631 server_type='ASYNC_GENERIC_SERVER', 632 use_generic_payload=True, 633 categories=[SMOKETEST, SCALABLE]) 634 635 yield _ping_pong_scenario( 636 'csharp_generic_async_streaming_ping_pong_insecure_1MB', 637 rpc_type='STREAMING', 638 client_type='ASYNC_CLIENT', 639 server_type='ASYNC_GENERIC_SERVER', 640 req_size=1024 * 1024, 641 resp_size=1024 * 1024, 642 use_generic_payload=True, 643 secure=False, 644 categories=[SMOKETEST, SCALABLE]) 645 646 yield _ping_pong_scenario( 647 'csharp_generic_async_streaming_qps_unconstrained_insecure', 648 rpc_type='STREAMING', 649 client_type='ASYNC_CLIENT', 650 server_type='ASYNC_GENERIC_SERVER', 651 unconstrained_client='async', 652 use_generic_payload=True, 653 secure=False, 654 categories=[SMOKETEST, SCALABLE]) 655 656 yield _ping_pong_scenario('csharp_protobuf_async_streaming_ping_pong', 657 rpc_type='STREAMING', 658 client_type='ASYNC_CLIENT', 659 server_type='ASYNC_SERVER') 660 661 yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong', 662 rpc_type='UNARY', 663 client_type='ASYNC_CLIENT', 664 server_type='ASYNC_SERVER', 665 categories=[SMOKETEST, SCALABLE]) 666 667 yield _ping_pong_scenario( 668 'csharp_protobuf_sync_to_async_unary_ping_pong', 669 rpc_type='UNARY', 670 client_type='SYNC_CLIENT', 671 server_type='ASYNC_SERVER') 672 673 yield _ping_pong_scenario( 674 'csharp_protobuf_async_unary_qps_unconstrained', 675 rpc_type='UNARY', 676 client_type='ASYNC_CLIENT', 677 server_type='ASYNC_SERVER', 678 unconstrained_client='async', 679 categories=[SMOKETEST, SCALABLE]) 680 681 yield _ping_pong_scenario( 682 'csharp_protobuf_async_streaming_qps_unconstrained', 683 rpc_type='STREAMING', 684 client_type='ASYNC_CLIENT', 685 server_type='ASYNC_SERVER', 686 unconstrained_client='async', 687 categories=[SCALABLE]) 688 689 yield _ping_pong_scenario('csharp_to_cpp_protobuf_sync_unary_ping_pong', 690 rpc_type='UNARY', 691 client_type='SYNC_CLIENT', 692 server_type='SYNC_SERVER', 693 server_language='c++', 694 async_server_threads=1, 695 categories=[SMOKETEST, SCALABLE]) 696 697 yield _ping_pong_scenario( 698 'csharp_to_cpp_protobuf_async_streaming_ping_pong', 699 rpc_type='STREAMING', 700 client_type='ASYNC_CLIENT', 701 server_type='ASYNC_SERVER', 702 server_language='c++', 703 async_server_threads=1) 704 705 yield _ping_pong_scenario( 706 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', 707 rpc_type='UNARY', 708 client_type='ASYNC_CLIENT', 709 server_type='ASYNC_SERVER', 710 unconstrained_client='async', 711 server_language='c++', 712 categories=[SCALABLE]) 713 714 yield _ping_pong_scenario( 715 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', 716 rpc_type='UNARY', 717 client_type='SYNC_CLIENT', 718 server_type='ASYNC_SERVER', 719 unconstrained_client='sync', 720 server_language='c++', 721 categories=[SCALABLE]) 722 723 yield _ping_pong_scenario( 724 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', 725 rpc_type='UNARY', 726 client_type='ASYNC_CLIENT', 727 server_type='ASYNC_SERVER', 728 unconstrained_client='async', 729 client_language='c++', 730 categories=[SCALABLE]) 731 732 yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong_1MB', 733 rpc_type='UNARY', 734 client_type='ASYNC_CLIENT', 735 server_type='ASYNC_SERVER', 736 req_size=1024 * 1024, 737 resp_size=1024 * 1024, 738 categories=[SMOKETEST, SCALABLE]) 739 740 def __str__(self): 741 return 'csharp' 742 743 744class DotnetLanguage(Language): 745 """The pure C# implementation from grpc/grpc-dotnet.""" 746 747 def worker_cmdline(self): 748 # grpc-dotnet worker is only supported by the new GKE based OSS benchmark 749 # framework, and the worker_cmdline() is only used by run_performance_tests.py 750 return ['grpc_dotnet_not_supported_by_legacy_performance_runner.sh'] 751 752 def worker_port_offset(self): 753 return 1100 754 755 def scenarios(self): 756 yield _ping_pong_scenario('dotnet_generic_async_streaming_ping_pong', 757 rpc_type='STREAMING', 758 client_type='ASYNC_CLIENT', 759 server_type='ASYNC_GENERIC_SERVER', 760 use_generic_payload=True, 761 categories=[SMOKETEST, SCALABLE]) 762 763 yield _ping_pong_scenario( 764 'dotnet_generic_async_streaming_ping_pong_insecure_1MB', 765 rpc_type='STREAMING', 766 client_type='ASYNC_CLIENT', 767 server_type='ASYNC_GENERIC_SERVER', 768 req_size=1024 * 1024, 769 resp_size=1024 * 1024, 770 use_generic_payload=True, 771 secure=False, 772 categories=[SMOKETEST, SCALABLE]) 773 774 yield _ping_pong_scenario( 775 'dotnet_generic_async_streaming_qps_unconstrained_insecure', 776 rpc_type='STREAMING', 777 client_type='ASYNC_CLIENT', 778 server_type='ASYNC_GENERIC_SERVER', 779 unconstrained_client='async', 780 use_generic_payload=True, 781 secure=False, 782 categories=[SMOKETEST, SCALABLE]) 783 784 yield _ping_pong_scenario('dotnet_protobuf_async_streaming_ping_pong', 785 rpc_type='STREAMING', 786 client_type='ASYNC_CLIENT', 787 server_type='ASYNC_SERVER') 788 789 yield _ping_pong_scenario('dotnet_protobuf_async_unary_ping_pong', 790 rpc_type='UNARY', 791 client_type='ASYNC_CLIENT', 792 server_type='ASYNC_SERVER', 793 categories=[SMOKETEST, SCALABLE]) 794 795 yield _ping_pong_scenario( 796 'dotnet_protobuf_sync_to_async_unary_ping_pong', 797 rpc_type='UNARY', 798 client_type='SYNC_CLIENT', 799 server_type='ASYNC_SERVER') 800 801 yield _ping_pong_scenario( 802 'dotnet_protobuf_async_unary_qps_unconstrained', 803 rpc_type='UNARY', 804 client_type='ASYNC_CLIENT', 805 server_type='ASYNC_SERVER', 806 unconstrained_client='async', 807 categories=[SMOKETEST, SCALABLE]) 808 809 yield _ping_pong_scenario( 810 'dotnet_protobuf_async_streaming_qps_unconstrained', 811 rpc_type='STREAMING', 812 client_type='ASYNC_CLIENT', 813 server_type='ASYNC_SERVER', 814 unconstrained_client='async', 815 categories=[SCALABLE]) 816 817 yield _ping_pong_scenario('dotnet_to_cpp_protobuf_sync_unary_ping_pong', 818 rpc_type='UNARY', 819 client_type='SYNC_CLIENT', 820 server_type='SYNC_SERVER', 821 server_language='c++', 822 async_server_threads=1, 823 categories=[SMOKETEST, SCALABLE]) 824 825 yield _ping_pong_scenario( 826 'dotnet_to_cpp_protobuf_async_streaming_ping_pong', 827 rpc_type='STREAMING', 828 client_type='ASYNC_CLIENT', 829 server_type='ASYNC_SERVER', 830 server_language='c++', 831 async_server_threads=1) 832 833 yield _ping_pong_scenario( 834 'dotnet_to_cpp_protobuf_async_unary_qps_unconstrained', 835 rpc_type='UNARY', 836 client_type='ASYNC_CLIENT', 837 server_type='ASYNC_SERVER', 838 unconstrained_client='async', 839 server_language='c++', 840 categories=[SCALABLE]) 841 842 yield _ping_pong_scenario( 843 'dotnet_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', 844 rpc_type='UNARY', 845 client_type='SYNC_CLIENT', 846 server_type='ASYNC_SERVER', 847 unconstrained_client='sync', 848 server_language='c++', 849 categories=[SCALABLE]) 850 851 yield _ping_pong_scenario( 852 'cpp_to_dotnet_protobuf_async_unary_qps_unconstrained', 853 rpc_type='UNARY', 854 client_type='ASYNC_CLIENT', 855 server_type='ASYNC_SERVER', 856 unconstrained_client='async', 857 client_language='c++', 858 categories=[SCALABLE]) 859 860 yield _ping_pong_scenario('dotnet_protobuf_async_unary_ping_pong_1MB', 861 rpc_type='UNARY', 862 client_type='ASYNC_CLIENT', 863 server_type='ASYNC_SERVER', 864 req_size=1024 * 1024, 865 resp_size=1024 * 1024, 866 categories=[SMOKETEST, SCALABLE]) 867 868 def __str__(self): 869 return 'dotnet' 870 871 872class PythonLanguage(Language): 873 874 def worker_cmdline(self): 875 return ['tools/run_tests/performance/run_worker_python.sh'] 876 877 def worker_port_offset(self): 878 return 500 879 880 def scenarios(self): 881 yield _ping_pong_scenario( 882 'python_protobuf_async_unary_5000rpcs_1KB_psm', 883 rpc_type='UNARY', 884 client_type='ASYNC_CLIENT', 885 server_type='ASYNC_SERVER', 886 req_size=1024, 887 resp_size=1024, 888 outstanding=5000, 889 channels=1, 890 num_clients=1, 891 secure=False, 892 async_server_threads=1, 893 categories=[PSM]) 894 895 yield _ping_pong_scenario('python_generic_sync_streaming_ping_pong', 896 rpc_type='STREAMING', 897 client_type='SYNC_CLIENT', 898 server_type='ASYNC_GENERIC_SERVER', 899 use_generic_payload=True, 900 categories=[SMOKETEST, SCALABLE]) 901 902 yield _ping_pong_scenario('python_protobuf_sync_streaming_ping_pong', 903 rpc_type='STREAMING', 904 client_type='SYNC_CLIENT', 905 server_type='ASYNC_SERVER') 906 907 yield _ping_pong_scenario('python_protobuf_async_unary_ping_pong', 908 rpc_type='UNARY', 909 client_type='ASYNC_CLIENT', 910 server_type='ASYNC_SERVER') 911 912 yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong', 913 rpc_type='UNARY', 914 client_type='SYNC_CLIENT', 915 server_type='ASYNC_SERVER', 916 categories=[SMOKETEST, SCALABLE]) 917 918 yield _ping_pong_scenario( 919 'python_protobuf_sync_unary_qps_unconstrained', 920 rpc_type='UNARY', 921 client_type='SYNC_CLIENT', 922 server_type='ASYNC_SERVER', 923 unconstrained_client='sync') 924 925 yield _ping_pong_scenario( 926 'python_protobuf_sync_streaming_qps_unconstrained', 927 rpc_type='STREAMING', 928 client_type='SYNC_CLIENT', 929 server_type='ASYNC_SERVER', 930 unconstrained_client='sync') 931 932 yield _ping_pong_scenario('python_to_cpp_protobuf_sync_unary_ping_pong', 933 rpc_type='UNARY', 934 client_type='SYNC_CLIENT', 935 server_type='ASYNC_SERVER', 936 server_language='c++', 937 async_server_threads=0, 938 categories=[SMOKETEST, SCALABLE]) 939 940 yield _ping_pong_scenario( 941 'python_to_cpp_protobuf_sync_streaming_ping_pong', 942 rpc_type='STREAMING', 943 client_type='SYNC_CLIENT', 944 server_type='ASYNC_SERVER', 945 server_language='c++', 946 async_server_threads=1) 947 948 yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong_1MB', 949 rpc_type='UNARY', 950 client_type='SYNC_CLIENT', 951 server_type='ASYNC_SERVER', 952 req_size=1024 * 1024, 953 resp_size=1024 * 1024, 954 categories=[SMOKETEST, SCALABLE]) 955 956 def __str__(self): 957 return 'python' 958 959 960class PythonAsyncIOLanguage(Language): 961 962 def worker_cmdline(self): 963 return ['tools/run_tests/performance/run_worker_python_asyncio.sh'] 964 965 def worker_port_offset(self): 966 return 1200 967 968 def scenarios(self): 969 yield _ping_pong_scenario( 970 'python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm', 971 rpc_type='UNARY', 972 client_type='ASYNC_CLIENT', 973 server_type='ASYNC_SERVER', 974 req_size=1024, 975 resp_size=1024, 976 outstanding=5000, 977 channels=1, 978 num_clients=1, 979 secure=False, 980 async_server_threads=1, 981 categories=[PSM]) 982 983 for outstanding in [64, 128, 256, 512]: 984 for channels in [1, 4]: 985 yield _ping_pong_scenario( 986 'python_asyncio_protobuf_async_unary_ping_pong_%dx%d_max' % 987 ( 988 outstanding, 989 channels, 990 ), 991 rpc_type='UNARY', 992 client_type='ASYNC_CLIENT', 993 server_type='ASYNC_SERVER', 994 outstanding=outstanding * channels, 995 channels=channels, 996 client_processes=0, 997 server_processes=0, 998 unconstrained_client='async', 999 categories=[SCALABLE]) 1000 1001 yield _ping_pong_scenario( 1002 'python_asyncio_protobuf_async_unary_ping_pong_%d_1thread' % 1003 outstanding, 1004 rpc_type='UNARY', 1005 client_type='ASYNC_CLIENT', 1006 server_type='ASYNC_SERVER', 1007 outstanding=outstanding, 1008 channels=1, 1009 client_processes=1, 1010 server_processes=1, 1011 unconstrained_client='async', 1012 categories=[SCALABLE]) 1013 1014 yield _ping_pong_scenario( 1015 'python_asyncio_generic_async_streaming_ping_pong', 1016 rpc_type='STREAMING', 1017 client_type='ASYNC_CLIENT', 1018 server_type='ASYNC_GENERIC_SERVER', 1019 channels=1, 1020 client_processes=1, 1021 server_processes=1, 1022 use_generic_payload=True, 1023 categories=[SMOKETEST, SCALABLE]) 1024 1025 yield _ping_pong_scenario( 1026 'python_asyncio_protobuf_async_streaming_ping_pong', 1027 rpc_type='STREAMING', 1028 client_type='ASYNC_CLIENT', 1029 server_type='ASYNC_SERVER', 1030 channels=1, 1031 client_processes=1, 1032 server_processes=1, 1033 categories=[SMOKETEST, SCALABLE]) 1034 1035 yield _ping_pong_scenario( 1036 'python_asyncio_protobuf_async_unary_ping_pong', 1037 rpc_type='UNARY', 1038 client_type='ASYNC_CLIENT', 1039 server_type='ASYNC_SERVER', 1040 client_processes=1, 1041 server_processes=1, 1042 categories=[SMOKETEST, SCALABLE]) 1043 1044 yield _ping_pong_scenario( 1045 'python_asyncio_protobuf_async_unary_ping_pong', 1046 rpc_type='UNARY', 1047 client_type='ASYNC_CLIENT', 1048 server_type='ASYNC_SERVER', 1049 channels=1, 1050 client_processes=1, 1051 server_processes=1, 1052 categories=[SMOKETEST, SCALABLE]) 1053 1054 yield _ping_pong_scenario( 1055 'python_asyncio_protobuf_async_unary_qps_unconstrained', 1056 rpc_type='UNARY', 1057 client_type='ASYNC_CLIENT', 1058 server_type='ASYNC_SERVER', 1059 channels=1, 1060 unconstrained_client='async') 1061 1062 yield _ping_pong_scenario( 1063 'python_asyncio_protobuf_async_streaming_qps_unconstrained', 1064 rpc_type='STREAMING', 1065 client_type='ASYNC_CLIENT', 1066 server_type='ASYNC_SERVER', 1067 channels=1, 1068 unconstrained_client='async') 1069 1070 yield _ping_pong_scenario( 1071 'python_asyncio_to_cpp_protobuf_async_unary_ping_pong_1thread', 1072 rpc_type='UNARY', 1073 client_type='ASYNC_CLIENT', 1074 server_type='ASYNC_SERVER', 1075 server_language='c++', 1076 channels=1, 1077 client_processes=1, 1078 unconstrained_client='async', 1079 categories=[SMOKETEST, SCALABLE]) 1080 1081 yield _ping_pong_scenario( 1082 'python_asyncio_to_cpp_protobuf_async_unary_ping_pong_max', 1083 rpc_type='UNARY', 1084 client_type='ASYNC_CLIENT', 1085 server_type='ASYNC_SERVER', 1086 unconstrained_client='async', 1087 channels=1, 1088 client_processes=0, 1089 server_language='c++', 1090 categories=[SMOKETEST, SCALABLE]) 1091 1092 yield _ping_pong_scenario( 1093 'python_asyncio_to_cpp_protobuf_sync_streaming_ping_pong_1thread', 1094 rpc_type='STREAMING', 1095 client_type='ASYNC_CLIENT', 1096 server_type='ASYNC_SERVER', 1097 channels=1, 1098 client_processes=1, 1099 server_processes=1, 1100 unconstrained_client='async', 1101 server_language='c++') 1102 1103 yield _ping_pong_scenario( 1104 'python_asyncio_protobuf_async_unary_ping_pong_1MB', 1105 rpc_type='UNARY', 1106 client_type='ASYNC_CLIENT', 1107 server_type='ASYNC_SERVER', 1108 req_size=1024 * 1024, 1109 resp_size=1024 * 1024, 1110 channels=1, 1111 client_processes=1, 1112 server_processes=1, 1113 categories=[SMOKETEST, SCALABLE]) 1114 1115 def __str__(self): 1116 return 'python_asyncio' 1117 1118 1119class RubyLanguage(Language): 1120 1121 def worker_cmdline(self): 1122 return ['tools/run_tests/performance/run_worker_ruby.sh'] 1123 1124 def worker_port_offset(self): 1125 return 300 1126 1127 def scenarios(self): 1128 yield _ping_pong_scenario('ruby_protobuf_sync_streaming_ping_pong', 1129 rpc_type='STREAMING', 1130 client_type='SYNC_CLIENT', 1131 server_type='SYNC_SERVER', 1132 categories=[SMOKETEST, SCALABLE]) 1133 1134 yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong', 1135 rpc_type='UNARY', 1136 client_type='SYNC_CLIENT', 1137 server_type='SYNC_SERVER', 1138 categories=[SMOKETEST, SCALABLE]) 1139 1140 yield _ping_pong_scenario('ruby_protobuf_sync_unary_qps_unconstrained', 1141 rpc_type='UNARY', 1142 client_type='SYNC_CLIENT', 1143 server_type='SYNC_SERVER', 1144 unconstrained_client='sync') 1145 1146 yield _ping_pong_scenario( 1147 'ruby_protobuf_sync_streaming_qps_unconstrained', 1148 rpc_type='STREAMING', 1149 client_type='SYNC_CLIENT', 1150 server_type='SYNC_SERVER', 1151 unconstrained_client='sync') 1152 1153 yield _ping_pong_scenario('ruby_to_cpp_protobuf_sync_unary_ping_pong', 1154 rpc_type='UNARY', 1155 client_type='SYNC_CLIENT', 1156 server_type='SYNC_SERVER', 1157 server_language='c++', 1158 async_server_threads=1) 1159 1160 yield _ping_pong_scenario( 1161 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', 1162 rpc_type='STREAMING', 1163 client_type='SYNC_CLIENT', 1164 server_type='SYNC_SERVER', 1165 server_language='c++', 1166 async_server_threads=1) 1167 1168 yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong_1MB', 1169 rpc_type='UNARY', 1170 client_type='SYNC_CLIENT', 1171 server_type='SYNC_SERVER', 1172 req_size=1024 * 1024, 1173 resp_size=1024 * 1024, 1174 categories=[SMOKETEST, SCALABLE]) 1175 1176 def __str__(self): 1177 return 'ruby' 1178 1179 1180class Php7Language(Language): 1181 1182 def __init__(self, php7_protobuf_c=False): 1183 super().__init__() 1184 self.php7_protobuf_c = php7_protobuf_c 1185 1186 def worker_cmdline(self): 1187 if self.php7_protobuf_c: 1188 return [ 1189 'tools/run_tests/performance/run_worker_php.sh', 1190 '--use_protobuf_c_extension' 1191 ] 1192 return ['tools/run_tests/performance/run_worker_php.sh'] 1193 1194 def worker_port_offset(self): 1195 if self.php7_protobuf_c: 1196 return 900 1197 return 800 1198 1199 def scenarios(self): 1200 php7_extension_mode = 'php7_protobuf_php_extension' 1201 if self.php7_protobuf_c: 1202 php7_extension_mode = 'php7_protobuf_c_extension' 1203 1204 yield _ping_pong_scenario( 1205 '%s_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm' % 1206 php7_extension_mode, 1207 rpc_type='UNARY', 1208 client_type='ASYNC_CLIENT', 1209 server_type='ASYNC_SERVER', 1210 server_language='c++', 1211 req_size=1024, 1212 resp_size=1024, 1213 outstanding=5000, 1214 channels=1, 1215 num_clients=1, 1216 secure=False, 1217 async_server_threads=1, 1218 categories=[PSM]) 1219 1220 yield _ping_pong_scenario('%s_to_cpp_protobuf_sync_unary_ping_pong' % 1221 php7_extension_mode, 1222 rpc_type='UNARY', 1223 client_type='SYNC_CLIENT', 1224 server_type='SYNC_SERVER', 1225 server_language='c++', 1226 async_server_threads=1) 1227 1228 yield _ping_pong_scenario( 1229 '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode, 1230 rpc_type='STREAMING', 1231 client_type='SYNC_CLIENT', 1232 server_type='SYNC_SERVER', 1233 server_language='c++', 1234 async_server_threads=1) 1235 1236 # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs 1237 # better than async_server_threads=0/CPU usage 490%. 1238 yield _ping_pong_scenario( 1239 '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' % 1240 php7_extension_mode, 1241 rpc_type='UNARY', 1242 client_type='SYNC_CLIENT', 1243 server_type='ASYNC_SERVER', 1244 server_language='c++', 1245 outstanding=1, 1246 async_server_threads=1, 1247 unconstrained_client='sync') 1248 1249 yield _ping_pong_scenario( 1250 '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' % 1251 php7_extension_mode, 1252 rpc_type='STREAMING', 1253 client_type='SYNC_CLIENT', 1254 server_type='ASYNC_SERVER', 1255 server_language='c++', 1256 outstanding=1, 1257 async_server_threads=1, 1258 unconstrained_client='sync') 1259 1260 def __str__(self): 1261 if self.php7_protobuf_c: 1262 return 'php7_protobuf_c' 1263 return 'php7' 1264 1265 1266class JavaLanguage(Language): 1267 1268 def worker_cmdline(self): 1269 return ['tools/run_tests/performance/run_worker_java.sh'] 1270 1271 def worker_port_offset(self): 1272 return 400 1273 1274 def scenarios(self): 1275 yield _ping_pong_scenario('java_protobuf_async_unary_5000rpcs_1KB_psm', 1276 rpc_type='UNARY', 1277 client_type='ASYNC_CLIENT', 1278 server_type='ASYNC_SERVER', 1279 req_size=1024, 1280 resp_size=1024, 1281 outstanding=5000, 1282 channels=1, 1283 num_clients=1, 1284 secure=False, 1285 async_server_threads=1, 1286 warmup_seconds=JAVA_WARMUP_SECONDS, 1287 categories=[PSM]) 1288 1289 for secure in [True, False]: 1290 secstr = 'secure' if secure else 'insecure' 1291 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1292 1293 yield _ping_pong_scenario( 1294 'java_generic_async_streaming_ping_pong_%s' % secstr, 1295 rpc_type='STREAMING', 1296 client_type='ASYNC_CLIENT', 1297 server_type='ASYNC_GENERIC_SERVER', 1298 use_generic_payload=True, 1299 async_server_threads=1, 1300 secure=secure, 1301 warmup_seconds=JAVA_WARMUP_SECONDS, 1302 categories=smoketest_categories) 1303 1304 yield _ping_pong_scenario( 1305 'java_protobuf_async_streaming_ping_pong_%s' % secstr, 1306 rpc_type='STREAMING', 1307 client_type='ASYNC_CLIENT', 1308 server_type='ASYNC_SERVER', 1309 async_server_threads=1, 1310 secure=secure, 1311 warmup_seconds=JAVA_WARMUP_SECONDS) 1312 1313 yield _ping_pong_scenario('java_protobuf_async_unary_ping_pong_%s' % 1314 secstr, 1315 rpc_type='UNARY', 1316 client_type='ASYNC_CLIENT', 1317 server_type='ASYNC_SERVER', 1318 async_server_threads=1, 1319 secure=secure, 1320 warmup_seconds=JAVA_WARMUP_SECONDS, 1321 categories=smoketest_categories) 1322 1323 yield _ping_pong_scenario('java_protobuf_unary_ping_pong_%s' % 1324 secstr, 1325 rpc_type='UNARY', 1326 client_type='SYNC_CLIENT', 1327 server_type='SYNC_SERVER', 1328 async_server_threads=1, 1329 secure=secure, 1330 warmup_seconds=JAVA_WARMUP_SECONDS) 1331 1332 yield _ping_pong_scenario( 1333 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, 1334 rpc_type='UNARY', 1335 client_type='ASYNC_CLIENT', 1336 server_type='ASYNC_SERVER', 1337 unconstrained_client='async', 1338 secure=secure, 1339 warmup_seconds=JAVA_WARMUP_SECONDS, 1340 categories=smoketest_categories + [SCALABLE]) 1341 1342 yield _ping_pong_scenario( 1343 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, 1344 rpc_type='STREAMING', 1345 client_type='ASYNC_CLIENT', 1346 server_type='ASYNC_SERVER', 1347 unconstrained_client='async', 1348 secure=secure, 1349 warmup_seconds=JAVA_WARMUP_SECONDS, 1350 categories=[SCALABLE]) 1351 1352 yield _ping_pong_scenario( 1353 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, 1354 rpc_type='STREAMING', 1355 client_type='ASYNC_CLIENT', 1356 server_type='ASYNC_GENERIC_SERVER', 1357 unconstrained_client='async', 1358 use_generic_payload=True, 1359 secure=secure, 1360 warmup_seconds=JAVA_WARMUP_SECONDS, 1361 categories=[SCALABLE]) 1362 1363 yield _ping_pong_scenario( 1364 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, 1365 rpc_type='STREAMING', 1366 client_type='ASYNC_CLIENT', 1367 server_type='ASYNC_GENERIC_SERVER', 1368 unconstrained_client='async-limited', 1369 use_generic_payload=True, 1370 async_server_threads=1, 1371 secure=secure, 1372 warmup_seconds=JAVA_WARMUP_SECONDS) 1373 1374 # TODO(jtattermusch): add scenarios java vs C++ 1375 1376 def __str__(self): 1377 return 'java' 1378 1379 1380class GoLanguage(Language): 1381 1382 def worker_cmdline(self): 1383 return ['tools/run_tests/performance/run_worker_go.sh'] 1384 1385 def worker_port_offset(self): 1386 return 600 1387 1388 def scenarios(self): 1389 yield _ping_pong_scenario('go_protobuf_async_unary_5000rpcs_1KB_psm', 1390 rpc_type='UNARY', 1391 client_type='ASYNC_CLIENT', 1392 server_type='ASYNC_SERVER', 1393 req_size=1024, 1394 resp_size=1024, 1395 outstanding=5000, 1396 channels=1, 1397 num_clients=1, 1398 secure=False, 1399 async_server_threads=1, 1400 categories=[PSM]) 1401 1402 for secure in [True, False]: 1403 secstr = 'secure' if secure else 'insecure' 1404 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1405 1406 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1407 # but that's mostly because of lack of better name of the enum value. 1408 yield _ping_pong_scenario('go_generic_sync_streaming_ping_pong_%s' % 1409 secstr, 1410 rpc_type='STREAMING', 1411 client_type='SYNC_CLIENT', 1412 server_type='ASYNC_GENERIC_SERVER', 1413 use_generic_payload=True, 1414 async_server_threads=1, 1415 secure=secure, 1416 categories=smoketest_categories) 1417 1418 yield _ping_pong_scenario( 1419 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, 1420 rpc_type='STREAMING', 1421 client_type='SYNC_CLIENT', 1422 server_type='SYNC_SERVER', 1423 async_server_threads=1, 1424 secure=secure) 1425 1426 yield _ping_pong_scenario('go_protobuf_sync_unary_ping_pong_%s' % 1427 secstr, 1428 rpc_type='UNARY', 1429 client_type='SYNC_CLIENT', 1430 server_type='SYNC_SERVER', 1431 async_server_threads=1, 1432 secure=secure, 1433 categories=smoketest_categories) 1434 1435 # unconstrained_client='async' is intended (client uses goroutines) 1436 yield _ping_pong_scenario( 1437 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, 1438 rpc_type='UNARY', 1439 client_type='SYNC_CLIENT', 1440 server_type='SYNC_SERVER', 1441 unconstrained_client='async', 1442 secure=secure, 1443 categories=smoketest_categories + [SCALABLE]) 1444 1445 # unconstrained_client='async' is intended (client uses goroutines) 1446 yield _ping_pong_scenario( 1447 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, 1448 rpc_type='STREAMING', 1449 client_type='SYNC_CLIENT', 1450 server_type='SYNC_SERVER', 1451 unconstrained_client='async', 1452 secure=secure, 1453 categories=[SCALABLE]) 1454 1455 # unconstrained_client='async' is intended (client uses goroutines) 1456 # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, 1457 # but that's mostly because of lack of better name of the enum value. 1458 yield _ping_pong_scenario( 1459 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, 1460 rpc_type='STREAMING', 1461 client_type='SYNC_CLIENT', 1462 server_type='ASYNC_GENERIC_SERVER', 1463 unconstrained_client='async', 1464 use_generic_payload=True, 1465 secure=secure, 1466 categories=[SCALABLE]) 1467 1468 # TODO(jtattermusch): add scenarios go vs C++ 1469 1470 def __str__(self): 1471 return 'go' 1472 1473 1474class NodeLanguage(Language): 1475 1476 def __init__(self, node_purejs=False): 1477 super().__init__() 1478 self.node_purejs = node_purejs 1479 1480 def worker_cmdline(self): 1481 fixture = 'native_js' if self.node_purejs else 'native_native' 1482 return [ 1483 'tools/run_tests/performance/run_worker_node.sh', fixture, 1484 '--benchmark_impl=grpc' 1485 ] 1486 1487 def worker_port_offset(self): 1488 if self.node_purejs: 1489 return 1100 1490 return 1000 1491 1492 def scenarios(self): 1493 node_implementation = 'node_purejs' if self.node_purejs else 'node' 1494 1495 yield _ping_pong_scenario( 1496 '%s_to_node_protobuf_async_unary_5000rpcs_1KB_psm' % 1497 (node_implementation), 1498 rpc_type='UNARY', 1499 client_type='ASYNC_CLIENT', 1500 server_type='ASYNC_SERVER', 1501 server_language='node', 1502 req_size=1024, 1503 resp_size=1024, 1504 outstanding=5000, 1505 channels=1, 1506 num_clients=1, 1507 secure=False, 1508 async_server_threads=1, 1509 categories=[PSM]) 1510 1511 for secure in [True, False]: 1512 secstr = 'secure' if secure else 'insecure' 1513 smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] 1514 1515 yield _ping_pong_scenario( 1516 '%s_to_node_generic_async_streaming_ping_pong_%s' % 1517 (node_implementation, secstr), 1518 rpc_type='STREAMING', 1519 client_type='ASYNC_CLIENT', 1520 server_type='ASYNC_GENERIC_SERVER', 1521 server_language='node', 1522 use_generic_payload=True, 1523 async_server_threads=1, 1524 secure=secure, 1525 categories=smoketest_categories) 1526 1527 yield _ping_pong_scenario( 1528 '%s_to_node_protobuf_async_streaming_ping_pong_%s' % 1529 (node_implementation, secstr), 1530 rpc_type='STREAMING', 1531 client_type='ASYNC_CLIENT', 1532 server_type='ASYNC_SERVER', 1533 server_language='node', 1534 async_server_threads=1, 1535 secure=secure) 1536 1537 yield _ping_pong_scenario( 1538 '%s_to_node_protobuf_async_unary_ping_pong_%s' % 1539 (node_implementation, secstr), 1540 rpc_type='UNARY', 1541 client_type='ASYNC_CLIENT', 1542 server_type='ASYNC_SERVER', 1543 server_language='node', 1544 async_server_threads=1, 1545 secure=secure, 1546 categories=smoketest_categories) 1547 1548 yield _ping_pong_scenario( 1549 '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' % 1550 (node_implementation, secstr), 1551 rpc_type='UNARY', 1552 client_type='ASYNC_CLIENT', 1553 server_type='ASYNC_SERVER', 1554 server_language='node', 1555 unconstrained_client='async', 1556 secure=secure, 1557 categories=smoketest_categories + [SCALABLE]) 1558 1559 yield _ping_pong_scenario( 1560 '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' % 1561 (node_implementation, secstr), 1562 rpc_type='STREAMING', 1563 client_type='ASYNC_CLIENT', 1564 server_type='ASYNC_SERVER', 1565 server_language='node', 1566 unconstrained_client='async', 1567 secure=secure, 1568 categories=[SCALABLE]) 1569 1570 yield _ping_pong_scenario( 1571 '%s_to_node_generic_async_streaming_qps_unconstrained_%s' % 1572 (node_implementation, secstr), 1573 rpc_type='STREAMING', 1574 client_type='ASYNC_CLIENT', 1575 server_type='ASYNC_GENERIC_SERVER', 1576 server_language='node', 1577 unconstrained_client='async', 1578 use_generic_payload=True, 1579 secure=secure, 1580 categories=[SCALABLE]) 1581 1582 # TODO(murgatroid99): add scenarios node vs C++ 1583 1584 def __str__(self): 1585 if self.node_purejs: 1586 return 'node_purejs' 1587 return 'node' 1588 1589class RustLanguage: 1590 1591 def __init__(self): 1592 self.safename = str(self) 1593 1594 def worker_cmdline(self): 1595 return ['tools/run_tests/performance/run_worker_rust.sh'] 1596 1597 def worker_port_offset(self): 1598 return 100 1599 1600 def scenarios(self): 1601 yield _ping_pong_scenario( 1602 'rust_generic_async_streaming_ping_pong', 1603 rpc_type='STREAMING', 1604 client_type='ASYNC_CLIENT', 1605 server_type='ASYNC_GENERIC_SERVER', 1606 use_generic_payload=True, 1607 categories=[SMOKETEST, SCALABLE]) 1608 1609 yield _ping_pong_scenario( 1610 'rust_protobuf_async_streaming_ping_pong', 1611 rpc_type='STREAMING', 1612 client_type='ASYNC_CLIENT', 1613 server_type='ASYNC_SERVER') 1614 1615 yield _ping_pong_scenario( 1616 'rust_protobuf_async_unary_ping_pong', 1617 rpc_type='UNARY', 1618 client_type='ASYNC_CLIENT', 1619 server_type='ASYNC_SERVER', 1620 categories=[SMOKETEST, SCALABLE]) 1621 1622 yield _ping_pong_scenario( 1623 'rust_protobuf_sync_to_async_unary_ping_pong', 1624 rpc_type='UNARY', 1625 client_type='SYNC_CLIENT', 1626 server_type='ASYNC_SERVER') 1627 1628 yield _ping_pong_scenario( 1629 'rust_protobuf_async_unary_qps_unconstrained', 1630 rpc_type='UNARY', 1631 client_type='ASYNC_CLIENT', 1632 server_type='ASYNC_SERVER', 1633 unconstrained_client='async', 1634 categories=[SMOKETEST,SCALABLE]) 1635 1636 yield _ping_pong_scenario( 1637 'rust_protobuf_async_streaming_qps_unconstrained', 1638 rpc_type='STREAMING', 1639 client_type='ASYNC_CLIENT', 1640 server_type='ASYNC_SERVER', 1641 unconstrained_client='async', 1642 categories=[SCALABLE]) 1643 1644 yield _ping_pong_scenario( 1645 'rust_protobuf_async_streaming_from_client_unconstrained', 1646 rpc_type='STREAMING_FROM_CLIENT', 1647 client_type='ASYNC_CLIENT', 1648 server_type='ASYNC_SERVER', 1649 unconstrained_client='async', 1650 categories=[SCALABLE]) 1651 1652 yield _ping_pong_scenario( 1653 'rust_to_cpp_protobuf_sync_unary_ping_pong', 1654 rpc_type='UNARY', 1655 client_type='SYNC_CLIENT', 1656 server_type='SYNC_SERVER', 1657 server_language='c++', 1658 async_server_threads=1, 1659 categories=[SMOKETEST, SCALABLE]) 1660 1661 yield _ping_pong_scenario( 1662 'rust_to_cpp_protobuf_async_streaming_ping_pong', 1663 rpc_type='STREAMING', 1664 client_type='ASYNC_CLIENT', 1665 server_type='ASYNC_SERVER', 1666 server_language='c++', 1667 async_server_threads=1) 1668 1669 yield _ping_pong_scenario( 1670 'rust_to_cpp_protobuf_async_unary_qps_unconstrained', 1671 rpc_type='UNARY', 1672 client_type='ASYNC_CLIENT', 1673 server_type='ASYNC_SERVER', 1674 unconstrained_client='async', 1675 server_language='c++', 1676 categories=[SCALABLE]) 1677 1678 yield _ping_pong_scenario( 1679 'rust_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', 1680 rpc_type='UNARY', 1681 client_type='SYNC_CLIENT', 1682 server_type='ASYNC_SERVER', 1683 unconstrained_client='sync', 1684 server_language='c++', 1685 categories=[SCALABLE]) 1686 1687 yield _ping_pong_scenario( 1688 'cpp_to_rust_protobuf_async_unary_qps_unconstrained', 1689 rpc_type='UNARY', 1690 client_type='ASYNC_CLIENT', 1691 server_type='ASYNC_SERVER', 1692 unconstrained_client='async', 1693 client_language='c++', 1694 categories=[SCALABLE]) 1695 1696 yield _ping_pong_scenario( 1697 'rust_protobuf_async_unary_ping_pong_1MB', 1698 rpc_type='UNARY', 1699 client_type='ASYNC_CLIENT', 1700 server_type='ASYNC_SERVER', 1701 req_size=1024*1024, 1702 resp_size=1024*1024, 1703 categories=[SMOKETEST, SCALABLE]) 1704 1705 1706 def __str__(self): 1707 return 'rust' 1708 1709LANGUAGES = { 1710 'c++': CXXLanguage(), 1711 'csharp': CSharpLanguage(), 1712 'dotnet': DotnetLanguage(), 1713 'ruby': RubyLanguage(), 1714 'rust': RustLanguage(), 1715 'php7': Php7Language(), 1716 'php7_protobuf_c': Php7Language(php7_protobuf_c=True), 1717 'java': JavaLanguage(), 1718 'python': PythonLanguage(), 1719 'python_asyncio': PythonAsyncIOLanguage(), 1720 'go': GoLanguage(), 1721 'node': NodeLanguage(), 1722 'node_purejs': NodeLanguage(node_purejs=True) 1723} 1724