1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "quiche/quic/core/congestion_control/bbr_sender.h"
6
7 #include <algorithm>
8 #include <map>
9 #include <memory>
10 #include <utility>
11
12 #include "quiche/quic/core/congestion_control/rtt_stats.h"
13 #include "quiche/quic/core/crypto/crypto_protocol.h"
14 #include "quiche/quic/core/quic_bandwidth.h"
15 #include "quiche/quic/core/quic_packets.h"
16 #include "quiche/quic/core/quic_types.h"
17 #include "quiche/quic/core/quic_utils.h"
18 #include "quiche/quic/platform/api/quic_flags.h"
19 #include "quiche/quic/platform/api/quic_logging.h"
20 #include "quiche/quic/platform/api/quic_test.h"
21 #include "quiche/quic/test_tools/mock_clock.h"
22 #include "quiche/quic/test_tools/quic_config_peer.h"
23 #include "quiche/quic/test_tools/quic_connection_peer.h"
24 #include "quiche/quic/test_tools/quic_sent_packet_manager_peer.h"
25 #include "quiche/quic/test_tools/quic_test_utils.h"
26 #include "quiche/quic/test_tools/send_algorithm_test_result.pb.h"
27 #include "quiche/quic/test_tools/send_algorithm_test_utils.h"
28 #include "quiche/quic/test_tools/simulator/quic_endpoint.h"
29 #include "quiche/quic/test_tools/simulator/simulator.h"
30 #include "quiche/quic/test_tools/simulator/switch.h"
31 #include "quiche/common/platform/api/quiche_command_line_flags.h"
32
33 using testing::AllOf;
34 using testing::Ge;
35 using testing::Le;
36
37 DEFINE_QUICHE_COMMAND_LINE_FLAG(
38 std::string, quic_bbr_test_regression_mode, "",
39 "One of a) 'record' to record test result (one file per test), or "
40 "b) 'regress' to regress against recorded results, or "
41 "c) <anything else> for non-regression mode.");
42
43 namespace quic {
44 namespace test {
45
46 // Use the initial CWND of 10, as 32 is too much for the test network.
47 const uint32_t kInitialCongestionWindowPackets = 10;
48 const uint32_t kDefaultWindowTCP =
49 kInitialCongestionWindowPackets * kDefaultTCPMSS;
50
51 // Test network parameters. Here, the topology of the network is:
52 //
53 // BBR sender
54 // |
55 // | <-- local link (10 Mbps, 2 ms delay)
56 // |
57 // Network switch
58 // * <-- the bottleneck queue in the direction
59 // | of the receiver
60 // |
61 // | <-- test link (4 Mbps, 30 ms delay)
62 // |
63 // |
64 // Receiver
65 //
66 // The reason the bandwidths chosen are relatively low is the fact that the
67 // connection simulator uses QuicTime for its internal clock, and as such has
68 // the granularity of 1us, meaning that at bandwidth higher than 20 Mbps the
69 // packets can start to land on the same timestamp.
70 const QuicBandwidth kTestLinkBandwidth =
71 QuicBandwidth::FromKBitsPerSecond(4000);
72 const QuicBandwidth kLocalLinkBandwidth =
73 QuicBandwidth::FromKBitsPerSecond(10000);
74 const QuicTime::Delta kTestPropagationDelay =
75 QuicTime::Delta::FromMilliseconds(30);
76 const QuicTime::Delta kLocalPropagationDelay =
77 QuicTime::Delta::FromMilliseconds(2);
78 const QuicTime::Delta kTestTransferTime =
79 kTestLinkBandwidth.TransferTime(kMaxOutgoingPacketSize) +
80 kLocalLinkBandwidth.TransferTime(kMaxOutgoingPacketSize);
81 const QuicTime::Delta kTestRtt =
82 (kTestPropagationDelay + kLocalPropagationDelay + kTestTransferTime) * 2;
83 const QuicByteCount kTestBdp = kTestRtt * kTestLinkBandwidth;
84
85 class BbrSenderTest : public QuicTest {
86 protected:
BbrSenderTest()87 BbrSenderTest()
88 : simulator_(&random_),
89 bbr_sender_(&simulator_, "BBR sender", "Receiver",
90 Perspective::IS_CLIENT,
91 /*connection_id=*/TestConnectionId(42)),
92 competing_sender_(&simulator_, "Competing sender", "Competing receiver",
93 Perspective::IS_CLIENT,
94 /*connection_id=*/TestConnectionId(43)),
95 receiver_(&simulator_, "Receiver", "BBR sender", Perspective::IS_SERVER,
96 /*connection_id=*/TestConnectionId(42)),
97 competing_receiver_(&simulator_, "Competing receiver",
98 "Competing sender", Perspective::IS_SERVER,
99 /*connection_id=*/TestConnectionId(43)),
100 receiver_multiplexer_("Receiver multiplexer",
101 {&receiver_, &competing_receiver_}) {
102 rtt_stats_ = bbr_sender_.connection()->sent_packet_manager().GetRttStats();
103 const int kTestMaxPacketSize = 1350;
104 bbr_sender_.connection()->SetMaxPacketLength(kTestMaxPacketSize);
105 sender_ = SetupBbrSender(&bbr_sender_);
106 SetConnectionOption(kBBRA);
107 clock_ = simulator_.GetClock();
108 }
109
SetUp()110 void SetUp() override {
111 if (quiche::GetQuicheCommandLineFlag(FLAGS_quic_bbr_test_regression_mode) ==
112 "regress") {
113 SendAlgorithmTestResult expected;
114 ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected));
115 random_seed_ = expected.random_seed();
116 } else {
117 random_seed_ = QuicRandom::GetInstance()->RandUint64();
118 }
119 random_.set_seed(random_seed_);
120 QUIC_LOG(INFO) << "BbrSenderTest simulator set up. Seed: " << random_seed_;
121 }
122
~BbrSenderTest()123 ~BbrSenderTest() {
124 const std::string regression_mode =
125 quiche::GetQuicheCommandLineFlag(FLAGS_quic_bbr_test_regression_mode);
126 const QuicTime::Delta simulated_duration = clock_->Now() - QuicTime::Zero();
127 if (regression_mode == "record") {
128 RecordSendAlgorithmTestResult(random_seed_,
129 simulated_duration.ToMicroseconds());
130 } else if (regression_mode == "regress") {
131 CompareSendAlgorithmTestResult(simulated_duration.ToMicroseconds());
132 }
133 }
134
135 uint64_t random_seed_;
136 SimpleRandom random_;
137 simulator::Simulator simulator_;
138 simulator::QuicEndpoint bbr_sender_;
139 simulator::QuicEndpoint competing_sender_;
140 simulator::QuicEndpoint receiver_;
141 simulator::QuicEndpoint competing_receiver_;
142 simulator::QuicEndpointMultiplexer receiver_multiplexer_;
143 std::unique_ptr<simulator::Switch> switch_;
144 std::unique_ptr<simulator::SymmetricLink> bbr_sender_link_;
145 std::unique_ptr<simulator::SymmetricLink> competing_sender_link_;
146 std::unique_ptr<simulator::SymmetricLink> receiver_link_;
147
148 // Owned by different components of the connection.
149 const QuicClock* clock_;
150 const RttStats* rtt_stats_;
151 BbrSender* sender_;
152
153 // Enables BBR on |endpoint| and returns the associated BBR congestion
154 // controller.
SetupBbrSender(simulator::QuicEndpoint * endpoint)155 BbrSender* SetupBbrSender(simulator::QuicEndpoint* endpoint) {
156 const RttStats* rtt_stats =
157 endpoint->connection()->sent_packet_manager().GetRttStats();
158 // Ownership of the sender will be overtaken by the endpoint.
159 BbrSender* sender = new BbrSender(
160 endpoint->connection()->clock()->Now(), rtt_stats,
161 QuicSentPacketManagerPeer::GetUnackedPacketMap(
162 QuicConnectionPeer::GetSentPacketManager(endpoint->connection())),
163 kInitialCongestionWindowPackets,
164 GetQuicFlag(quic_max_congestion_window), &random_,
165 QuicConnectionPeer::GetStats(endpoint->connection()));
166 QuicConnectionPeer::SetSendAlgorithm(endpoint->connection(), sender);
167 endpoint->RecordTrace();
168 return sender;
169 }
170
171 // Creates a default setup, which is a network with a bottleneck between the
172 // receiver and the switch. The switch has the buffers four times larger than
173 // the bottleneck BDP, which should guarantee a lack of losses.
CreateDefaultSetup()174 void CreateDefaultSetup() {
175 switch_ = std::make_unique<simulator::Switch>(&simulator_, "Switch", 8,
176 2 * kTestBdp);
177 bbr_sender_link_ = std::make_unique<simulator::SymmetricLink>(
178 &bbr_sender_, switch_->port(1), kLocalLinkBandwidth,
179 kLocalPropagationDelay);
180 receiver_link_ = std::make_unique<simulator::SymmetricLink>(
181 &receiver_, switch_->port(2), kTestLinkBandwidth,
182 kTestPropagationDelay);
183 }
184
185 // Same as the default setup, except the buffer now is half of the BDP.
CreateSmallBufferSetup()186 void CreateSmallBufferSetup() {
187 switch_ = std::make_unique<simulator::Switch>(&simulator_, "Switch", 8,
188 0.5 * kTestBdp);
189 bbr_sender_link_ = std::make_unique<simulator::SymmetricLink>(
190 &bbr_sender_, switch_->port(1), kLocalLinkBandwidth,
191 kLocalPropagationDelay);
192 receiver_link_ = std::make_unique<simulator::SymmetricLink>(
193 &receiver_, switch_->port(2), kTestLinkBandwidth,
194 kTestPropagationDelay);
195 }
196
197 // Creates the variation of the default setup in which there is another sender
198 // that competes for the same bottleneck link.
CreateCompetitionSetup()199 void CreateCompetitionSetup() {
200 switch_ = std::make_unique<simulator::Switch>(&simulator_, "Switch", 8,
201 2 * kTestBdp);
202
203 // Add a small offset to the competing link in order to avoid
204 // synchronization effects.
205 const QuicTime::Delta small_offset = QuicTime::Delta::FromMicroseconds(3);
206 bbr_sender_link_ = std::make_unique<simulator::SymmetricLink>(
207 &bbr_sender_, switch_->port(1), kLocalLinkBandwidth,
208 kLocalPropagationDelay);
209 competing_sender_link_ = std::make_unique<simulator::SymmetricLink>(
210 &competing_sender_, switch_->port(3), kLocalLinkBandwidth,
211 kLocalPropagationDelay + small_offset);
212 receiver_link_ = std::make_unique<simulator::SymmetricLink>(
213 &receiver_multiplexer_, switch_->port(2), kTestLinkBandwidth,
214 kTestPropagationDelay);
215 }
216
217 // Creates a BBR vs BBR competition setup.
CreateBbrVsBbrSetup()218 void CreateBbrVsBbrSetup() {
219 SetupBbrSender(&competing_sender_);
220 CreateCompetitionSetup();
221 }
222
EnableAggregation(QuicByteCount aggregation_bytes,QuicTime::Delta aggregation_timeout)223 void EnableAggregation(QuicByteCount aggregation_bytes,
224 QuicTime::Delta aggregation_timeout) {
225 // Enable aggregation on the path from the receiver to the sender.
226 switch_->port_queue(1)->EnableAggregation(aggregation_bytes,
227 aggregation_timeout);
228 }
229
DoSimpleTransfer(QuicByteCount transfer_size,QuicTime::Delta deadline)230 void DoSimpleTransfer(QuicByteCount transfer_size, QuicTime::Delta deadline) {
231 bbr_sender_.AddBytesToTransfer(transfer_size);
232 // TODO(vasilvv): consider rewriting this to run until the receiver actually
233 // receives the intended amount of bytes.
234 bool simulator_result = simulator_.RunUntilOrTimeout(
235 [this]() { return bbr_sender_.bytes_to_transfer() == 0; }, deadline);
236 EXPECT_TRUE(simulator_result)
237 << "Simple transfer failed. Bytes remaining: "
238 << bbr_sender_.bytes_to_transfer();
239 QUIC_LOG(INFO) << "Simple transfer state: " << sender_->ExportDebugState();
240 }
241
242 // Drive the simulator by sending enough data to enter PROBE_BW.
DriveOutOfStartup()243 void DriveOutOfStartup() {
244 ASSERT_FALSE(sender_->ExportDebugState().is_at_full_bandwidth);
245 DoSimpleTransfer(1024 * 1024, QuicTime::Delta::FromSeconds(15));
246 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
247 EXPECT_APPROX_EQ(kTestLinkBandwidth,
248 sender_->ExportDebugState().max_bandwidth, 0.02f);
249 }
250
251 // Send |bytes|-sized bursts of data |number_of_bursts| times, waiting for
252 // |wait_time| between each burst.
SendBursts(size_t number_of_bursts,QuicByteCount bytes,QuicTime::Delta wait_time)253 void SendBursts(size_t number_of_bursts, QuicByteCount bytes,
254 QuicTime::Delta wait_time) {
255 ASSERT_EQ(0u, bbr_sender_.bytes_to_transfer());
256 for (size_t i = 0; i < number_of_bursts; i++) {
257 bbr_sender_.AddBytesToTransfer(bytes);
258
259 // Transfer data and wait for three seconds between each transfer.
260 simulator_.RunFor(wait_time);
261
262 // Ensure the connection did not time out.
263 ASSERT_TRUE(bbr_sender_.connection()->connected());
264 ASSERT_TRUE(receiver_.connection()->connected());
265 }
266
267 simulator_.RunFor(wait_time + kTestRtt);
268 ASSERT_EQ(0u, bbr_sender_.bytes_to_transfer());
269 }
270
SetConnectionOption(QuicTag option)271 void SetConnectionOption(QuicTag option) {
272 QuicConfig config;
273 QuicTagVector options;
274 options.push_back(option);
275 QuicConfigPeer::SetReceivedConnectionOptions(&config, options);
276 sender_->SetFromConfig(config, Perspective::IS_SERVER);
277 }
278 };
279
TEST_F(BbrSenderTest,SetInitialCongestionWindow)280 TEST_F(BbrSenderTest, SetInitialCongestionWindow) {
281 EXPECT_NE(3u * kDefaultTCPMSS, sender_->GetCongestionWindow());
282 sender_->SetInitialCongestionWindowInPackets(3);
283 EXPECT_EQ(3u * kDefaultTCPMSS, sender_->GetCongestionWindow());
284 }
285
286 // Test a simple long data transfer in the default setup.
TEST_F(BbrSenderTest,SimpleTransfer)287 TEST_F(BbrSenderTest, SimpleTransfer) {
288 CreateDefaultSetup();
289
290 // At startup make sure we are at the default.
291 EXPECT_EQ(kDefaultWindowTCP, sender_->GetCongestionWindow());
292 // At startup make sure we can send.
293 EXPECT_TRUE(sender_->CanSend(0));
294 // And that window is un-affected.
295 EXPECT_EQ(kDefaultWindowTCP, sender_->GetCongestionWindow());
296
297 // Verify that Sender is in slow start.
298 EXPECT_TRUE(sender_->InSlowStart());
299
300 // Verify that pacing rate is based on the initial RTT.
301 QuicBandwidth expected_pacing_rate = QuicBandwidth::FromBytesAndTimeDelta(
302 2.885 * kDefaultWindowTCP, rtt_stats_->initial_rtt());
303 EXPECT_APPROX_EQ(expected_pacing_rate.ToBitsPerSecond(),
304 sender_->PacingRate(0).ToBitsPerSecond(), 0.01f);
305
306 ASSERT_GE(kTestBdp, kDefaultWindowTCP + kDefaultTCPMSS);
307
308 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(30));
309 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
310 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
311 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
312
313 // The margin here is quite high, since there exists a possibility that the
314 // connection just exited high gain cycle.
315 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->smoothed_rtt(), 0.2f);
316 }
317
TEST_F(BbrSenderTest,SimpleTransferBBRB)318 TEST_F(BbrSenderTest, SimpleTransferBBRB) {
319 SetConnectionOption(kBBRB);
320 CreateDefaultSetup();
321
322 // At startup make sure we are at the default.
323 EXPECT_EQ(kDefaultWindowTCP, sender_->GetCongestionWindow());
324 // At startup make sure we can send.
325 EXPECT_TRUE(sender_->CanSend(0));
326 // And that window is un-affected.
327 EXPECT_EQ(kDefaultWindowTCP, sender_->GetCongestionWindow());
328
329 // Verify that Sender is in slow start.
330 EXPECT_TRUE(sender_->InSlowStart());
331
332 // Verify that pacing rate is based on the initial RTT.
333 QuicBandwidth expected_pacing_rate = QuicBandwidth::FromBytesAndTimeDelta(
334 2.885 * kDefaultWindowTCP, rtt_stats_->initial_rtt());
335 EXPECT_APPROX_EQ(expected_pacing_rate.ToBitsPerSecond(),
336 sender_->PacingRate(0).ToBitsPerSecond(), 0.01f);
337
338 ASSERT_GE(kTestBdp, kDefaultWindowTCP + kDefaultTCPMSS);
339
340 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(30));
341 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
342 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
343 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
344
345 // The margin here is quite high, since there exists a possibility that the
346 // connection just exited high gain cycle.
347 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->smoothed_rtt(), 0.2f);
348 }
349
350 // Test a simple transfer in a situation when the buffer is less than BDP.
TEST_F(BbrSenderTest,SimpleTransferSmallBuffer)351 TEST_F(BbrSenderTest, SimpleTransferSmallBuffer) {
352 CreateSmallBufferSetup();
353
354 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(30));
355 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
356 EXPECT_APPROX_EQ(kTestLinkBandwidth,
357 sender_->ExportDebugState().max_bandwidth, 0.01f);
358 EXPECT_GE(bbr_sender_.connection()->GetStats().packets_lost, 0u);
359 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
360
361 // The margin here is quite high, since there exists a possibility that the
362 // connection just exited high gain cycle.
363 EXPECT_APPROX_EQ(kTestRtt, sender_->GetMinRtt(), 0.2f);
364 }
365
TEST_F(BbrSenderTest,RemoveBytesLostInRecovery)366 TEST_F(BbrSenderTest, RemoveBytesLostInRecovery) {
367 CreateDefaultSetup();
368
369 DriveOutOfStartup();
370
371 // Drop a packet to enter recovery.
372 receiver_.DropNextIncomingPacket();
373 ASSERT_TRUE(
374 simulator_.RunUntilOrTimeout([this]() { return sender_->InRecovery(); },
375 QuicTime::Delta::FromSeconds(30)));
376
377 QuicUnackedPacketMap* unacked_packets =
378 QuicSentPacketManagerPeer::GetUnackedPacketMap(
379 QuicConnectionPeer::GetSentPacketManager(bbr_sender_.connection()));
380 QuicPacketNumber largest_sent =
381 bbr_sender_.connection()->sent_packet_manager().GetLargestSentPacket();
382 // least_inflight is the smallest inflight packet.
383 QuicPacketNumber least_inflight =
384 bbr_sender_.connection()->sent_packet_manager().GetLeastUnacked();
385 while (!unacked_packets->GetTransmissionInfo(least_inflight).in_flight) {
386 ASSERT_LE(least_inflight, largest_sent);
387 least_inflight++;
388 }
389 QuicPacketLength least_inflight_packet_size =
390 unacked_packets->GetTransmissionInfo(least_inflight).bytes_sent;
391 QuicByteCount prior_recovery_window =
392 sender_->ExportDebugState().recovery_window;
393 QuicByteCount prior_inflight = unacked_packets->bytes_in_flight();
394 QUIC_LOG(INFO) << "Recovery window:" << prior_recovery_window
395 << ", least_inflight_packet_size:"
396 << least_inflight_packet_size
397 << ", bytes_in_flight:" << prior_inflight;
398 ASSERT_GT(prior_recovery_window, least_inflight_packet_size);
399
400 // Lose the least inflight packet and expect the recovery window to drop.
401 unacked_packets->RemoveFromInFlight(least_inflight);
402 LostPacketVector lost_packets;
403 lost_packets.emplace_back(least_inflight, least_inflight_packet_size);
404 sender_->OnCongestionEvent(false, prior_inflight, clock_->Now(), {},
405 lost_packets, 0, 0);
406 EXPECT_EQ(sender_->ExportDebugState().recovery_window,
407 prior_inflight - least_inflight_packet_size);
408 EXPECT_LT(sender_->ExportDebugState().recovery_window, prior_recovery_window);
409 }
410
411 // Test a simple long data transfer with 2 rtts of aggregation.
TEST_F(BbrSenderTest,SimpleTransfer2RTTAggregationBytes)412 TEST_F(BbrSenderTest, SimpleTransfer2RTTAggregationBytes) {
413 SetConnectionOption(kBSAO);
414 CreateDefaultSetup();
415 // 2 RTTs of aggregation, with a max of 10kb.
416 EnableAggregation(10 * 1024, 2 * kTestRtt);
417
418 // Transfer 12MB.
419 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(35));
420 EXPECT_TRUE(sender_->ExportDebugState().mode == BbrSender::PROBE_BW ||
421 sender_->ExportDebugState().mode == BbrSender::PROBE_RTT);
422
423 EXPECT_APPROX_EQ(kTestLinkBandwidth,
424 sender_->ExportDebugState().max_bandwidth, 0.01f);
425
426 // The margin here is high, because the aggregation greatly increases
427 // smoothed rtt.
428 EXPECT_GE(kTestRtt * 4, rtt_stats_->smoothed_rtt());
429 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->min_rtt(), 0.5f);
430 }
431
432 // Test a simple long data transfer with 2 rtts of aggregation.
TEST_F(BbrSenderTest,SimpleTransferAckDecimation)433 TEST_F(BbrSenderTest, SimpleTransferAckDecimation) {
434 SetConnectionOption(kBSAO);
435 // Decrease the CWND gain so extra CWND is required with stretch acks.
436 SetQuicFlag(quic_bbr_cwnd_gain, 1.0);
437 sender_ = new BbrSender(
438 bbr_sender_.connection()->clock()->Now(), rtt_stats_,
439 QuicSentPacketManagerPeer::GetUnackedPacketMap(
440 QuicConnectionPeer::GetSentPacketManager(bbr_sender_.connection())),
441 kInitialCongestionWindowPackets, GetQuicFlag(quic_max_congestion_window),
442 &random_, QuicConnectionPeer::GetStats(bbr_sender_.connection()));
443 QuicConnectionPeer::SetSendAlgorithm(bbr_sender_.connection(), sender_);
444 CreateDefaultSetup();
445
446 // Transfer 12MB.
447 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(35));
448 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
449
450 EXPECT_APPROX_EQ(kTestLinkBandwidth,
451 sender_->ExportDebugState().max_bandwidth, 0.01f);
452
453 // TODO(ianswett): Expect 0 packets are lost once BBR no longer measures
454 // bandwidth higher than the link rate.
455 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
456 // The margin here is high, because the aggregation greatly increases
457 // smoothed rtt.
458 EXPECT_GE(kTestRtt * 2, rtt_stats_->smoothed_rtt());
459 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->min_rtt(), 0.1f);
460 }
461
462 // Test a simple long data transfer with 2 rtts of aggregation.
463 // TODO(b/172302465) Re-enable this test.
TEST_F(BbrSenderTest,QUIC_TEST_DISABLED_IN_CHROME (SimpleTransfer2RTTAggregationBytes20RTTWindow))464 TEST_F(BbrSenderTest, QUIC_TEST_DISABLED_IN_CHROME(
465 SimpleTransfer2RTTAggregationBytes20RTTWindow)) {
466 SetConnectionOption(kBSAO);
467 CreateDefaultSetup();
468 SetConnectionOption(kBBR4);
469 // 2 RTTs of aggregation, with a max of 10kb.
470 EnableAggregation(10 * 1024, 2 * kTestRtt);
471
472 // Transfer 12MB.
473 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(35));
474 EXPECT_TRUE(sender_->ExportDebugState().mode == BbrSender::PROBE_BW ||
475 sender_->ExportDebugState().mode == BbrSender::PROBE_RTT);
476
477 EXPECT_APPROX_EQ(kTestLinkBandwidth,
478 sender_->ExportDebugState().max_bandwidth, 0.01f);
479
480 // TODO(ianswett): Expect 0 packets are lost once BBR no longer measures
481 // bandwidth higher than the link rate.
482 // The margin here is high, because the aggregation greatly increases
483 // smoothed rtt.
484 EXPECT_GE(kTestRtt * 4, rtt_stats_->smoothed_rtt());
485 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->min_rtt(), 0.25f);
486 }
487
488 // Test a simple long data transfer with 2 rtts of aggregation.
TEST_F(BbrSenderTest,SimpleTransfer2RTTAggregationBytes40RTTWindow)489 TEST_F(BbrSenderTest, SimpleTransfer2RTTAggregationBytes40RTTWindow) {
490 SetConnectionOption(kBSAO);
491 CreateDefaultSetup();
492 SetConnectionOption(kBBR5);
493 // 2 RTTs of aggregation, with a max of 10kb.
494 EnableAggregation(10 * 1024, 2 * kTestRtt);
495
496 // Transfer 12MB.
497 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(35));
498 EXPECT_TRUE(sender_->ExportDebugState().mode == BbrSender::PROBE_BW ||
499 sender_->ExportDebugState().mode == BbrSender::PROBE_RTT);
500
501 EXPECT_APPROX_EQ(kTestLinkBandwidth,
502 sender_->ExportDebugState().max_bandwidth, 0.01f);
503
504 // TODO(ianswett): Expect 0 packets are lost once BBR no longer measures
505 // bandwidth higher than the link rate.
506 // The margin here is high, because the aggregation greatly increases
507 // smoothed rtt.
508 EXPECT_GE(kTestRtt * 4, rtt_stats_->smoothed_rtt());
509 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->min_rtt(), 0.25f);
510 }
511
512 // Test the number of losses incurred by the startup phase in a situation when
513 // the buffer is less than BDP.
TEST_F(BbrSenderTest,PacketLossOnSmallBufferStartup)514 TEST_F(BbrSenderTest, PacketLossOnSmallBufferStartup) {
515 CreateSmallBufferSetup();
516
517 DriveOutOfStartup();
518 float loss_rate =
519 static_cast<float>(bbr_sender_.connection()->GetStats().packets_lost) /
520 bbr_sender_.connection()->GetStats().packets_sent;
521 EXPECT_LE(loss_rate, 0.31);
522 }
523
524 // Test the number of losses incurred by the startup phase in a situation when
525 // the buffer is less than BDP, with a STARTUP CWND gain of 2.
TEST_F(BbrSenderTest,PacketLossOnSmallBufferStartupDerivedCWNDGain)526 TEST_F(BbrSenderTest, PacketLossOnSmallBufferStartupDerivedCWNDGain) {
527 CreateSmallBufferSetup();
528
529 SetConnectionOption(kBBQ2);
530 DriveOutOfStartup();
531 float loss_rate =
532 static_cast<float>(bbr_sender_.connection()->GetStats().packets_lost) /
533 bbr_sender_.connection()->GetStats().packets_sent;
534 EXPECT_LE(loss_rate, 0.1);
535 }
536
537 // Ensures the code transitions loss recovery states correctly (NOT_IN_RECOVERY
538 // -> CONSERVATION -> GROWTH -> NOT_IN_RECOVERY).
TEST_F(BbrSenderTest,RecoveryStates)539 TEST_F(BbrSenderTest, RecoveryStates) {
540 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(10);
541 bool simulator_result;
542 CreateSmallBufferSetup();
543
544 bbr_sender_.AddBytesToTransfer(100 * 1024 * 1024);
545 ASSERT_EQ(BbrSender::NOT_IN_RECOVERY,
546 sender_->ExportDebugState().recovery_state);
547
548 simulator_result = simulator_.RunUntilOrTimeout(
549 [this]() {
550 return sender_->ExportDebugState().recovery_state !=
551 BbrSender::NOT_IN_RECOVERY;
552 },
553 timeout);
554 ASSERT_TRUE(simulator_result);
555 ASSERT_EQ(BbrSender::CONSERVATION,
556 sender_->ExportDebugState().recovery_state);
557
558 simulator_result = simulator_.RunUntilOrTimeout(
559 [this]() {
560 return sender_->ExportDebugState().recovery_state !=
561 BbrSender::CONSERVATION;
562 },
563 timeout);
564 ASSERT_TRUE(simulator_result);
565 ASSERT_EQ(BbrSender::GROWTH, sender_->ExportDebugState().recovery_state);
566
567 simulator_result = simulator_.RunUntilOrTimeout(
568 [this]() {
569 return sender_->ExportDebugState().recovery_state != BbrSender::GROWTH;
570 },
571 timeout);
572
573 ASSERT_EQ(BbrSender::NOT_IN_RECOVERY,
574 sender_->ExportDebugState().recovery_state);
575 ASSERT_TRUE(simulator_result);
576 }
577
578 // Verify the behavior of the algorithm in the case when the connection sends
579 // small bursts of data after sending continuously for a while.
TEST_F(BbrSenderTest,ApplicationLimitedBursts)580 TEST_F(BbrSenderTest, ApplicationLimitedBursts) {
581 CreateDefaultSetup();
582 EXPECT_FALSE(sender_->HasGoodBandwidthEstimateForResumption());
583
584 DriveOutOfStartup();
585 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
586 EXPECT_TRUE(sender_->HasGoodBandwidthEstimateForResumption());
587
588 SendBursts(20, 512, QuicTime::Delta::FromSeconds(3));
589 EXPECT_TRUE(sender_->ExportDebugState().last_sample_is_app_limited);
590 EXPECT_TRUE(sender_->HasGoodBandwidthEstimateForResumption());
591 EXPECT_APPROX_EQ(kTestLinkBandwidth,
592 sender_->ExportDebugState().max_bandwidth, 0.01f);
593 }
594
595 // Verify the behavior of the algorithm in the case when the connection sends
596 // small bursts of data and then starts sending continuously.
TEST_F(BbrSenderTest,ApplicationLimitedBurstsWithoutPrior)597 TEST_F(BbrSenderTest, ApplicationLimitedBurstsWithoutPrior) {
598 CreateDefaultSetup();
599
600 SendBursts(40, 512, QuicTime::Delta::FromSeconds(3));
601 EXPECT_TRUE(sender_->ExportDebugState().last_sample_is_app_limited);
602
603 DriveOutOfStartup();
604 EXPECT_APPROX_EQ(kTestLinkBandwidth,
605 sender_->ExportDebugState().max_bandwidth, 0.01f);
606 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
607 }
608
609 // Verify that the DRAIN phase works correctly.
TEST_F(BbrSenderTest,Drain)610 TEST_F(BbrSenderTest, Drain) {
611 CreateDefaultSetup();
612 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(10);
613 // Get the queue at the bottleneck, which is the outgoing queue at the port to
614 // which the receiver is connected.
615 const simulator::Queue* queue = switch_->port_queue(2);
616 bool simulator_result;
617
618 // We have no intention of ever finishing this transfer.
619 bbr_sender_.AddBytesToTransfer(100 * 1024 * 1024);
620
621 // Run the startup, and verify that it fills up the queue.
622 ASSERT_EQ(BbrSender::STARTUP, sender_->ExportDebugState().mode);
623 simulator_result = simulator_.RunUntilOrTimeout(
624 [this]() {
625 return sender_->ExportDebugState().mode != BbrSender::STARTUP;
626 },
627 timeout);
628 ASSERT_TRUE(simulator_result);
629 ASSERT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
630 EXPECT_APPROX_EQ(sender_->BandwidthEstimate() * (1 / 2.885f),
631 sender_->PacingRate(0), 0.01f);
632
633 // BBR uses CWND gain of 2 during STARTUP, hence it will fill the buffer
634 // with approximately 1 BDP. Here, we use 0.8 to give some margin for
635 // error.
636 EXPECT_GE(queue->bytes_queued(), 0.8 * kTestBdp);
637
638 // Observe increased RTT due to bufferbloat.
639 const QuicTime::Delta queueing_delay =
640 kTestLinkBandwidth.TransferTime(queue->bytes_queued());
641 EXPECT_APPROX_EQ(kTestRtt + queueing_delay, rtt_stats_->latest_rtt(), 0.1f);
642
643 // Transition to the drain phase and verify that it makes the queue
644 // have at most a BDP worth of packets.
645 simulator_result = simulator_.RunUntilOrTimeout(
646 [this]() { return sender_->ExportDebugState().mode != BbrSender::DRAIN; },
647 timeout);
648 ASSERT_TRUE(simulator_result);
649 ASSERT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
650 EXPECT_LE(queue->bytes_queued(), kTestBdp);
651
652 // Wait for a few round trips and ensure we're in appropriate phase of gain
653 // cycling before taking an RTT measurement.
654 const QuicRoundTripCount start_round_trip =
655 sender_->ExportDebugState().round_trip_count;
656 simulator_result = simulator_.RunUntilOrTimeout(
657 [this, start_round_trip]() {
658 QuicRoundTripCount rounds_passed =
659 sender_->ExportDebugState().round_trip_count - start_round_trip;
660 return rounds_passed >= 4 &&
661 sender_->ExportDebugState().gain_cycle_index == 7;
662 },
663 timeout);
664 ASSERT_TRUE(simulator_result);
665
666 // Observe the bufferbloat go away.
667 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->smoothed_rtt(), 0.1f);
668 }
669
670 // TODO(wub): Re-enable this test once default drain_gain changed to 0.75.
671 // Verify that the DRAIN phase works correctly.
TEST_F(BbrSenderTest,DISABLED_ShallowDrain)672 TEST_F(BbrSenderTest, DISABLED_ShallowDrain) {
673 CreateDefaultSetup();
674 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(10);
675 // Get the queue at the bottleneck, which is the outgoing queue at the port to
676 // which the receiver is connected.
677 const simulator::Queue* queue = switch_->port_queue(2);
678 bool simulator_result;
679
680 // We have no intention of ever finishing this transfer.
681 bbr_sender_.AddBytesToTransfer(100 * 1024 * 1024);
682
683 // Run the startup, and verify that it fills up the queue.
684 ASSERT_EQ(BbrSender::STARTUP, sender_->ExportDebugState().mode);
685 simulator_result = simulator_.RunUntilOrTimeout(
686 [this]() {
687 return sender_->ExportDebugState().mode != BbrSender::STARTUP;
688 },
689 timeout);
690 ASSERT_TRUE(simulator_result);
691 ASSERT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
692 EXPECT_EQ(0.75 * sender_->BandwidthEstimate(), sender_->PacingRate(0));
693 // BBR uses CWND gain of 2.88 during STARTUP, hence it will fill the buffer
694 // with approximately 1.88 BDPs. Here, we use 1.5 to give some margin for
695 // error.
696 EXPECT_GE(queue->bytes_queued(), 1.5 * kTestBdp);
697
698 // Observe increased RTT due to bufferbloat.
699 const QuicTime::Delta queueing_delay =
700 kTestLinkBandwidth.TransferTime(queue->bytes_queued());
701 EXPECT_APPROX_EQ(kTestRtt + queueing_delay, rtt_stats_->latest_rtt(), 0.1f);
702
703 // Transition to the drain phase and verify that it makes the queue
704 // have at most a BDP worth of packets.
705 simulator_result = simulator_.RunUntilOrTimeout(
706 [this]() { return sender_->ExportDebugState().mode != BbrSender::DRAIN; },
707 timeout);
708 ASSERT_TRUE(simulator_result);
709 ASSERT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
710 EXPECT_LE(queue->bytes_queued(), kTestBdp);
711
712 // Wait for a few round trips and ensure we're in appropriate phase of gain
713 // cycling before taking an RTT measurement.
714 const QuicRoundTripCount start_round_trip =
715 sender_->ExportDebugState().round_trip_count;
716 simulator_result = simulator_.RunUntilOrTimeout(
717 [this, start_round_trip]() {
718 QuicRoundTripCount rounds_passed =
719 sender_->ExportDebugState().round_trip_count - start_round_trip;
720 return rounds_passed >= 4 &&
721 sender_->ExportDebugState().gain_cycle_index == 7;
722 },
723 timeout);
724 ASSERT_TRUE(simulator_result);
725
726 // Observe the bufferbloat go away.
727 EXPECT_APPROX_EQ(kTestRtt, rtt_stats_->smoothed_rtt(), 0.1f);
728 }
729
730 // Verify that the connection enters and exits PROBE_RTT correctly.
TEST_F(BbrSenderTest,ProbeRtt)731 TEST_F(BbrSenderTest, ProbeRtt) {
732 CreateDefaultSetup();
733 DriveOutOfStartup();
734
735 // We have no intention of ever finishing this transfer.
736 bbr_sender_.AddBytesToTransfer(100 * 1024 * 1024);
737
738 // Wait until the connection enters PROBE_RTT.
739 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(12);
740 bool simulator_result = simulator_.RunUntilOrTimeout(
741 [this]() {
742 return sender_->ExportDebugState().mode == BbrSender::PROBE_RTT;
743 },
744 timeout);
745 ASSERT_TRUE(simulator_result);
746 ASSERT_EQ(BbrSender::PROBE_RTT, sender_->ExportDebugState().mode);
747
748 // Exit PROBE_RTT.
749 const QuicTime probe_rtt_start = clock_->Now();
750 const QuicTime::Delta time_to_exit_probe_rtt =
751 kTestRtt + QuicTime::Delta::FromMilliseconds(200);
752 simulator_.RunFor(1.5 * time_to_exit_probe_rtt);
753 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
754 EXPECT_GE(sender_->ExportDebugState().min_rtt_timestamp, probe_rtt_start);
755 }
756
757 // Ensure that a connection that is app-limited and is at sufficiently low
758 // bandwidth will not exit high gain phase, and similarly ensure that the
759 // connection will exit low gain early if the number of bytes in flight is low.
760 // TODO(crbug.com/1145095): Re-enable this test.
TEST_F(BbrSenderTest,QUIC_TEST_DISABLED_IN_CHROME (InFlightAwareGainCycling))761 TEST_F(BbrSenderTest, QUIC_TEST_DISABLED_IN_CHROME(InFlightAwareGainCycling)) {
762 CreateDefaultSetup();
763 DriveOutOfStartup();
764
765 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
766 while (!(sender_->ExportDebugState().gain_cycle_index >= 4 &&
767 bbr_sender_.bytes_to_transfer() == 0)) {
768 bbr_sender_.AddBytesToTransfer(kTestLinkBandwidth.ToBytesPerSecond());
769 ASSERT_TRUE(simulator_.RunUntilOrTimeout(
770 [this]() { return bbr_sender_.bytes_to_transfer() == 0; }, timeout));
771 }
772
773 // Send at 10% of available rate. Run for 3 seconds, checking in the middle
774 // and at the end. The pacing gain should be high throughout.
775 QuicBandwidth target_bandwidth = 0.1f * kTestLinkBandwidth;
776 QuicTime::Delta burst_interval = QuicTime::Delta::FromMilliseconds(300);
777 for (int i = 0; i < 2; i++) {
778 SendBursts(5, target_bandwidth * burst_interval, burst_interval);
779 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
780 EXPECT_EQ(0, sender_->ExportDebugState().gain_cycle_index);
781 EXPECT_APPROX_EQ(kTestLinkBandwidth,
782 sender_->ExportDebugState().max_bandwidth, 0.02f);
783 }
784
785 // Now that in-flight is almost zero and the pacing gain is still above 1,
786 // send approximately 1.25 BDPs worth of data. This should cause the
787 // PROBE_BW mode to enter low gain cycle, and exit it earlier than one min_rtt
788 // due to running out of data to send.
789 bbr_sender_.AddBytesToTransfer(1.3 * kTestBdp);
790 ASSERT_TRUE(simulator_.RunUntilOrTimeout(
791 [this]() { return sender_->ExportDebugState().gain_cycle_index == 1; },
792 timeout));
793
794 simulator_.RunFor(0.75 * sender_->ExportDebugState().min_rtt);
795 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
796 EXPECT_EQ(2, sender_->ExportDebugState().gain_cycle_index);
797 }
798
799 // Ensure that the pacing rate does not drop at startup.
TEST_F(BbrSenderTest,NoBandwidthDropOnStartup)800 TEST_F(BbrSenderTest, NoBandwidthDropOnStartup) {
801 CreateDefaultSetup();
802
803 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
804 bool simulator_result;
805
806 QuicBandwidth initial_rate = QuicBandwidth::FromBytesAndTimeDelta(
807 kInitialCongestionWindowPackets * kDefaultTCPMSS,
808 rtt_stats_->initial_rtt());
809 EXPECT_GE(sender_->PacingRate(0), initial_rate);
810
811 // Send a packet.
812 bbr_sender_.AddBytesToTransfer(1000);
813 simulator_result = simulator_.RunUntilOrTimeout(
814 [this]() { return receiver_.bytes_received() == 1000; }, timeout);
815 ASSERT_TRUE(simulator_result);
816 EXPECT_GE(sender_->PacingRate(0), initial_rate);
817
818 // Wait for a while.
819 simulator_.RunFor(QuicTime::Delta::FromSeconds(2));
820 EXPECT_GE(sender_->PacingRate(0), initial_rate);
821
822 // Send another packet.
823 bbr_sender_.AddBytesToTransfer(1000);
824 simulator_result = simulator_.RunUntilOrTimeout(
825 [this]() { return receiver_.bytes_received() == 2000; }, timeout);
826 ASSERT_TRUE(simulator_result);
827 EXPECT_GE(sender_->PacingRate(0), initial_rate);
828 }
829
830 // Test exiting STARTUP earlier due to the 1RTT connection option.
TEST_F(BbrSenderTest,SimpleTransfer1RTTStartup)831 TEST_F(BbrSenderTest, SimpleTransfer1RTTStartup) {
832 CreateDefaultSetup();
833
834 SetConnectionOption(k1RTT);
835 EXPECT_EQ(1u, sender_->num_startup_rtts());
836
837 // Run until the full bandwidth is reached and check how many rounds it was.
838 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
839 QuicRoundTripCount max_bw_round = 0;
840 QuicBandwidth max_bw(QuicBandwidth::Zero());
841 bool simulator_result = simulator_.RunUntilOrTimeout(
842 [this, &max_bw, &max_bw_round]() {
843 if (max_bw < sender_->ExportDebugState().max_bandwidth) {
844 max_bw = sender_->ExportDebugState().max_bandwidth;
845 max_bw_round = sender_->ExportDebugState().round_trip_count;
846 }
847 return sender_->ExportDebugState().is_at_full_bandwidth;
848 },
849 QuicTime::Delta::FromSeconds(5));
850 ASSERT_TRUE(simulator_result);
851 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
852 EXPECT_EQ(1u, sender_->ExportDebugState().round_trip_count - max_bw_round);
853 EXPECT_EQ(1u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
854 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
855 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
856 }
857
858 // Test exiting STARTUP earlier due to the 2RTT connection option.
TEST_F(BbrSenderTest,SimpleTransfer2RTTStartup)859 TEST_F(BbrSenderTest, SimpleTransfer2RTTStartup) {
860 CreateDefaultSetup();
861
862 SetConnectionOption(k2RTT);
863 EXPECT_EQ(2u, sender_->num_startup_rtts());
864
865 // Run until the full bandwidth is reached and check how many rounds it was.
866 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
867 QuicRoundTripCount max_bw_round = 0;
868 QuicBandwidth max_bw(QuicBandwidth::Zero());
869 bool simulator_result = simulator_.RunUntilOrTimeout(
870 [this, &max_bw, &max_bw_round]() {
871 if (max_bw * 1.001 < sender_->ExportDebugState().max_bandwidth) {
872 max_bw = sender_->ExportDebugState().max_bandwidth;
873 max_bw_round = sender_->ExportDebugState().round_trip_count;
874 }
875 return sender_->ExportDebugState().is_at_full_bandwidth;
876 },
877 QuicTime::Delta::FromSeconds(5));
878 ASSERT_TRUE(simulator_result);
879 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
880 EXPECT_EQ(2u, sender_->ExportDebugState().round_trip_count - max_bw_round);
881 EXPECT_EQ(2u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
882 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
883 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
884 }
885
886 // Test exiting STARTUP earlier upon loss.
TEST_F(BbrSenderTest,SimpleTransferExitStartupOnLoss)887 TEST_F(BbrSenderTest, SimpleTransferExitStartupOnLoss) {
888 CreateDefaultSetup();
889
890 EXPECT_EQ(3u, sender_->num_startup_rtts());
891
892 // Run until the full bandwidth is reached and check how many rounds it was.
893 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
894 QuicRoundTripCount max_bw_round = 0;
895 QuicBandwidth max_bw(QuicBandwidth::Zero());
896 bool simulator_result = simulator_.RunUntilOrTimeout(
897 [this, &max_bw, &max_bw_round]() {
898 if (max_bw * 1.001 < sender_->ExportDebugState().max_bandwidth) {
899 max_bw = sender_->ExportDebugState().max_bandwidth;
900 max_bw_round = sender_->ExportDebugState().round_trip_count;
901 }
902 return sender_->ExportDebugState().is_at_full_bandwidth;
903 },
904 QuicTime::Delta::FromSeconds(5));
905 ASSERT_TRUE(simulator_result);
906 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
907 EXPECT_EQ(3u, sender_->ExportDebugState().round_trip_count - max_bw_round);
908 EXPECT_EQ(3u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
909 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
910 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
911 }
912
913 // Test exiting STARTUP earlier upon loss with a small buffer.
TEST_F(BbrSenderTest,SimpleTransferExitStartupOnLossSmallBuffer)914 TEST_F(BbrSenderTest, SimpleTransferExitStartupOnLossSmallBuffer) {
915 CreateSmallBufferSetup();
916
917 EXPECT_EQ(3u, sender_->num_startup_rtts());
918
919 // Run until the full bandwidth is reached and check how many rounds it was.
920 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
921 QuicRoundTripCount max_bw_round = 0;
922 QuicBandwidth max_bw(QuicBandwidth::Zero());
923 bool simulator_result = simulator_.RunUntilOrTimeout(
924 [this, &max_bw, &max_bw_round]() {
925 if (max_bw < sender_->ExportDebugState().max_bandwidth) {
926 max_bw = sender_->ExportDebugState().max_bandwidth;
927 max_bw_round = sender_->ExportDebugState().round_trip_count;
928 }
929 return sender_->ExportDebugState().is_at_full_bandwidth;
930 },
931 QuicTime::Delta::FromSeconds(5));
932 ASSERT_TRUE(simulator_result);
933 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
934 EXPECT_GE(2u, sender_->ExportDebugState().round_trip_count - max_bw_round);
935 EXPECT_EQ(1u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
936 EXPECT_NE(0u, bbr_sender_.connection()->GetStats().packets_lost);
937 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
938 }
939
TEST_F(BbrSenderTest,DerivedPacingGainStartup)940 TEST_F(BbrSenderTest, DerivedPacingGainStartup) {
941 CreateDefaultSetup();
942
943 SetConnectionOption(kBBQ1);
944 EXPECT_EQ(3u, sender_->num_startup_rtts());
945 // Verify that Sender is in slow start.
946 EXPECT_TRUE(sender_->InSlowStart());
947 // Verify that pacing rate is based on the initial RTT.
948 QuicBandwidth expected_pacing_rate = QuicBandwidth::FromBytesAndTimeDelta(
949 2.773 * kDefaultWindowTCP, rtt_stats_->initial_rtt());
950 EXPECT_APPROX_EQ(expected_pacing_rate.ToBitsPerSecond(),
951 sender_->PacingRate(0).ToBitsPerSecond(), 0.01f);
952
953 // Run until the full bandwidth is reached and check how many rounds it was.
954 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
955 bool simulator_result = simulator_.RunUntilOrTimeout(
956 [this]() { return sender_->ExportDebugState().is_at_full_bandwidth; },
957 QuicTime::Delta::FromSeconds(5));
958 ASSERT_TRUE(simulator_result);
959 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
960 EXPECT_EQ(3u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
961 EXPECT_APPROX_EQ(kTestLinkBandwidth,
962 sender_->ExportDebugState().max_bandwidth, 0.01f);
963 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
964 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
965 }
966
TEST_F(BbrSenderTest,DerivedCWNDGainStartup)967 TEST_F(BbrSenderTest, DerivedCWNDGainStartup) {
968 CreateSmallBufferSetup();
969
970 EXPECT_EQ(3u, sender_->num_startup_rtts());
971 // Verify that Sender is in slow start.
972 EXPECT_TRUE(sender_->InSlowStart());
973 // Verify that pacing rate is based on the initial RTT.
974 QuicBandwidth expected_pacing_rate = QuicBandwidth::FromBytesAndTimeDelta(
975 2.885 * kDefaultWindowTCP, rtt_stats_->initial_rtt());
976 EXPECT_APPROX_EQ(expected_pacing_rate.ToBitsPerSecond(),
977 sender_->PacingRate(0).ToBitsPerSecond(), 0.01f);
978
979 // Run until the full bandwidth is reached and check how many rounds it was.
980 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
981 bool simulator_result = simulator_.RunUntilOrTimeout(
982 [this]() { return sender_->ExportDebugState().is_at_full_bandwidth; },
983 QuicTime::Delta::FromSeconds(5));
984 ASSERT_TRUE(simulator_result);
985 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
986 if (!bbr_sender_.connection()->GetStats().bbr_exit_startup_due_to_loss) {
987 EXPECT_EQ(3u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
988 }
989 EXPECT_APPROX_EQ(kTestLinkBandwidth,
990 sender_->ExportDebugState().max_bandwidth, 0.01f);
991 float loss_rate =
992 static_cast<float>(bbr_sender_.connection()->GetStats().packets_lost) /
993 bbr_sender_.connection()->GetStats().packets_sent;
994 EXPECT_LT(loss_rate, 0.15f);
995 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
996 // Expect an SRTT less than 2.7 * Min RTT on exit from STARTUP.
997 EXPECT_GT(kTestRtt * 2.7, rtt_stats_->smoothed_rtt());
998 }
999
TEST_F(BbrSenderTest,AckAggregationInStartup)1000 TEST_F(BbrSenderTest, AckAggregationInStartup) {
1001 CreateDefaultSetup();
1002
1003 SetConnectionOption(kBBQ3);
1004 EXPECT_EQ(3u, sender_->num_startup_rtts());
1005 // Verify that Sender is in slow start.
1006 EXPECT_TRUE(sender_->InSlowStart());
1007 // Verify that pacing rate is based on the initial RTT.
1008 QuicBandwidth expected_pacing_rate = QuicBandwidth::FromBytesAndTimeDelta(
1009 2.885 * kDefaultWindowTCP, rtt_stats_->initial_rtt());
1010 EXPECT_APPROX_EQ(expected_pacing_rate.ToBitsPerSecond(),
1011 sender_->PacingRate(0).ToBitsPerSecond(), 0.01f);
1012
1013 // Run until the full bandwidth is reached and check how many rounds it was.
1014 bbr_sender_.AddBytesToTransfer(12 * 1024 * 1024);
1015 bool simulator_result = simulator_.RunUntilOrTimeout(
1016 [this]() { return sender_->ExportDebugState().is_at_full_bandwidth; },
1017 QuicTime::Delta::FromSeconds(5));
1018 ASSERT_TRUE(simulator_result);
1019 EXPECT_EQ(BbrSender::DRAIN, sender_->ExportDebugState().mode);
1020 EXPECT_EQ(3u, sender_->ExportDebugState().rounds_without_bandwidth_gain);
1021 EXPECT_APPROX_EQ(kTestLinkBandwidth,
1022 sender_->ExportDebugState().max_bandwidth, 0.01f);
1023 EXPECT_EQ(0u, bbr_sender_.connection()->GetStats().packets_lost);
1024 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
1025 }
1026
1027 // Test that two BBR flows started slightly apart from each other terminate.
TEST_F(BbrSenderTest,SimpleCompetition)1028 TEST_F(BbrSenderTest, SimpleCompetition) {
1029 const QuicByteCount transfer_size = 10 * 1024 * 1024;
1030 const QuicTime::Delta transfer_time =
1031 kTestLinkBandwidth.TransferTime(transfer_size);
1032 CreateBbrVsBbrSetup();
1033
1034 // Transfer 10% of data in first transfer.
1035 bbr_sender_.AddBytesToTransfer(transfer_size);
1036 bool simulator_result = simulator_.RunUntilOrTimeout(
1037 [this]() { return receiver_.bytes_received() >= 0.1 * transfer_size; },
1038 transfer_time);
1039 ASSERT_TRUE(simulator_result);
1040
1041 // Start the second transfer and wait until both finish.
1042 competing_sender_.AddBytesToTransfer(transfer_size);
1043 simulator_result = simulator_.RunUntilOrTimeout(
1044 [this]() {
1045 return receiver_.bytes_received() == transfer_size &&
1046 competing_receiver_.bytes_received() == transfer_size;
1047 },
1048 3 * transfer_time);
1049 ASSERT_TRUE(simulator_result);
1050 }
1051
1052 // Test that BBR can resume bandwidth from cached network parameters.
TEST_F(BbrSenderTest,ResumeConnectionState)1053 TEST_F(BbrSenderTest, ResumeConnectionState) {
1054 CreateDefaultSetup();
1055
1056 bbr_sender_.connection()->AdjustNetworkParameters(
1057 SendAlgorithmInterface::NetworkParams(kTestLinkBandwidth, kTestRtt,
1058 false));
1059 EXPECT_EQ(kTestLinkBandwidth * kTestRtt,
1060 sender_->ExportDebugState().congestion_window);
1061
1062 EXPECT_EQ(kTestLinkBandwidth, sender_->PacingRate(/*bytes_in_flight=*/0));
1063
1064 EXPECT_APPROX_EQ(kTestRtt, sender_->ExportDebugState().min_rtt, 0.01f);
1065
1066 DriveOutOfStartup();
1067 }
1068
1069 // Test with a min CWND of 1 instead of 4 packets.
TEST_F(BbrSenderTest,ProbeRTTMinCWND1)1070 TEST_F(BbrSenderTest, ProbeRTTMinCWND1) {
1071 CreateDefaultSetup();
1072 SetConnectionOption(kMIN1);
1073 DriveOutOfStartup();
1074
1075 // We have no intention of ever finishing this transfer.
1076 bbr_sender_.AddBytesToTransfer(100 * 1024 * 1024);
1077
1078 // Wait until the connection enters PROBE_RTT.
1079 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(12);
1080 bool simulator_result = simulator_.RunUntilOrTimeout(
1081 [this]() {
1082 return sender_->ExportDebugState().mode == BbrSender::PROBE_RTT;
1083 },
1084 timeout);
1085 ASSERT_TRUE(simulator_result);
1086 ASSERT_EQ(BbrSender::PROBE_RTT, sender_->ExportDebugState().mode);
1087 // The PROBE_RTT CWND should be 1 if the min CWND is 1.
1088 EXPECT_EQ(kDefaultTCPMSS, sender_->GetCongestionWindow());
1089
1090 // Exit PROBE_RTT.
1091 const QuicTime probe_rtt_start = clock_->Now();
1092 const QuicTime::Delta time_to_exit_probe_rtt =
1093 kTestRtt + QuicTime::Delta::FromMilliseconds(200);
1094 simulator_.RunFor(1.5 * time_to_exit_probe_rtt);
1095 EXPECT_EQ(BbrSender::PROBE_BW, sender_->ExportDebugState().mode);
1096 EXPECT_GE(sender_->ExportDebugState().min_rtt_timestamp, probe_rtt_start);
1097 }
1098
TEST_F(BbrSenderTest,StartupStats)1099 TEST_F(BbrSenderTest, StartupStats) {
1100 CreateDefaultSetup();
1101
1102 DriveOutOfStartup();
1103 ASSERT_FALSE(sender_->InSlowStart());
1104
1105 const QuicConnectionStats& stats = bbr_sender_.connection()->GetStats();
1106 EXPECT_EQ(1u, stats.slowstart_count);
1107 EXPECT_THAT(stats.slowstart_num_rtts, AllOf(Ge(5u), Le(15u)));
1108 EXPECT_THAT(stats.slowstart_packets_sent, AllOf(Ge(100u), Le(1000u)));
1109 EXPECT_THAT(stats.slowstart_bytes_sent, AllOf(Ge(100000u), Le(1000000u)));
1110 EXPECT_LE(stats.slowstart_packets_lost, 10u);
1111 EXPECT_LE(stats.slowstart_bytes_lost, 10000u);
1112 EXPECT_FALSE(stats.slowstart_duration.IsRunning());
1113 EXPECT_THAT(stats.slowstart_duration.GetTotalElapsedTime(),
1114 AllOf(Ge(QuicTime::Delta::FromMilliseconds(500)),
1115 Le(QuicTime::Delta::FromMilliseconds(1500))));
1116 EXPECT_EQ(stats.slowstart_duration.GetTotalElapsedTime(),
1117 QuicConnectionPeer::GetSentPacketManager(bbr_sender_.connection())
1118 ->GetSlowStartDuration());
1119 }
1120
1121 // Regression test for b/143540157.
TEST_F(BbrSenderTest,RecalculatePacingRateOnCwndChange1RTT)1122 TEST_F(BbrSenderTest, RecalculatePacingRateOnCwndChange1RTT) {
1123 CreateDefaultSetup();
1124
1125 bbr_sender_.AddBytesToTransfer(1 * 1024 * 1024);
1126 // Wait until an ACK comes back.
1127 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
1128 bool simulator_result = simulator_.RunUntilOrTimeout(
1129 [this]() { return !sender_->ExportDebugState().min_rtt.IsZero(); },
1130 timeout);
1131 ASSERT_TRUE(simulator_result);
1132 const QuicByteCount previous_cwnd =
1133 sender_->ExportDebugState().congestion_window;
1134
1135 // Bootstrap cwnd.
1136 bbr_sender_.connection()->AdjustNetworkParameters(
1137 SendAlgorithmInterface::NetworkParams(kTestLinkBandwidth,
1138 QuicTime::Delta::Zero(), false));
1139 EXPECT_LT(previous_cwnd, sender_->ExportDebugState().congestion_window);
1140
1141 // Verify pacing rate is re-calculated based on the new cwnd and min_rtt.
1142 EXPECT_APPROX_EQ(QuicBandwidth::FromBytesAndTimeDelta(
1143 sender_->ExportDebugState().congestion_window,
1144 sender_->ExportDebugState().min_rtt),
1145 sender_->PacingRate(/*bytes_in_flight=*/0), 0.01f);
1146 }
1147
TEST_F(BbrSenderTest,RecalculatePacingRateOnCwndChange0RTT)1148 TEST_F(BbrSenderTest, RecalculatePacingRateOnCwndChange0RTT) {
1149 CreateDefaultSetup();
1150 // Initial RTT is available.
1151 const_cast<RttStats*>(rtt_stats_)->set_initial_rtt(kTestRtt);
1152
1153 // Bootstrap cwnd.
1154 bbr_sender_.connection()->AdjustNetworkParameters(
1155 SendAlgorithmInterface::NetworkParams(kTestLinkBandwidth,
1156 QuicTime::Delta::Zero(), false));
1157 EXPECT_LT(kInitialCongestionWindowPackets * kDefaultTCPMSS,
1158 sender_->ExportDebugState().congestion_window);
1159 // No Rtt sample is available.
1160 EXPECT_TRUE(sender_->ExportDebugState().min_rtt.IsZero());
1161
1162 // Verify pacing rate is re-calculated based on the new cwnd and initial
1163 // RTT.
1164 EXPECT_APPROX_EQ(QuicBandwidth::FromBytesAndTimeDelta(
1165 sender_->ExportDebugState().congestion_window,
1166 rtt_stats_->initial_rtt()),
1167 sender_->PacingRate(/*bytes_in_flight=*/0), 0.01f);
1168 }
1169
TEST_F(BbrSenderTest,MitigateCwndBootstrappingOvershoot)1170 TEST_F(BbrSenderTest, MitigateCwndBootstrappingOvershoot) {
1171 CreateDefaultSetup();
1172 bbr_sender_.AddBytesToTransfer(1 * 1024 * 1024);
1173
1174 // Wait until an ACK comes back.
1175 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
1176 bool simulator_result = simulator_.RunUntilOrTimeout(
1177 [this]() { return !sender_->ExportDebugState().min_rtt.IsZero(); },
1178 timeout);
1179 ASSERT_TRUE(simulator_result);
1180
1181 // Bootstrap cwnd by a overly large bandwidth sample.
1182 bbr_sender_.connection()->AdjustNetworkParameters(
1183 SendAlgorithmInterface::NetworkParams(8 * kTestLinkBandwidth,
1184 QuicTime::Delta::Zero(), false));
1185 QuicBandwidth pacing_rate = sender_->PacingRate(0);
1186 EXPECT_EQ(8 * kTestLinkBandwidth, pacing_rate);
1187
1188 // Wait until pacing_rate decreases.
1189 simulator_result = simulator_.RunUntilOrTimeout(
1190 [this, pacing_rate]() { return sender_->PacingRate(0) < pacing_rate; },
1191 timeout);
1192 ASSERT_TRUE(simulator_result);
1193 EXPECT_EQ(BbrSender::STARTUP, sender_->ExportDebugState().mode);
1194 if (GetQuicReloadableFlag(quic_conservative_cwnd_and_pacing_gains)) {
1195 EXPECT_APPROX_EQ(2.0f * sender_->BandwidthEstimate(),
1196 sender_->PacingRate(0), 0.01f);
1197 } else {
1198 EXPECT_APPROX_EQ(2.885f * sender_->BandwidthEstimate(),
1199 sender_->PacingRate(0), 0.01f);
1200 }
1201 }
1202
1203 TEST_F(BbrSenderTest, 200InitialCongestionWindowWithNetworkParameterAdjusted) {
1204 CreateDefaultSetup();
1205
1206 bbr_sender_.AddBytesToTransfer(1 * 1024 * 1024);
1207 // Wait until an ACK comes back.
1208 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
1209 bool simulator_result = simulator_.RunUntilOrTimeout(
__anon9253fc6d1e02() 1210 [this]() { return !sender_->ExportDebugState().min_rtt.IsZero(); },
1211 timeout);
1212 ASSERT_TRUE(simulator_result);
1213
1214 // Bootstrap cwnd by a overly large bandwidth sample.
1215 bbr_sender_.connection()->AdjustNetworkParameters(
1216 SendAlgorithmInterface::NetworkParams(1024 * kTestLinkBandwidth,
1217 QuicTime::Delta::Zero(), false));
1218 // Verify cwnd is capped at 200.
1219 EXPECT_EQ(200 * kDefaultTCPMSS,
1220 sender_->ExportDebugState().congestion_window);
1221 EXPECT_GT(1024 * kTestLinkBandwidth, sender_->PacingRate(0));
1222 }
1223
1224 TEST_F(BbrSenderTest, 100InitialCongestionWindowFromNetworkParameter) {
1225 CreateDefaultSetup();
1226
1227 bbr_sender_.AddBytesToTransfer(1 * 1024 * 1024);
1228 // Wait until an ACK comes back.
1229 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
1230 bool simulator_result = simulator_.RunUntilOrTimeout(
__anon9253fc6d1f02() 1231 [this]() { return !sender_->ExportDebugState().min_rtt.IsZero(); },
1232 timeout);
1233 ASSERT_TRUE(simulator_result);
1234
1235 // Bootstrap cwnd by a overly large bandwidth sample.
1236 SendAlgorithmInterface::NetworkParams network_params(
1237 1024 * kTestLinkBandwidth, QuicTime::Delta::Zero(), false);
1238 network_params.max_initial_congestion_window = 100;
1239 bbr_sender_.connection()->AdjustNetworkParameters(network_params);
1240 // Verify cwnd is capped at 100.
1241 EXPECT_EQ(100 * kDefaultTCPMSS,
1242 sender_->ExportDebugState().congestion_window);
1243 EXPECT_GT(1024 * kTestLinkBandwidth, sender_->PacingRate(0));
1244 }
1245
1246 TEST_F(BbrSenderTest, 100InitialCongestionWindowWithNetworkParameterAdjusted) {
1247 SetConnectionOption(kICW1);
1248 CreateDefaultSetup();
1249
1250 bbr_sender_.AddBytesToTransfer(1 * 1024 * 1024);
1251 // Wait until an ACK comes back.
1252 const QuicTime::Delta timeout = QuicTime::Delta::FromSeconds(5);
1253 bool simulator_result = simulator_.RunUntilOrTimeout(
__anon9253fc6d2002() 1254 [this]() { return !sender_->ExportDebugState().min_rtt.IsZero(); },
1255 timeout);
1256 ASSERT_TRUE(simulator_result);
1257
1258 // Bootstrap cwnd by a overly large bandwidth sample.
1259 bbr_sender_.connection()->AdjustNetworkParameters(
1260 SendAlgorithmInterface::NetworkParams(1024 * kTestLinkBandwidth,
1261 QuicTime::Delta::Zero(), false));
1262 // Verify cwnd is capped at 100.
1263 EXPECT_EQ(100 * kDefaultTCPMSS,
1264 sender_->ExportDebugState().congestion_window);
1265 EXPECT_GT(1024 * kTestLinkBandwidth, sender_->PacingRate(0));
1266 }
1267
1268 // Ensures bandwidth estimate does not change after a loss only event.
1269 // Regression test for b/151239871.
TEST_F(BbrSenderTest,LossOnlyCongestionEvent)1270 TEST_F(BbrSenderTest, LossOnlyCongestionEvent) {
1271 CreateDefaultSetup();
1272
1273 DriveOutOfStartup();
1274 EXPECT_FALSE(sender_->ExportDebugState().last_sample_is_app_limited);
1275
1276 // Send some bursts, each burst increments round count by 1, since it only
1277 // generates small, app-limited samples, the max_bandwidth_ will not be
1278 // updated. At the end of all bursts, all estimates in max_bandwidth_ will
1279 // look very old such that any Update() will reset all estimates.
1280 SendBursts(20, 512, QuicTime::Delta::FromSeconds(3));
1281
1282 QuicUnackedPacketMap* unacked_packets =
1283 QuicSentPacketManagerPeer::GetUnackedPacketMap(
1284 QuicConnectionPeer::GetSentPacketManager(bbr_sender_.connection()));
1285 // Run until we have something in flight.
1286 bbr_sender_.AddBytesToTransfer(50 * 1024 * 1024);
1287 bool simulator_result = simulator_.RunUntilOrTimeout(
1288 [&]() { return unacked_packets->bytes_in_flight() > 0; },
1289 QuicTime::Delta::FromSeconds(5));
1290 ASSERT_TRUE(simulator_result);
1291
1292 const QuicBandwidth prior_bandwidth_estimate = sender_->BandwidthEstimate();
1293 EXPECT_APPROX_EQ(kTestLinkBandwidth, prior_bandwidth_estimate, 0.01f);
1294
1295 // Lose the least unacked packet.
1296 LostPacketVector lost_packets;
1297 lost_packets.emplace_back(
1298 bbr_sender_.connection()->sent_packet_manager().GetLeastUnacked(),
1299 kDefaultMaxPacketSize);
1300
1301 QuicTime now = simulator_.GetClock()->Now() + kTestRtt * 0.25;
1302 sender_->OnCongestionEvent(false, unacked_packets->bytes_in_flight(), now, {},
1303 lost_packets, 0, 0);
1304
1305 // Bandwidth estimate should not change for the loss only event.
1306 EXPECT_EQ(prior_bandwidth_estimate, sender_->BandwidthEstimate());
1307 }
1308
TEST_F(BbrSenderTest,EnableOvershootingDetection)1309 TEST_F(BbrSenderTest, EnableOvershootingDetection) {
1310 SetConnectionOption(kDTOS);
1311 CreateSmallBufferSetup();
1312 // Set a overly large initial cwnd.
1313 sender_->SetInitialCongestionWindowInPackets(200);
1314 const QuicConnectionStats& stats = bbr_sender_.connection()->GetStats();
1315 EXPECT_FALSE(stats.overshooting_detected_with_network_parameters_adjusted);
1316 DoSimpleTransfer(12 * 1024 * 1024, QuicTime::Delta::FromSeconds(30));
1317
1318 // Verify overshooting is detected.
1319 EXPECT_TRUE(stats.overshooting_detected_with_network_parameters_adjusted);
1320 }
1321
1322 } // namespace test
1323 } // namespace quic
1324