1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <memory>
12
13 #include "absl/algorithm/container.h"
14 #include "api/task_queue/task_queue_base.h"
15 #include "api/test/simulated_network.h"
16 #include "api/test/video/function_video_encoder_factory.h"
17 #include "api/units/time_delta.h"
18 #include "call/fake_network_pipe.h"
19 #include "call/simulated_network.h"
20 #include "modules/rtp_rtcp/source/rtp_packet.h"
21 #include "modules/video_coding/codecs/vp8/include/vp8.h"
22 #include "rtc_base/event.h"
23 #include "rtc_base/synchronization/mutex.h"
24 #include "rtc_base/task_queue_for_test.h"
25 #include "test/call_test.h"
26 #include "test/field_trial.h"
27 #include "test/gtest.h"
28 #include "test/rtcp_packet_parser.h"
29
30 namespace webrtc {
31 namespace {
32 enum : int { // The first valid value is 1.
33 kVideoRotationExtensionId = 1,
34 };
35 } // namespace
36
37 class RetransmissionEndToEndTest : public test::CallTest {
38 public:
RetransmissionEndToEndTest()39 RetransmissionEndToEndTest() {
40 RegisterRtpExtension(RtpExtension(RtpExtension::kVideoRotationUri,
41 kVideoRotationExtensionId));
42 }
43
44 protected:
45 void DecodesRetransmittedFrame(bool enable_rtx, bool enable_red);
46 void ReceivesPliAndRecovers(int rtp_history_ms);
47 };
48
TEST_F(RetransmissionEndToEndTest,ReceivesAndRetransmitsNack)49 TEST_F(RetransmissionEndToEndTest, ReceivesAndRetransmitsNack) {
50 static const int kNumberOfNacksToObserve = 2;
51 static const int kLossBurstSize = 2;
52 static const int kPacketsBetweenLossBursts = 9;
53 class NackObserver : public test::EndToEndTest {
54 public:
55 NackObserver()
56 : EndToEndTest(kLongTimeout),
57 sent_rtp_packets_(0),
58 packets_left_to_drop_(0),
59 nacks_left_(kNumberOfNacksToObserve) {}
60
61 private:
62 Action OnSendRtp(const uint8_t* packet, size_t length) override {
63 MutexLock lock(&mutex_);
64 RtpPacket rtp_packet;
65 EXPECT_TRUE(rtp_packet.Parse(packet, length));
66
67 // Never drop retransmitted packets.
68 if (dropped_packets_.find(rtp_packet.SequenceNumber()) !=
69 dropped_packets_.end()) {
70 retransmitted_packets_.insert(rtp_packet.SequenceNumber());
71 return SEND_PACKET;
72 }
73
74 if (nacks_left_ <= 0 &&
75 retransmitted_packets_.size() == dropped_packets_.size()) {
76 observation_complete_.Set();
77 }
78
79 ++sent_rtp_packets_;
80
81 // Enough NACKs received, stop dropping packets.
82 if (nacks_left_ <= 0)
83 return SEND_PACKET;
84
85 // Check if it's time for a new loss burst.
86 if (sent_rtp_packets_ % kPacketsBetweenLossBursts == 0)
87 packets_left_to_drop_ = kLossBurstSize;
88
89 // Never drop padding packets as those won't be retransmitted.
90 if (packets_left_to_drop_ > 0 && rtp_packet.padding_size() == 0) {
91 --packets_left_to_drop_;
92 dropped_packets_.insert(rtp_packet.SequenceNumber());
93 return DROP_PACKET;
94 }
95
96 return SEND_PACKET;
97 }
98
99 Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
100 MutexLock lock(&mutex_);
101 test::RtcpPacketParser parser;
102 EXPECT_TRUE(parser.Parse(packet, length));
103 nacks_left_ -= parser.nack()->num_packets();
104 return SEND_PACKET;
105 }
106
107 void ModifyVideoConfigs(
108 VideoSendStream::Config* send_config,
109 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
110 VideoEncoderConfig* encoder_config) override {
111 send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
112 (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
113 }
114
115 void PerformTest() override {
116 EXPECT_TRUE(Wait())
117 << "Timed out waiting for packets to be NACKed, retransmitted and "
118 "rendered.";
119 }
120
121 Mutex mutex_;
122 std::set<uint16_t> dropped_packets_;
123 std::set<uint16_t> retransmitted_packets_;
124 uint64_t sent_rtp_packets_;
125 int packets_left_to_drop_;
126 int nacks_left_ RTC_GUARDED_BY(&mutex_);
127 } test;
128
129 RunBaseTest(&test);
130 }
131
TEST_F(RetransmissionEndToEndTest,ReceivesNackAndRetransmitsAudio)132 TEST_F(RetransmissionEndToEndTest, ReceivesNackAndRetransmitsAudio) {
133 class NackObserver : public test::EndToEndTest {
134 public:
135 NackObserver()
136 : EndToEndTest(kLongTimeout),
137 local_ssrc_(0),
138 remote_ssrc_(0),
139 receive_transport_(nullptr) {}
140
141 private:
142 size_t GetNumVideoStreams() const override { return 0; }
143 size_t GetNumAudioStreams() const override { return 1; }
144
145 std::unique_ptr<test::PacketTransport> CreateReceiveTransport(
146 TaskQueueBase* task_queue) override {
147 auto receive_transport = std::make_unique<test::PacketTransport>(
148 task_queue, nullptr, this, test::PacketTransport::kReceiver,
149 payload_type_map_,
150 std::make_unique<FakeNetworkPipe>(
151 Clock::GetRealTimeClock(), std::make_unique<SimulatedNetwork>(
152 BuiltInNetworkBehaviorConfig())));
153 receive_transport_ = receive_transport.get();
154 return receive_transport;
155 }
156
157 Action OnSendRtp(const uint8_t* packet, size_t length) override {
158 RtpPacket rtp_packet;
159 EXPECT_TRUE(rtp_packet.Parse(packet, length));
160
161 if (!sequence_number_to_retransmit_) {
162 sequence_number_to_retransmit_ = rtp_packet.SequenceNumber();
163
164 // Don't ask for retransmission straight away, may be deduped in pacer.
165 } else if (rtp_packet.SequenceNumber() ==
166 *sequence_number_to_retransmit_) {
167 observation_complete_.Set();
168 } else {
169 // Send a NACK as often as necessary until retransmission is received.
170 rtcp::Nack nack;
171 nack.SetSenderSsrc(local_ssrc_);
172 nack.SetMediaSsrc(remote_ssrc_);
173 uint16_t nack_list[] = {*sequence_number_to_retransmit_};
174 nack.SetPacketIds(nack_list, 1);
175 rtc::Buffer buffer = nack.Build();
176
177 EXPECT_TRUE(receive_transport_->SendRtcp(buffer.data(), buffer.size()));
178 }
179
180 return SEND_PACKET;
181 }
182
183 void ModifyAudioConfigs(AudioSendStream::Config* send_config,
184 std::vector<AudioReceiveStreamInterface::Config>*
185 receive_configs) override {
186 (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
187 local_ssrc_ = (*receive_configs)[0].rtp.local_ssrc;
188 remote_ssrc_ = (*receive_configs)[0].rtp.remote_ssrc;
189 }
190
191 void PerformTest() override {
192 EXPECT_TRUE(Wait())
193 << "Timed out waiting for packets to be NACKed, retransmitted and "
194 "rendered.";
195 }
196
197 uint32_t local_ssrc_;
198 uint32_t remote_ssrc_;
199 Transport* receive_transport_;
200 absl::optional<uint16_t> sequence_number_to_retransmit_;
201 } test;
202
203 RunBaseTest(&test);
204 }
205
TEST_F(RetransmissionEndToEndTest,StopSendingKeyframeRequestsForInactiveStream)206 TEST_F(RetransmissionEndToEndTest,
207 StopSendingKeyframeRequestsForInactiveStream) {
208 class KeyframeRequestObserver : public test::EndToEndTest {
209 public:
210 explicit KeyframeRequestObserver(TaskQueueBase* task_queue)
211 : clock_(Clock::GetRealTimeClock()), task_queue_(task_queue) {}
212
213 void OnVideoStreamsCreated(VideoSendStream* send_stream,
214 const std::vector<VideoReceiveStreamInterface*>&
215 receive_streams) override {
216 RTC_DCHECK_EQ(1, receive_streams.size());
217 send_stream_ = send_stream;
218 receive_stream_ = receive_streams[0];
219 }
220
221 Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
222 test::RtcpPacketParser parser;
223 EXPECT_TRUE(parser.Parse(packet, length));
224 if (parser.pli()->num_packets() > 0)
225 task_queue_->PostTask([this] { Run(); });
226 return SEND_PACKET;
227 }
228
229 bool PollStats() {
230 if (receive_stream_->GetStats().frames_decoded > 0) {
231 frame_decoded_ = true;
232 } else if (clock_->TimeInMilliseconds() - start_time_ < 5000) {
233 task_queue_->PostDelayedTask([this] { Run(); }, TimeDelta::Millis(100));
234 return false;
235 }
236 return true;
237 }
238
239 void PerformTest() override {
240 start_time_ = clock_->TimeInMilliseconds();
241 task_queue_->PostTask([this] { Run(); });
242 test_done_.Wait(rtc::Event::kForever);
243 }
244
245 void Run() {
246 if (!frame_decoded_) {
247 if (PollStats()) {
248 send_stream_->Stop();
249 if (!frame_decoded_) {
250 test_done_.Set();
251 } else {
252 // Now we wait for the PLI packet. Once we receive it, a task
253 // will be posted (see OnReceiveRtcp) and we'll check the stats
254 // once more before signaling that we're done.
255 }
256 }
257 } else {
258 EXPECT_EQ(
259 1U,
260 receive_stream_->GetStats().rtcp_packet_type_counts.pli_packets);
261 test_done_.Set();
262 }
263 }
264
265 private:
266 Clock* const clock_;
267 VideoSendStream* send_stream_;
268 VideoReceiveStreamInterface* receive_stream_;
269 TaskQueueBase* const task_queue_;
270 rtc::Event test_done_;
271 bool frame_decoded_ = false;
272 int64_t start_time_ = 0;
273 } test(task_queue());
274
275 RunBaseTest(&test);
276 }
277
ReceivesPliAndRecovers(int rtp_history_ms)278 void RetransmissionEndToEndTest::ReceivesPliAndRecovers(int rtp_history_ms) {
279 static const int kPacketsToDrop = 1;
280
281 class PliObserver : public test::EndToEndTest,
282 public rtc::VideoSinkInterface<VideoFrame> {
283 public:
284 explicit PliObserver(int rtp_history_ms)
285 : EndToEndTest(kLongTimeout),
286 rtp_history_ms_(rtp_history_ms),
287 nack_enabled_(rtp_history_ms > 0),
288 highest_dropped_timestamp_(0),
289 frames_to_drop_(0),
290 received_pli_(false) {}
291
292 private:
293 Action OnSendRtp(const uint8_t* packet, size_t length) override {
294 MutexLock lock(&mutex_);
295 RtpPacket rtp_packet;
296 EXPECT_TRUE(rtp_packet.Parse(packet, length));
297
298 // Drop all retransmitted packets to force a PLI.
299 if (rtp_packet.Timestamp() <= highest_dropped_timestamp_)
300 return DROP_PACKET;
301
302 if (frames_to_drop_ > 0) {
303 highest_dropped_timestamp_ = rtp_packet.Timestamp();
304 --frames_to_drop_;
305 return DROP_PACKET;
306 }
307
308 return SEND_PACKET;
309 }
310
311 Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
312 MutexLock lock(&mutex_);
313 test::RtcpPacketParser parser;
314 EXPECT_TRUE(parser.Parse(packet, length));
315 if (!nack_enabled_)
316 EXPECT_EQ(0, parser.nack()->num_packets());
317 if (parser.pli()->num_packets() > 0)
318 received_pli_ = true;
319 return SEND_PACKET;
320 }
321
322 void OnFrame(const VideoFrame& video_frame) override {
323 MutexLock lock(&mutex_);
324 if (received_pli_ &&
325 video_frame.timestamp() > highest_dropped_timestamp_) {
326 observation_complete_.Set();
327 }
328 if (!received_pli_)
329 frames_to_drop_ = kPacketsToDrop;
330 }
331
332 void ModifyVideoConfigs(
333 VideoSendStream::Config* send_config,
334 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
335 VideoEncoderConfig* encoder_config) override {
336 send_config->rtp.nack.rtp_history_ms = rtp_history_ms_;
337 (*receive_configs)[0].rtp.nack.rtp_history_ms = rtp_history_ms_;
338 (*receive_configs)[0].renderer = this;
339 }
340
341 void PerformTest() override {
342 EXPECT_TRUE(Wait()) << "Timed out waiting for PLI to be "
343 "received and a frame to be "
344 "rendered afterwards.";
345 }
346
347 Mutex mutex_;
348 int rtp_history_ms_;
349 bool nack_enabled_;
350 uint32_t highest_dropped_timestamp_ RTC_GUARDED_BY(&mutex_);
351 int frames_to_drop_ RTC_GUARDED_BY(&mutex_);
352 bool received_pli_ RTC_GUARDED_BY(&mutex_);
353 } test(rtp_history_ms);
354
355 RunBaseTest(&test);
356 }
357
TEST_F(RetransmissionEndToEndTest,ReceivesPliAndRecoversWithNack)358 TEST_F(RetransmissionEndToEndTest, ReceivesPliAndRecoversWithNack) {
359 ReceivesPliAndRecovers(1000);
360 }
361
TEST_F(RetransmissionEndToEndTest,ReceivesPliAndRecoversWithoutNack)362 TEST_F(RetransmissionEndToEndTest, ReceivesPliAndRecoversWithoutNack) {
363 ReceivesPliAndRecovers(0);
364 }
365
366 // This test drops second RTP packet with a marker bit set, makes sure it's
367 // retransmitted and renders. Retransmission SSRCs are also checked.
DecodesRetransmittedFrame(bool enable_rtx,bool enable_red)368 void RetransmissionEndToEndTest::DecodesRetransmittedFrame(bool enable_rtx,
369 bool enable_red) {
370 static const int kDroppedFrameNumber = 10;
371 class RetransmissionObserver : public test::EndToEndTest,
372 public rtc::VideoSinkInterface<VideoFrame> {
373 public:
374 RetransmissionObserver(bool enable_rtx, bool enable_red)
375 : EndToEndTest(kDefaultTimeout),
376 payload_type_(GetPayloadType(false, enable_red)),
377 retransmission_ssrc_(enable_rtx ? kSendRtxSsrcs[0]
378 : kVideoSendSsrcs[0]),
379 retransmission_payload_type_(GetPayloadType(enable_rtx, enable_red)),
380 encoder_factory_([]() { return VP8Encoder::Create(); }),
381 marker_bits_observed_(0),
382 retransmitted_timestamp_(0) {}
383
384 private:
385 Action OnSendRtp(const uint8_t* packet, size_t length) override {
386 MutexLock lock(&mutex_);
387 RtpPacket rtp_packet;
388 EXPECT_TRUE(rtp_packet.Parse(packet, length));
389
390 // Ignore padding-only packets over RTX.
391 if (rtp_packet.PayloadType() != payload_type_) {
392 EXPECT_EQ(retransmission_ssrc_, rtp_packet.Ssrc());
393 if (rtp_packet.payload_size() == 0)
394 return SEND_PACKET;
395 }
396
397 if (rtp_packet.Timestamp() == retransmitted_timestamp_) {
398 EXPECT_EQ(retransmission_ssrc_, rtp_packet.Ssrc());
399 EXPECT_EQ(retransmission_payload_type_, rtp_packet.PayloadType());
400 return SEND_PACKET;
401 }
402
403 // Found the final packet of the frame to inflict loss to, drop this and
404 // expect a retransmission.
405 if (rtp_packet.PayloadType() == payload_type_ && rtp_packet.Marker() &&
406 ++marker_bits_observed_ == kDroppedFrameNumber) {
407 // This should be the only dropped packet.
408 EXPECT_EQ(0u, retransmitted_timestamp_);
409 retransmitted_timestamp_ = rtp_packet.Timestamp();
410 if (absl::c_linear_search(rendered_timestamps_,
411 retransmitted_timestamp_)) {
412 // Frame was rendered before last packet was scheduled for sending.
413 // This is extremly rare but possible scenario because prober able to
414 // resend packet before it was send.
415 // TODO(danilchap): Remove this corner case when prober would not be
416 // able to sneak in between packet saved to history for resending and
417 // pacer notified about existance of that packet for sending.
418 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5540 for
419 // details.
420 observation_complete_.Set();
421 }
422 return DROP_PACKET;
423 }
424
425 return SEND_PACKET;
426 }
427
428 void OnFrame(const VideoFrame& frame) override {
429 EXPECT_EQ(kVideoRotation_90, frame.rotation());
430 {
431 MutexLock lock(&mutex_);
432 if (frame.timestamp() == retransmitted_timestamp_)
433 observation_complete_.Set();
434 rendered_timestamps_.push_back(frame.timestamp());
435 }
436 orig_renderer_->OnFrame(frame);
437 }
438
439 void ModifyVideoConfigs(
440 VideoSendStream::Config* send_config,
441 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
442 VideoEncoderConfig* encoder_config) override {
443 send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
444
445 // Insert ourselves into the rendering pipeline.
446 RTC_DCHECK(!orig_renderer_);
447 orig_renderer_ = (*receive_configs)[0].renderer;
448 RTC_DCHECK(orig_renderer_);
449 // To avoid post-decode frame dropping, disable the prerender buffer.
450 (*receive_configs)[0].enable_prerenderer_smoothing = false;
451 (*receive_configs)[0].renderer = this;
452
453 (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
454
455 if (payload_type_ == kRedPayloadType) {
456 send_config->rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType;
457 send_config->rtp.ulpfec.red_payload_type = kRedPayloadType;
458 if (retransmission_ssrc_ == kSendRtxSsrcs[0])
459 send_config->rtp.ulpfec.red_rtx_payload_type = kRtxRedPayloadType;
460 (*receive_configs)[0].rtp.ulpfec_payload_type =
461 send_config->rtp.ulpfec.ulpfec_payload_type;
462 (*receive_configs)[0].rtp.red_payload_type =
463 send_config->rtp.ulpfec.red_payload_type;
464 }
465
466 if (retransmission_ssrc_ == kSendRtxSsrcs[0]) {
467 send_config->rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
468 send_config->rtp.rtx.payload_type = kSendRtxPayloadType;
469 (*receive_configs)[0].rtp.rtx_ssrc = kSendRtxSsrcs[0];
470 (*receive_configs)[0]
471 .rtp.rtx_associated_payload_types[(payload_type_ == kRedPayloadType)
472 ? kRtxRedPayloadType
473 : kSendRtxPayloadType] =
474 payload_type_;
475 }
476 // Configure encoding and decoding with VP8, since generic packetization
477 // doesn't support FEC with NACK.
478 RTC_DCHECK_EQ(1, (*receive_configs)[0].decoders.size());
479 send_config->encoder_settings.encoder_factory = &encoder_factory_;
480 send_config->rtp.payload_name = "VP8";
481 encoder_config->codec_type = kVideoCodecVP8;
482 (*receive_configs)[0].decoders[0].video_format = SdpVideoFormat("VP8");
483 }
484
485 void OnFrameGeneratorCapturerCreated(
486 test::FrameGeneratorCapturer* frame_generator_capturer) override {
487 frame_generator_capturer->SetFakeRotation(kVideoRotation_90);
488 }
489
490 void PerformTest() override {
491 EXPECT_TRUE(Wait())
492 << "Timed out while waiting for retransmission to render.";
493 }
494
495 int GetPayloadType(bool use_rtx, bool use_fec) {
496 if (use_fec) {
497 if (use_rtx)
498 return kRtxRedPayloadType;
499 return kRedPayloadType;
500 }
501 if (use_rtx)
502 return kSendRtxPayloadType;
503 return kFakeVideoSendPayloadType;
504 }
505
506 Mutex mutex_;
507 rtc::VideoSinkInterface<VideoFrame>* orig_renderer_ = nullptr;
508 const int payload_type_;
509 const uint32_t retransmission_ssrc_;
510 const int retransmission_payload_type_;
511 test::FunctionVideoEncoderFactory encoder_factory_;
512 const std::string payload_name_;
513 int marker_bits_observed_;
514 uint32_t retransmitted_timestamp_ RTC_GUARDED_BY(&mutex_);
515 std::vector<uint32_t> rendered_timestamps_ RTC_GUARDED_BY(&mutex_);
516 } test(enable_rtx, enable_red);
517
518 RunBaseTest(&test);
519 }
520
TEST_F(RetransmissionEndToEndTest,DecodesRetransmittedFrame)521 TEST_F(RetransmissionEndToEndTest, DecodesRetransmittedFrame) {
522 DecodesRetransmittedFrame(false, false);
523 }
524
TEST_F(RetransmissionEndToEndTest,DecodesRetransmittedFrameOverRtx)525 TEST_F(RetransmissionEndToEndTest, DecodesRetransmittedFrameOverRtx) {
526 DecodesRetransmittedFrame(true, false);
527 }
528
TEST_F(RetransmissionEndToEndTest,DecodesRetransmittedFrameByRed)529 TEST_F(RetransmissionEndToEndTest, DecodesRetransmittedFrameByRed) {
530 DecodesRetransmittedFrame(false, true);
531 }
532
TEST_F(RetransmissionEndToEndTest,DecodesRetransmittedFrameByRedOverRtx)533 TEST_F(RetransmissionEndToEndTest, DecodesRetransmittedFrameByRedOverRtx) {
534 DecodesRetransmittedFrame(true, true);
535 }
536
537 } // namespace webrtc
538