1 /*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/video_receive_stream2.h"
12
13 #include <algorithm>
14 #include <cstddef>
15 #include <deque>
16 #include <limits>
17 #include <memory>
18 #include <ostream>
19 #include <queue>
20 #include <tuple>
21 #include <utility>
22 #include <vector>
23
24 #include "absl/memory/memory.h"
25 #include "absl/types/optional.h"
26 #include "api/metronome/test/fake_metronome.h"
27 #include "api/test/mock_video_decoder.h"
28 #include "api/test/mock_video_decoder_factory.h"
29 #include "api/test/time_controller.h"
30 #include "api/units/frequency.h"
31 #include "api/units/time_delta.h"
32 #include "api/video/encoded_image.h"
33 #include "api/video/recordable_encoded_frame.h"
34 #include "api/video/test/video_frame_matchers.h"
35 #include "api/video/video_frame.h"
36 #include "api/video_codecs/sdp_video_format.h"
37 #include "api/video_codecs/video_decoder.h"
38 #include "call/rtp_stream_receiver_controller.h"
39 #include "call/video_receive_stream.h"
40 #include "common_video/test/utilities.h"
41 #include "media/engine/fake_webrtc_call.h"
42 #include "modules/pacing/packet_router.h"
43 #include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
44 #include "modules/video_coding/encoded_frame.h"
45 #include "rtc_base/logging.h"
46 #include "system_wrappers/include/clock.h"
47 #include "test/fake_decoder.h"
48 #include "test/fake_encoded_frame.h"
49 #include "test/gmock.h"
50 #include "test/gtest.h"
51 #include "test/mock_transport.h"
52 #include "test/rtcp_packet_parser.h"
53 #include "test/time_controller/simulated_time_controller.h"
54 #include "test/video_decoder_proxy_factory.h"
55 #include "video/call_stats2.h"
56
57 namespace webrtc {
58
59 // Printing SdpVideoFormat for gmock argument matchers.
PrintTo(const SdpVideoFormat & value,std::ostream * os)60 void PrintTo(const SdpVideoFormat& value, std::ostream* os) {
61 *os << value.ToString();
62 }
63
PrintTo(const RecordableEncodedFrame::EncodedResolution & value,std::ostream * os)64 void PrintTo(const RecordableEncodedFrame::EncodedResolution& value,
65 std::ostream* os) {
66 *os << value.width << "x" << value.height;
67 }
68
PrintTo(const RecordableEncodedFrame & value,std::ostream * os)69 void PrintTo(const RecordableEncodedFrame& value, std::ostream* os) {
70 *os << "RecordableEncodedFrame(render_time=" << value.render_time()
71 << " resolution=" << ::testing::PrintToString(value.resolution()) << ")";
72 }
73
74 } // namespace webrtc
75
76 namespace webrtc {
77
78 namespace {
79
80 using test::video_frame_matchers::NtpTimestamp;
81 using test::video_frame_matchers::PacketInfos;
82 using test::video_frame_matchers::Rotation;
83 using ::testing::_;
84 using ::testing::AllOf;
85 using ::testing::AnyNumber;
86 using ::testing::ElementsAreArray;
87 using ::testing::Eq;
88 using ::testing::Field;
89 using ::testing::InSequence;
90 using ::testing::Invoke;
91 using ::testing::IsEmpty;
92 using ::testing::Optional;
93 using ::testing::Pointee;
94 using ::testing::Property;
95 using ::testing::Return;
96 using ::testing::SizeIs;
97 using ::testing::WithoutArgs;
98
RenderedFrameWith(::testing::Matcher<VideoFrame> m)99 auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
100 return Optional(m);
101 }
RenderedFrame()102 auto RenderedFrame() {
103 return RenderedFrameWith(_);
104 }
DidNotReceiveFrame()105 testing::Matcher<absl::optional<VideoFrame>> DidNotReceiveFrame() {
106 return Eq(absl::nullopt);
107 }
108
109 constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
110 constexpr int kDefaultNumCpuCores = 2;
111
112 constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
113 constexpr Frequency k30Fps = Frequency::Hertz(30);
114 constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
115 constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
116 constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
117 constexpr uint32_t kFirstRtpTimestamp = 90000;
118
119 class FakeVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> {
120 public:
FakeVideoRenderer(TimeController * time_controller)121 explicit FakeVideoRenderer(TimeController* time_controller)
122 : time_controller_(time_controller) {}
123 ~FakeVideoRenderer() override = default;
124
OnFrame(const VideoFrame & frame)125 void OnFrame(const VideoFrame& frame) override {
126 RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
127 << frame.timestamp();
128 if (!last_frame_.empty()) {
129 RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
130 << last_frame_.back().timestamp();
131 }
132 last_frame_.push_back(frame);
133 }
134
135 // If `advance_time`, then the clock will always advance by `timeout`.
WaitForFrame(TimeDelta timeout,bool advance_time=false)136 absl::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
137 bool advance_time = false) {
138 auto start = time_controller_->GetClock()->CurrentTime();
139 if (last_frame_.empty()) {
140 time_controller_->AdvanceTime(TimeDelta::Zero());
141 time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
142 }
143 absl::optional<VideoFrame> ret;
144 if (!last_frame_.empty()) {
145 ret = last_frame_.front();
146 last_frame_.pop_front();
147 }
148 if (advance_time) {
149 time_controller_->AdvanceTime(
150 timeout - (time_controller_->GetClock()->CurrentTime() - start));
151 }
152 return ret;
153 }
154
155 private:
156 std::deque<VideoFrame> last_frame_;
157 TimeController* const time_controller_;
158 };
159
160 MATCHER_P2(MatchResolution, w, h, "") {
161 return arg.resolution().width == w && arg.resolution().height == h;
162 }
163
164 MATCHER_P(RtpTimestamp, timestamp, "") {
165 if (arg.timestamp() != timestamp) {
166 *result_listener->stream()
167 << "rtp timestamp was " << arg.timestamp() << " != " << timestamp;
168 return false;
169 }
170 return true;
171 }
172
173 // Rtp timestamp for in order frame at 30fps.
RtpTimestampForFrame(int id)174 uint32_t RtpTimestampForFrame(int id) {
175 return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
176 }
177
178 // Receive time for in order frame at 30fps.
ReceiveTimeForFrame(int id)179 Timestamp ReceiveTimeForFrame(int id) {
180 return kStartTime + id * k30FpsDelay;
181 }
182
183 } // namespace
184
185 class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
186 public:
DefaultDecodeAction()187 auto DefaultDecodeAction() {
188 return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
189 }
190
UseMetronome() const191 bool UseMetronome() const { return GetParam(); }
192
VideoReceiveStream2Test()193 VideoReceiveStream2Test()
194 : time_controller_(kStartTime),
195 clock_(time_controller_.GetClock()),
196 config_(&mock_transport_, &mock_h264_decoder_factory_),
197 call_stats_(clock_, time_controller_.GetMainThread()),
198 fake_renderer_(&time_controller_),
199 fake_metronome_(TimeDelta::Millis(16)),
200 decode_sync_(clock_,
201 &fake_metronome_,
202 time_controller_.GetMainThread()),
203 h264_decoder_factory_(&mock_decoder_) {
204 // By default, mock decoder factory is backed by VideoDecoderProxyFactory.
205 ON_CALL(mock_h264_decoder_factory_, CreateVideoDecoder)
206 .WillByDefault(
207 Invoke(&h264_decoder_factory_,
208 &test::VideoDecoderProxyFactory::CreateVideoDecoder));
209
210 // By default, mock decode will wrap the fake decoder.
211 ON_CALL(mock_decoder_, Configure)
212 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
213 ON_CALL(mock_decoder_, Decode).WillByDefault(DefaultDecodeAction());
214 ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
215 .WillByDefault(
216 Invoke(&fake_decoder_,
217 &test::FakeDecoder::RegisterDecodeCompleteCallback));
218 ON_CALL(mock_decoder_, Release)
219 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
220 ON_CALL(mock_transport_, SendRtcp)
221 .WillByDefault(
222 Invoke(&rtcp_packet_parser_, &test::RtcpPacketParser::Parse));
223 }
224
~VideoReceiveStream2Test()225 ~VideoReceiveStream2Test() override {
226 if (video_receive_stream_) {
227 video_receive_stream_->Stop();
228 video_receive_stream_->UnregisterFromTransport();
229 }
230 time_controller_.AdvanceTime(TimeDelta::Zero());
231 }
232
SetUp()233 void SetUp() override {
234 config_.rtp.remote_ssrc = 1111;
235 config_.rtp.local_ssrc = 2222;
236 config_.renderer = &fake_renderer_;
237 VideoReceiveStreamInterface::Decoder h264_decoder;
238 h264_decoder.payload_type = 99;
239 h264_decoder.video_format = SdpVideoFormat("H264");
240 h264_decoder.video_format.parameters.insert(
241 {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
242 VideoReceiveStreamInterface::Decoder h265_decoder;
243 h265_decoder.payload_type = 100;
244 h265_decoder.video_format = SdpVideoFormat("H265");
245
246 config_.decoders = {h265_decoder, h264_decoder};
247
248 RecreateReceiveStream();
249 }
250
RecreateReceiveStream(absl::optional<VideoReceiveStreamInterface::RecordingState> state=absl::nullopt)251 void RecreateReceiveStream(
252 absl::optional<VideoReceiveStreamInterface::RecordingState> state =
253 absl::nullopt) {
254 if (video_receive_stream_) {
255 video_receive_stream_->UnregisterFromTransport();
256 video_receive_stream_ = nullptr;
257 }
258 timing_ = new VCMTiming(clock_, fake_call_.trials());
259 video_receive_stream_ =
260 std::make_unique<webrtc::internal::VideoReceiveStream2>(
261 time_controller_.GetTaskQueueFactory(), &fake_call_,
262 kDefaultNumCpuCores, &packet_router_, config_.Copy(), &call_stats_,
263 clock_, absl::WrapUnique(timing_), &nack_periodic_processor_,
264 UseMetronome() ? &decode_sync_ : nullptr, nullptr);
265 video_receive_stream_->RegisterWithTransport(
266 &rtp_stream_receiver_controller_);
267 if (state)
268 video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
269 }
270
271 protected:
272 GlobalSimulatedTimeController time_controller_;
273 Clock* const clock_;
274 NackPeriodicProcessor nack_periodic_processor_;
275 testing::NiceMock<MockVideoDecoderFactory> mock_h264_decoder_factory_;
276 VideoReceiveStreamInterface::Config config_;
277 internal::CallStats call_stats_;
278 testing::NiceMock<MockVideoDecoder> mock_decoder_;
279 FakeVideoRenderer fake_renderer_;
280 cricket::FakeCall fake_call_;
281 MockTransport mock_transport_;
282 test::RtcpPacketParser rtcp_packet_parser_;
283 PacketRouter packet_router_;
284 RtpStreamReceiverController rtp_stream_receiver_controller_;
285 std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
286 VCMTiming* timing_;
287 test::FakeMetronome fake_metronome_;
288 DecodeSynchronizer decode_sync_;
289
290 private:
291 test::VideoDecoderProxyFactory h264_decoder_factory_;
292 test::FakeDecoder fake_decoder_;
293 };
294
TEST_P(VideoReceiveStream2Test,CreateFrameFromH264FmtpSpropAndIdr)295 TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
296 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
297 RtpPacketToSend rtppacket(nullptr);
298 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
299 memcpy(payload, idr_nalu, sizeof(idr_nalu));
300 rtppacket.SetMarker(true);
301 rtppacket.SetSsrc(1111);
302 rtppacket.SetPayloadType(99);
303 rtppacket.SetSequenceNumber(1);
304 rtppacket.SetTimestamp(0);
305 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
306 video_receive_stream_->Start();
307 EXPECT_CALL(mock_decoder_, Decode(_, false, _));
308 RtpPacketReceived parsed_packet;
309 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
310 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
311 EXPECT_CALL(mock_decoder_, Release());
312
313 time_controller_.AdvanceTime(TimeDelta::Zero());
314 }
315
TEST_P(VideoReceiveStream2Test,PlayoutDelay)316 TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
317 const VideoPlayoutDelay kPlayoutDelayMs = {123, 321};
318 std::unique_ptr<test::FakeEncodedFrame> test_frame =
319 test::FakeFrameBuilder().Id(0).AsLast().Build();
320 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
321
322 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
323 auto timings = timing_->GetTimings();
324 EXPECT_EQ(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
325 EXPECT_EQ(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
326
327 // Check that the biggest minimum delay is chosen.
328 video_receive_stream_->SetMinimumPlayoutDelay(400);
329 timings = timing_->GetTimings();
330 EXPECT_EQ(400, timings.min_playout_delay.ms());
331
332 // Check base minimum delay validation.
333 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
334 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
335 EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
336 timings = timing_->GetTimings();
337 EXPECT_EQ(500, timings.min_playout_delay.ms());
338
339 // Check that intermidiate values are remembered and the biggest remembered
340 // is chosen.
341 video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
342 timings = timing_->GetTimings();
343 EXPECT_EQ(400, timings.min_playout_delay.ms());
344
345 video_receive_stream_->SetMinimumPlayoutDelay(0);
346 timings = timing_->GetTimings();
347 EXPECT_EQ(123, timings.min_playout_delay.ms());
348 }
349
TEST_P(VideoReceiveStream2Test,PlayoutDelayPreservesDefaultMaxValue)350 TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMaxValue) {
351 const TimeDelta default_max_playout_latency =
352 timing_->GetTimings().max_playout_delay;
353 const VideoPlayoutDelay kPlayoutDelayMs = {123, -1};
354
355 std::unique_ptr<test::FakeEncodedFrame> test_frame =
356 test::FakeFrameBuilder().Id(0).AsLast().Build();
357 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
358
359 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
360
361 // Ensure that -1 preserves default maximum value from `timing_`.
362 auto timings = timing_->GetTimings();
363 EXPECT_EQ(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
364 EXPECT_NE(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
365 EXPECT_EQ(default_max_playout_latency, timings.max_playout_delay);
366 }
367
TEST_P(VideoReceiveStream2Test,PlayoutDelayPreservesDefaultMinValue)368 TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) {
369 const TimeDelta default_min_playout_latency =
370 timing_->GetTimings().min_playout_delay;
371 const VideoPlayoutDelay kPlayoutDelayMs = {-1, 321};
372
373 std::unique_ptr<test::FakeEncodedFrame> test_frame =
374 test::FakeFrameBuilder().Id(0).AsLast().Build();
375 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
376
377 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
378
379 // Ensure that -1 preserves default minimum value from `timing_`.
380 auto timings = timing_->GetTimings();
381 EXPECT_NE(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
382 EXPECT_EQ(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
383 EXPECT_EQ(default_min_playout_latency, timings.min_playout_delay);
384 }
385
TEST_P(VideoReceiveStream2Test,RenderParametersSetToDefaultValues)386 TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
387 // Default render parameters.
388 const VideoFrame::RenderParameters kDefaultRenderParameters;
389 // Default with no playout delay set.
390 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
391 test::FakeFrameBuilder().Id(0).AsLast().Build();
392 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
393 EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
394 }
395
TEST_P(VideoReceiveStream2Test,UseLowLatencyRenderingSetFromPlayoutDelay)396 TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
397 // use_low_latency_rendering set if playout delay set to min=0, max<=500 ms.
398 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
399 test::FakeFrameBuilder().Id(0).AsLast().Build();
400 test_frame0->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/0});
401 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
402 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
403
404 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
405 test::FakeFrameBuilder().Id(1).AsLast().Build();
406 test_frame1->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/500});
407 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
408 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
409 }
410
TEST_P(VideoReceiveStream2Test,MaxCompositionDelaySetFromMaxPlayoutDelay)411 TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
412 // The max composition delay is dependent on the number of frames in the
413 // pre-decode queue. It's therefore important to advance the time as the test
414 // runs to get the correct expectations of max_composition_delay_in_frames.
415 video_receive_stream_->Start();
416 // Max composition delay not set if no playout delay is set.
417 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
418 test::FakeFrameBuilder()
419 .Id(0)
420 .Time(RtpTimestampForFrame(0))
421 .ReceivedTime(ReceiveTimeForFrame(0))
422 .AsLast()
423 .Build();
424 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
425 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
426 Eq(absl::nullopt));
427 time_controller_.AdvanceTime(k30FpsDelay);
428
429 // Max composition delay not set for playout delay 0,0.
430 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
431 test::FakeFrameBuilder()
432 .Id(1)
433 .Time(RtpTimestampForFrame(1))
434 .ReceivedTime(ReceiveTimeForFrame(1))
435 .AsLast()
436 .Build();
437 test_frame1->SetPlayoutDelay({0, 0});
438 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
439 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
440 Eq(absl::nullopt));
441 time_controller_.AdvanceTime(k30FpsDelay);
442
443 // Max composition delay not set for playout delay X,Y, where X,Y>0.
444 std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
445 test::FakeFrameBuilder()
446 .Id(2)
447 .Time(RtpTimestampForFrame(2))
448 .ReceivedTime(ReceiveTimeForFrame(2))
449 .AsLast()
450 .Build();
451 test_frame2->SetPlayoutDelay({10, 30});
452 video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
453 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
454 Eq(absl::nullopt));
455
456 time_controller_.AdvanceTime(k30FpsDelay);
457
458 // Max composition delay set if playout delay X,Y, where X=0,Y>0.
459 const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
460 std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
461 test::FakeFrameBuilder()
462 .Id(3)
463 .Time(RtpTimestampForFrame(3))
464 .ReceivedTime(ReceiveTimeForFrame(3))
465 .AsLast()
466 .Build();
467 test_frame3->SetPlayoutDelay({0, 50});
468 video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
469 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
470 Optional(kExpectedMaxCompositionDelayInFrames));
471 }
472
TEST_P(VideoReceiveStream2Test,LazyDecoderCreation)473 TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
474 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
475 RtpPacketToSend rtppacket(nullptr);
476 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
477 memcpy(payload, idr_nalu, sizeof(idr_nalu));
478 rtppacket.SetMarker(true);
479 rtppacket.SetSsrc(1111);
480 // H265 payload type.
481 rtppacket.SetPayloadType(99);
482 rtppacket.SetSequenceNumber(1);
483 rtppacket.SetTimestamp(0);
484
485 // No decoders are created by default.
486 EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
487 video_receive_stream_->Start();
488 time_controller_.AdvanceTime(TimeDelta::Zero());
489
490 EXPECT_TRUE(
491 testing::Mock::VerifyAndClearExpectations(&mock_h264_decoder_factory_));
492 // Verify that the decoder is created when we receive payload data and tries
493 // to decode a frame.
494 EXPECT_CALL(
495 mock_h264_decoder_factory_,
496 CreateVideoDecoder(Field(&SdpVideoFormat::name, testing::Eq("H264"))));
497 EXPECT_CALL(mock_decoder_, Configure);
498 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
499 EXPECT_CALL(mock_decoder_, Decode);
500 RtpPacketReceived parsed_packet;
501 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
502 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
503 EXPECT_CALL(mock_decoder_, Release);
504
505 // Make sure the decoder thread had a chance to run.
506 time_controller_.AdvanceTime(TimeDelta::Zero());
507 }
508
TEST_P(VideoReceiveStream2Test,PassesNtpTime)509 TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
510 const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
511 std::unique_ptr<test::FakeEncodedFrame> test_frame =
512 test::FakeFrameBuilder()
513 .Id(0)
514 .PayloadType(99)
515 .NtpTime(kNtpTimestamp)
516 .AsLast()
517 .Build();
518
519 video_receive_stream_->Start();
520 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
521 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
522 RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
523 }
524
TEST_P(VideoReceiveStream2Test,PassesRotation)525 TEST_P(VideoReceiveStream2Test, PassesRotation) {
526 const webrtc::VideoRotation kRotation = webrtc::kVideoRotation_180;
527 std::unique_ptr<test::FakeEncodedFrame> test_frame = test::FakeFrameBuilder()
528 .Id(0)
529 .PayloadType(99)
530 .Rotation(kRotation)
531 .AsLast()
532 .Build();
533
534 video_receive_stream_->Start();
535 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
536 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
537 RenderedFrameWith(Rotation(kRotation)));
538 }
539
TEST_P(VideoReceiveStream2Test,PassesPacketInfos)540 TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
541 RtpPacketInfos packet_infos = CreatePacketInfos(3);
542 auto test_frame = test::FakeFrameBuilder()
543 .Id(0)
544 .PayloadType(99)
545 .PacketInfos(packet_infos)
546 .AsLast()
547 .Build();
548
549 video_receive_stream_->Start();
550 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
551 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
552 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
553 }
554
TEST_P(VideoReceiveStream2Test,RenderedFrameUpdatesGetSources)555 TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
556 constexpr uint32_t kSsrc = 1111;
557 constexpr uint32_t kCsrc = 9001;
558 constexpr uint32_t kRtpTimestamp = 12345;
559
560 // Prepare one video frame with per-packet information.
561 auto test_frame =
562 test::FakeFrameBuilder().Id(0).PayloadType(99).AsLast().Build();
563 RtpPacketInfos packet_infos;
564 {
565 RtpPacketInfos::vector_type infos;
566
567 RtpPacketInfo info;
568 info.set_ssrc(kSsrc);
569 info.set_csrcs({kCsrc});
570 info.set_rtp_timestamp(kRtpTimestamp);
571
572 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(5000));
573 infos.push_back(info);
574
575 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(3000));
576 infos.push_back(info);
577
578 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(2000));
579 infos.push_back(info);
580
581 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(1000));
582 infos.push_back(info);
583
584 packet_infos = RtpPacketInfos(std::move(infos));
585 }
586 test_frame->SetPacketInfos(packet_infos);
587
588 // Start receive stream.
589 video_receive_stream_->Start();
590 EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
591
592 // Render one video frame.
593 int64_t timestamp_ms_min = clock_->TimeInMilliseconds();
594 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
595 // Verify that the per-packet information is passed to the renderer.
596 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
597 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
598 int64_t timestamp_ms_max = clock_->TimeInMilliseconds();
599
600 // Verify that the per-packet information also updates `GetSources()`.
601 std::vector<RtpSource> sources = video_receive_stream_->GetSources();
602 ASSERT_THAT(sources, SizeIs(2));
603 {
604 auto it = std::find_if(sources.begin(), sources.end(),
605 [](const RtpSource& source) {
606 return source.source_type() == RtpSourceType::SSRC;
607 });
608 ASSERT_NE(it, sources.end());
609
610 EXPECT_EQ(it->source_id(), kSsrc);
611 EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
612 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
613 EXPECT_GE(it->timestamp_ms(), timestamp_ms_min);
614 EXPECT_LE(it->timestamp_ms(), timestamp_ms_max);
615 }
616 {
617 auto it = std::find_if(sources.begin(), sources.end(),
618 [](const RtpSource& source) {
619 return source.source_type() == RtpSourceType::CSRC;
620 });
621 ASSERT_NE(it, sources.end());
622
623 EXPECT_EQ(it->source_id(), kCsrc);
624 EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
625 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
626 EXPECT_GE(it->timestamp_ms(), timestamp_ms_min);
627 EXPECT_LE(it->timestamp_ms(), timestamp_ms_max);
628 }
629 }
630
MakeFrameWithResolution(VideoFrameType frame_type,int picture_id,int width,int height)631 std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
632 VideoFrameType frame_type,
633 int picture_id,
634 int width,
635 int height) {
636 auto frame =
637 test::FakeFrameBuilder().Id(picture_id).PayloadType(99).AsLast().Build();
638 frame->SetFrameType(frame_type);
639 frame->_encodedWidth = width;
640 frame->_encodedHeight = height;
641 return frame;
642 }
643
MakeFrame(VideoFrameType frame_type,int picture_id)644 std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
645 int picture_id) {
646 return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
647 }
648
TEST_P(VideoReceiveStream2Test,PassesFrameWhenEncodedFramesCallbackSet)649 TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
650 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
651 video_receive_stream_->Start();
652 EXPECT_CALL(callback, Call);
653 video_receive_stream_->SetAndGetRecordingState(
654 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
655 true);
656 video_receive_stream_->OnCompleteFrame(
657 MakeFrame(VideoFrameType::kVideoFrameKey, 0));
658 EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
659
660 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
661
662 video_receive_stream_->Stop();
663 }
664
TEST_P(VideoReceiveStream2Test,MovesEncodedFrameDispatchStateWhenReCreating)665 TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
666 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
667 video_receive_stream_->Start();
668 // Expect a key frame request over RTCP.
669 video_receive_stream_->SetAndGetRecordingState(
670 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
671 true);
672 video_receive_stream_->Stop();
673 VideoReceiveStreamInterface::RecordingState old_state =
674 video_receive_stream_->SetAndGetRecordingState(
675 VideoReceiveStreamInterface::RecordingState(), false);
676 RecreateReceiveStream(std::move(old_state));
677
678 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
679
680 video_receive_stream_->Stop();
681 }
682
TEST_P(VideoReceiveStream2Test,RequestsKeyFramesUntilKeyFrameReceived)683 TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
684 // Recreate receive stream with shorter delay to test rtx.
685 TimeDelta rtx_delay = TimeDelta::Millis(50);
686 config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
687 auto tick = rtx_delay / 2;
688 RecreateReceiveStream();
689 video_receive_stream_->Start();
690
691 video_receive_stream_->GenerateKeyFrame();
692 video_receive_stream_->OnCompleteFrame(
693 MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
694 fake_renderer_.WaitForFrame(kDefaultTimeOut);
695 time_controller_.AdvanceTime(tick);
696 video_receive_stream_->OnCompleteFrame(
697 MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
698 fake_renderer_.WaitForFrame(kDefaultTimeOut);
699 time_controller_.AdvanceTime(TimeDelta::Zero());
700 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
701
702 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
703
704 // T+keyframetimeout: still no key frame received, expect key frame request
705 // sent again.
706 time_controller_.AdvanceTime(tick);
707 video_receive_stream_->OnCompleteFrame(
708 MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
709 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
710 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
711
712 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
713
714 // T+keyframetimeout: now send a key frame - we should not observe new key
715 // frame requests after this.
716 video_receive_stream_->OnCompleteFrame(
717 MakeFrame(VideoFrameType::kVideoFrameKey, 3));
718 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
719 time_controller_.AdvanceTime(2 * tick);
720 video_receive_stream_->OnCompleteFrame(
721 MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
722 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
723
724 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
725 }
726
TEST_P(VideoReceiveStream2Test,DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution)727 TEST_P(VideoReceiveStream2Test,
728 DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
729 video_receive_stream_->Start();
730 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
731 video_receive_stream_->SetAndGetRecordingState(
732 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
733 /*generate_key_frame=*/false);
734
735 InSequence s;
736 EXPECT_CALL(callback,
737 Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
738 test::FakeDecoder::kDefaultHeight)));
739 EXPECT_CALL(callback, Call);
740
741 video_receive_stream_->OnCompleteFrame(
742 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
743 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
744 video_receive_stream_->OnCompleteFrame(
745 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
746 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
747
748 video_receive_stream_->Stop();
749 }
750
TEST_P(VideoReceiveStream2Test,DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution)751 TEST_P(VideoReceiveStream2Test,
752 DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
753 video_receive_stream_->Start();
754 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
755 video_receive_stream_->SetAndGetRecordingState(
756 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
757 /*generate_key_frame=*/false);
758
759 InSequence s;
760 EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
761 EXPECT_CALL(callback, Call);
762
763 video_receive_stream_->OnCompleteFrame(
764 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
765 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
766 video_receive_stream_->OnCompleteFrame(
767 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
768 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
769
770 video_receive_stream_->Stop();
771 }
772
TEST_P(VideoReceiveStream2Test,DependantFramesAreScheduled)773 TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
774 video_receive_stream_->Start();
775
776 auto key_frame = test::FakeFrameBuilder()
777 .Id(0)
778 .PayloadType(99)
779 .Time(kFirstRtpTimestamp)
780 .ReceivedTime(kStartTime)
781 .AsLast()
782 .Build();
783 auto delta_frame = test::FakeFrameBuilder()
784 .Id(1)
785 .PayloadType(99)
786 .Time(RtpTimestampForFrame(1))
787 .ReceivedTime(ReceiveTimeForFrame(1))
788 .Refs({0})
789 .AsLast()
790 .Build();
791
792 // Expect frames are decoded in order.
793 InSequence seq;
794 EXPECT_CALL(mock_decoder_,
795 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _));
796 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
797 k30FpsRtpTimestampDelta),
798 _, _))
799 .Times(1);
800 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
801 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
802
803 time_controller_.AdvanceTime(k30FpsDelay);
804 video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
805 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
806
807 video_receive_stream_->Stop();
808 }
809
TEST_P(VideoReceiveStream2Test,FramesScheduledInOrder)810 TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
811 video_receive_stream_->Start();
812
813 auto key_frame = test::FakeFrameBuilder()
814 .Id(0)
815 .PayloadType(99)
816 .Time(kFirstRtpTimestamp)
817 .AsLast()
818 .Build();
819 auto delta_frame1 = test::FakeFrameBuilder()
820 .Id(1)
821 .PayloadType(99)
822 .Time(RtpTimestampForFrame(1))
823 .Refs({0})
824 .AsLast()
825 .Build();
826 auto delta_frame2 = test::FakeFrameBuilder()
827 .Id(2)
828 .PayloadType(99)
829 .Time(RtpTimestampForFrame(2))
830 .Refs({1})
831 .AsLast()
832 .Build();
833
834 // Expect frames are decoded in order despite delta_frame1 arriving first.
835 InSequence seq;
836 EXPECT_CALL(mock_decoder_,
837 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
838 .Times(1);
839 EXPECT_CALL(mock_decoder_,
840 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _, _))
841 .Times(1);
842 EXPECT_CALL(mock_decoder_,
843 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _))
844 .Times(1);
845 key_frame->SetReceivedTime(clock_->CurrentTime().ms());
846 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
847 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
848
849 delta_frame2->SetReceivedTime(clock_->CurrentTime().ms());
850 video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
851 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
852 // `delta_frame1` arrives late.
853 delta_frame1->SetReceivedTime(clock_->CurrentTime().ms());
854 video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
855 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
856 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame());
857 video_receive_stream_->Stop();
858 }
859
TEST_P(VideoReceiveStream2Test,WaitsforAllSpatialLayers)860 TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
861 video_receive_stream_->Start();
862 auto sl0 = test::FakeFrameBuilder()
863 .Id(0)
864 .PayloadType(99)
865 .Time(kFirstRtpTimestamp)
866 .ReceivedTime(kStartTime)
867 .Build();
868 auto sl1 = test::FakeFrameBuilder()
869 .Id(1)
870 .PayloadType(99)
871 .ReceivedTime(kStartTime)
872 .Time(kFirstRtpTimestamp)
873 .Refs({0})
874 .Build();
875 auto sl2 = test::FakeFrameBuilder()
876 .Id(2)
877 .PayloadType(99)
878 .ReceivedTime(kStartTime)
879 .Time(kFirstRtpTimestamp)
880 .Refs({0, 1})
881 .AsLast()
882 .Build();
883
884 // No decodes should be called until `sl2` is received.
885 EXPECT_CALL(mock_decoder_, Decode).Times(0);
886 sl0->SetReceivedTime(clock_->CurrentTime().ms());
887 video_receive_stream_->OnCompleteFrame(std::move(sl0));
888 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
889 DidNotReceiveFrame());
890 video_receive_stream_->OnCompleteFrame(std::move(sl1));
891 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
892 DidNotReceiveFrame());
893 // When `sl2` arrives decode should happen.
894 EXPECT_CALL(mock_decoder_,
895 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
896 .Times(1);
897 video_receive_stream_->OnCompleteFrame(std::move(sl2));
898 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
899 video_receive_stream_->Stop();
900 }
901
TEST_P(VideoReceiveStream2Test,FramesFastForwardOnSystemHalt)902 TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
903 video_receive_stream_->Start();
904
905 // The frame structure looks like this,
906 // F1
907 // /
908 // F0 --> F2
909 //
910 // In this case we will have a system halt simulated. By the time the system
911 // resumes, F1 will be old and so F2 should be decoded.
912 auto key_frame = test::FakeFrameBuilder()
913 .Id(0)
914 .PayloadType(99)
915 .Time(kFirstRtpTimestamp)
916 .AsLast()
917 .Build();
918 auto ffwd_frame = test::FakeFrameBuilder()
919 .Id(1)
920 .PayloadType(99)
921 .Time(RtpTimestampForFrame(1))
922 .Refs({0})
923 .AsLast()
924 .Build();
925 auto rendered_frame = test::FakeFrameBuilder()
926 .Id(2)
927 .PayloadType(99)
928 .Time(RtpTimestampForFrame(2))
929 .Refs({0})
930 .AsLast()
931 .Build();
932 InSequence seq;
933 EXPECT_CALL(mock_decoder_,
934 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
935 .WillOnce(testing::DoAll(Invoke([&] {
936 // System halt will be simulated in the decode.
937 time_controller_.AdvanceTime(k30FpsDelay * 2);
938 }),
939 DefaultDecodeAction()));
940 EXPECT_CALL(mock_decoder_,
941 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _));
942 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
943 video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
944 video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
945 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
946 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
947 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
948 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
949
950 // Check stats show correct dropped frames.
951 auto stats = video_receive_stream_->GetStats();
952 EXPECT_EQ(stats.frames_dropped, 1u);
953
954 video_receive_stream_->Stop();
955 }
956
TEST_P(VideoReceiveStream2Test,BetterFrameInsertedWhileWaitingToDecodeFrame)957 TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
958 video_receive_stream_->Start();
959
960 auto key_frame = test::FakeFrameBuilder()
961 .Id(0)
962 .PayloadType(99)
963 .Time(kFirstRtpTimestamp)
964 .ReceivedTime(ReceiveTimeForFrame(0))
965 .AsLast()
966 .Build();
967 auto f1 = test::FakeFrameBuilder()
968 .Id(1)
969 .PayloadType(99)
970 .Time(RtpTimestampForFrame(1))
971 .ReceivedTime(ReceiveTimeForFrame(1))
972 .Refs({0})
973 .AsLast()
974 .Build();
975 auto f2 = test::FakeFrameBuilder()
976 .Id(2)
977 .PayloadType(99)
978 .Time(RtpTimestampForFrame(2))
979 .ReceivedTime(ReceiveTimeForFrame(2))
980 .Refs({0})
981 .AsLast()
982 .Build();
983
984 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
985 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
986
987 InSequence seq;
988 EXPECT_CALL(mock_decoder_,
989 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _, _))
990 .Times(1);
991 EXPECT_CALL(mock_decoder_,
992 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _))
993 .Times(1);
994 // Simulate f1 arriving after f2 but before f2 is decoded.
995 video_receive_stream_->OnCompleteFrame(std::move(f2));
996 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
997 video_receive_stream_->OnCompleteFrame(std::move(f1));
998 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
999 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1000
1001 video_receive_stream_->Stop();
1002 }
1003
1004 // Note: This test takes a long time (~10s) to run if the fake metronome is
1005 // active. Since the test needs to wait for the timestamp to rollover, it has a
1006 // fake delay of around 6.5 hours. Even though time is simulated, this will be
1007 // around 1,500,000 metronome tick invocations.
TEST_P(VideoReceiveStream2Test,RtpTimestampWrapAround)1008 TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
1009 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1010 video_receive_stream_->Start();
1011
1012 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
1013 video_receive_stream_->OnCompleteFrame(
1014 test::FakeFrameBuilder()
1015 .Id(0)
1016 .PayloadType(99)
1017 .Time(kBaseRtp)
1018 .ReceivedTime(clock_->CurrentTime())
1019 .AsLast()
1020 .Build());
1021 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1022 time_controller_.AdvanceTime(k30FpsDelay);
1023 video_receive_stream_->OnCompleteFrame(
1024 test::FakeFrameBuilder()
1025 .Id(1)
1026 .PayloadType(99)
1027 .Time(kBaseRtp + k30FpsRtpTimestampDelta)
1028 .ReceivedTime(clock_->CurrentTime())
1029 .AsLast()
1030 .Build());
1031 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1032
1033 // Pause stream so that RTP timestamp wraps around.
1034 constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
1035 constexpr uint32_t kWrapAroundRtp =
1036 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
1037 // Pause for corresponding delay such that RTP timestamp would increase this
1038 // much at 30fps.
1039 constexpr TimeDelta kWrapAroundDelay =
1040 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
1041
1042 time_controller_.AdvanceTime(kWrapAroundDelay);
1043 video_receive_stream_->OnCompleteFrame(
1044 test::FakeFrameBuilder()
1045 .Id(2)
1046 .PayloadType(99)
1047 .Time(kWrapAroundRtp)
1048 .ReceivedTime(clock_->CurrentTime())
1049 .AsLast()
1050 .Build());
1051 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _, _))
1052 .Times(1);
1053 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1054
1055 video_receive_stream_->Stop();
1056 }
1057
1058 // If a frame was lost causing the stream to become temporarily non-decodable
1059 // and the sender reduces their framerate during this time, the video stream
1060 // should start decoding at the new framerate. However, if the connection is
1061 // poor, a keyframe will take a long time to send. If the timing of the incoming
1062 // frames was not kept up to date with the new framerate while the stream was
1063 // decodable, this late frame will have a large delay as the rtp timestamp of
1064 // this keyframe will look like the frame arrived early if the frame-rate was
1065 // not updated.
TEST_P(VideoReceiveStream2Test,PoorConnectionWithFpsChangeDuringLostFrame)1066 TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
1067 video_receive_stream_->Start();
1068
1069 constexpr Frequency k15Fps = Frequency::Hertz(15);
1070 constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
1071 constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
1072
1073 // Initial keyframe and frames at 30fps.
1074 video_receive_stream_->OnCompleteFrame(
1075 test::FakeFrameBuilder()
1076 .Id(0)
1077 .PayloadType(99)
1078 .Time(RtpTimestampForFrame(0))
1079 .ReceivedTime(ReceiveTimeForFrame(0))
1080 .AsLast()
1081 .Build());
1082 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1083 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1084
1085 video_receive_stream_->OnCompleteFrame(
1086 test::FakeFrameBuilder()
1087 .Id(1)
1088 .PayloadType(99)
1089 .Time(RtpTimestampForFrame(1))
1090 .ReceivedTime(ReceiveTimeForFrame(1))
1091 .Refs({0})
1092 .AsLast()
1093 .Build());
1094 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1095 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
1096
1097 // Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
1098 // 2 second of frames at 15 fps, and then a keyframe.
1099 time_controller_.AdvanceTime(k30FpsDelay);
1100
1101 Timestamp send_30fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2);
1102 int id = 3;
1103 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1104 while (clock_->CurrentTime() < send_30fps_end_time) {
1105 ++id;
1106 video_receive_stream_->OnCompleteFrame(
1107 test::FakeFrameBuilder()
1108 .Id(id)
1109 .PayloadType(99)
1110 .Time(RtpTimestampForFrame(id))
1111 .ReceivedTime(ReceiveTimeForFrame(id))
1112 .Refs({id - 1})
1113 .AsLast()
1114 .Build());
1115 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1116 Eq(absl::nullopt));
1117 }
1118 uint32_t current_rtp = RtpTimestampForFrame(id);
1119 Timestamp send_15fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2);
1120 while (clock_->CurrentTime() < send_15fps_end_time) {
1121 ++id;
1122 current_rtp += k15FpsRtpTimestampDelta;
1123 video_receive_stream_->OnCompleteFrame(
1124 test::FakeFrameBuilder()
1125 .Id(id)
1126 .PayloadType(99)
1127 .Time(current_rtp)
1128 .ReceivedTime(clock_->CurrentTime())
1129 .Refs({id - 1})
1130 .AsLast()
1131 .Build());
1132 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1133 Eq(absl::nullopt));
1134 }
1135
1136 ++id;
1137 current_rtp += k15FpsRtpTimestampDelta;
1138 // Insert keyframe which will recover the stream. However, on a poor
1139 // connection the keyframe will take significant time to send.
1140 constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
1141 video_receive_stream_->OnCompleteFrame(
1142 test::FakeFrameBuilder()
1143 .Id(id)
1144 .PayloadType(99)
1145 .Time(current_rtp)
1146 .ReceivedTime(clock_->CurrentTime() + kKeyframeDelay)
1147 .AsLast()
1148 .Build());
1149 // If the framerate was not updated to be 15fps from the frames that arrived
1150 // previously, this will fail, as the delay will be longer.
1151 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1152 RenderedFrameWith(RtpTimestamp(current_rtp)));
1153
1154 video_receive_stream_->Stop();
1155 }
1156
TEST_P(VideoReceiveStream2Test,StreamShouldNotTimeoutWhileWaitingForFrame)1157 TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
1158 // Disable smoothing since this makes it hard to test frame timing.
1159 config_.enable_prerenderer_smoothing = false;
1160 RecreateReceiveStream();
1161
1162 video_receive_stream_->Start();
1163 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1164
1165 video_receive_stream_->OnCompleteFrame(
1166 test::FakeFrameBuilder()
1167 .Id(0)
1168 .PayloadType(99)
1169 .Time(RtpTimestampForFrame(0))
1170 .ReceivedTime(ReceiveTimeForFrame(0))
1171 .AsLast()
1172 .Build());
1173 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1174 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1175
1176 for (int id = 1; id < 30; ++id) {
1177 video_receive_stream_->OnCompleteFrame(
1178 test::FakeFrameBuilder()
1179 .Id(id)
1180 .PayloadType(99)
1181 .Time(RtpTimestampForFrame(id))
1182 .ReceivedTime(ReceiveTimeForFrame(id))
1183 .Refs({0})
1184 .AsLast()
1185 .Build());
1186 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1187 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
1188 }
1189
1190 // Simulate a pause in the stream, followed by a decodable frame that is ready
1191 // long in the future. The stream should not timeout in this case, but rather
1192 // decode the frame just before the timeout.
1193 time_controller_.AdvanceTime(TimeDelta::Millis(2900));
1194 uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
1195 video_receive_stream_->OnCompleteFrame(
1196 test::FakeFrameBuilder()
1197 .Id(121)
1198 .PayloadType(99)
1199 .Time(late_decode_rtp)
1200 .ReceivedTime(clock_->CurrentTime())
1201 .AsLast()
1202 .Build());
1203 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
1204 /*advance_time=*/true),
1205 RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
1206
1207 video_receive_stream_->Stop();
1208 }
1209
1210 INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
1211 VideoReceiveStream2Test,
1212 testing::Bool(),
__anon4a717d8c0602(const auto& test_param_info) 1213 [](const auto& test_param_info) {
1214 return (test_param_info.param
1215 ? "ScheduleDecodesWithMetronome"
1216 : "ScheduleDecodesWithPostTask");
1217 });
1218
1219 } // namespace webrtc
1220