1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <memory>
12
13 #include "api/test/simulated_network.h"
14 #include "api/test/video/function_video_encoder_factory.h"
15 #include "call/fake_network_pipe.h"
16 #include "call/simulated_network.h"
17 #include "media/engine/internal_encoder_factory.h"
18 #include "media/engine/simulcast_encoder_adapter.h"
19 #include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
20 #include "modules/rtp_rtcp/source/rtp_packet.h"
21 #include "modules/video_coding/codecs/vp8/include/vp8.h"
22 #include "modules/video_coding/codecs/vp9/include/vp9.h"
23 #include "rtc_base/numerics/safe_conversions.h"
24 #include "rtc_base/numerics/sequence_number_util.h"
25 #include "rtc_base/synchronization/mutex.h"
26 #include "rtc_base/task_queue_for_test.h"
27 #include "test/call_test.h"
28
29 namespace webrtc {
30 namespace {
31 const int kFrameMaxWidth = 1280;
32 const int kFrameMaxHeight = 720;
33 const int kFrameRate = 30;
34 const int kMaxSecondsLost = 5;
35 const int kMaxFramesLost = kFrameRate * kMaxSecondsLost;
36 const int kMinPacketsToObserve = 10;
37 const int kEncoderBitrateBps = 300000;
38 const uint32_t kPictureIdWraparound = (1 << 15);
39 const size_t kNumTemporalLayers[] = {1, 2, 3};
40
41 } // namespace
42
43 class PictureIdObserver : public test::RtpRtcpObserver {
44 public:
PictureIdObserver(VideoCodecType codec_type)45 explicit PictureIdObserver(VideoCodecType codec_type)
46 : test::RtpRtcpObserver(test::CallTest::kDefaultTimeout),
47 depacketizer_(CreateVideoRtpDepacketizer(codec_type)),
48 max_expected_picture_id_gap_(0),
49 max_expected_tl0_idx_gap_(0),
50 num_ssrcs_to_observe_(1) {}
51
SetExpectedSsrcs(size_t num_expected_ssrcs)52 void SetExpectedSsrcs(size_t num_expected_ssrcs) {
53 MutexLock lock(&mutex_);
54 num_ssrcs_to_observe_ = num_expected_ssrcs;
55 }
56
ResetObservedSsrcs()57 void ResetObservedSsrcs() {
58 MutexLock lock(&mutex_);
59 // Do not clear the timestamp and picture_id, to ensure that we check
60 // consistency between reinits and recreations.
61 num_packets_sent_.clear();
62 observed_ssrcs_.clear();
63 }
64
SetMaxExpectedPictureIdGap(int max_expected_picture_id_gap)65 void SetMaxExpectedPictureIdGap(int max_expected_picture_id_gap) {
66 MutexLock lock(&mutex_);
67 max_expected_picture_id_gap_ = max_expected_picture_id_gap;
68 // Expect smaller gap for `tl0_pic_idx` (running index for temporal_idx 0).
69 max_expected_tl0_idx_gap_ = max_expected_picture_id_gap_ / 2;
70 }
71
72 private:
73 struct ParsedPacket {
74 uint32_t timestamp;
75 uint32_t ssrc;
76 int16_t picture_id;
77 int16_t tl0_pic_idx;
78 uint8_t temporal_idx;
79 VideoFrameType frame_type;
80 };
81
ParsePayload(const uint8_t * packet,size_t length,ParsedPacket * parsed) const82 bool ParsePayload(const uint8_t* packet,
83 size_t length,
84 ParsedPacket* parsed) const {
85 RtpPacket rtp_packet;
86 EXPECT_TRUE(rtp_packet.Parse(packet, length));
87 EXPECT_TRUE(rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[0] ||
88 rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[1] ||
89 rtp_packet.Ssrc() == test::CallTest::kVideoSendSsrcs[2])
90 << "Unknown SSRC sent.";
91
92 if (rtp_packet.payload_size() == 0) {
93 return false; // Padding packet.
94 }
95
96 parsed->timestamp = rtp_packet.Timestamp();
97 parsed->ssrc = rtp_packet.Ssrc();
98
99 absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
100 depacketizer_->Parse(rtp_packet.PayloadBuffer());
101 EXPECT_TRUE(parsed_payload);
102
103 if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
104 &parsed_payload->video_header.video_type_header)) {
105 parsed->picture_id = vp8_header->pictureId;
106 parsed->tl0_pic_idx = vp8_header->tl0PicIdx;
107 parsed->temporal_idx = vp8_header->temporalIdx;
108 } else if (const auto* vp9_header = absl::get_if<RTPVideoHeaderVP9>(
109 &parsed_payload->video_header.video_type_header)) {
110 parsed->picture_id = vp9_header->picture_id;
111 parsed->tl0_pic_idx = vp9_header->tl0_pic_idx;
112 parsed->temporal_idx = vp9_header->temporal_idx;
113 } else {
114 RTC_DCHECK_NOTREACHED();
115 }
116
117 parsed->frame_type = parsed_payload->video_header.frame_type;
118 return true;
119 }
120
121 // Verify continuity and monotonicity of picture_id sequence.
VerifyPictureId(const ParsedPacket & current,const ParsedPacket & last) const122 void VerifyPictureId(const ParsedPacket& current,
123 const ParsedPacket& last) const
124 RTC_EXCLUSIVE_LOCKS_REQUIRED(&mutex_) {
125 if (current.timestamp == last.timestamp) {
126 EXPECT_EQ(last.picture_id, current.picture_id);
127 return; // Same frame.
128 }
129
130 // Packet belongs to a new frame.
131 // Picture id should be increasing.
132 EXPECT_TRUE((AheadOf<uint16_t, kPictureIdWraparound>(current.picture_id,
133 last.picture_id)));
134
135 // Expect continuously increasing picture id.
136 int diff = ForwardDiff<uint16_t, kPictureIdWraparound>(last.picture_id,
137 current.picture_id);
138 EXPECT_LE(diff - 1, max_expected_picture_id_gap_);
139 if (diff > 2) {
140 // If the VideoSendStream is destroyed, any frames still in queue is lost.
141 // This can result in a two-frame gap, which will result in logs like
142 // "packet transmission failed, no matching RTP module found, or
143 // transmission error".
144 // A larger gap is only possible for first frame after a recreation, i.e.
145 // key frames.
146 EXPECT_EQ(VideoFrameType::kVideoFrameKey, current.frame_type);
147 }
148 }
149
VerifyTl0Idx(const ParsedPacket & current,const ParsedPacket & last) const150 void VerifyTl0Idx(const ParsedPacket& current, const ParsedPacket& last) const
151 RTC_EXCLUSIVE_LOCKS_REQUIRED(&mutex_) {
152 if (current.tl0_pic_idx == kNoTl0PicIdx ||
153 current.temporal_idx == kNoTemporalIdx) {
154 return; // No temporal layers.
155 }
156
157 if (current.timestamp == last.timestamp || current.temporal_idx != 0) {
158 EXPECT_EQ(last.tl0_pic_idx, current.tl0_pic_idx);
159 return;
160 }
161
162 // New frame with `temporal_idx` 0.
163 // `tl0_pic_idx` should be increasing.
164 EXPECT_TRUE(AheadOf<uint8_t>(current.tl0_pic_idx, last.tl0_pic_idx));
165
166 // Expect continuously increasing idx.
167 int diff = ForwardDiff<uint8_t>(last.tl0_pic_idx, current.tl0_pic_idx);
168 if (diff > 1) {
169 // If the VideoSendStream is destroyed, any frames still in queue is lost.
170 // Gaps only possible for first frame after a recreation, i.e. key frames.
171 EXPECT_EQ(VideoFrameType::kVideoFrameKey, current.frame_type);
172 EXPECT_LE(diff - 1, max_expected_tl0_idx_gap_);
173 }
174 }
175
OnSendRtp(const uint8_t * packet,size_t length)176 Action OnSendRtp(const uint8_t* packet, size_t length) override {
177 MutexLock lock(&mutex_);
178
179 ParsedPacket parsed;
180 if (!ParsePayload(packet, length, &parsed))
181 return SEND_PACKET;
182
183 uint32_t ssrc = parsed.ssrc;
184 if (last_observed_packet_.find(ssrc) != last_observed_packet_.end()) {
185 // Compare to last packet.
186 VerifyPictureId(parsed, last_observed_packet_[ssrc]);
187 VerifyTl0Idx(parsed, last_observed_packet_[ssrc]);
188 }
189
190 last_observed_packet_[ssrc] = parsed;
191
192 // Pass the test when enough media packets have been received on all
193 // streams.
194 if (++num_packets_sent_[ssrc] >= kMinPacketsToObserve &&
195 observed_ssrcs_.find(ssrc) == observed_ssrcs_.end()) {
196 observed_ssrcs_.insert(ssrc);
197 if (observed_ssrcs_.size() == num_ssrcs_to_observe_) {
198 observation_complete_.Set();
199 }
200 }
201 return SEND_PACKET;
202 }
203
204 Mutex mutex_;
205 const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
206 std::map<uint32_t, ParsedPacket> last_observed_packet_ RTC_GUARDED_BY(mutex_);
207 std::map<uint32_t, size_t> num_packets_sent_ RTC_GUARDED_BY(mutex_);
208 int max_expected_picture_id_gap_ RTC_GUARDED_BY(mutex_);
209 int max_expected_tl0_idx_gap_ RTC_GUARDED_BY(mutex_);
210 size_t num_ssrcs_to_observe_ RTC_GUARDED_BY(mutex_);
211 std::set<uint32_t> observed_ssrcs_ RTC_GUARDED_BY(mutex_);
212 };
213
214 class PictureIdTest : public test::CallTest,
215 public ::testing::WithParamInterface<size_t> {
216 public:
PictureIdTest()217 PictureIdTest() : num_temporal_layers_(GetParam()) {}
218
~PictureIdTest()219 virtual ~PictureIdTest() {
220 SendTask(task_queue(), [this]() {
221 send_transport_.reset();
222 receive_transport_.reset();
223 DestroyCalls();
224 });
225 }
226
227 void SetupEncoder(VideoEncoderFactory* encoder_factory,
228 const std::string& payload_name);
229 void SetVideoEncoderConfig(int num_streams);
230 void TestPictureIdContinuousAfterReconfigure(
231 const std::vector<int>& ssrc_counts);
232 void TestPictureIdIncreaseAfterRecreateStreams(
233 const std::vector<int>& ssrc_counts);
234
235 private:
236 const size_t num_temporal_layers_;
237 std::unique_ptr<PictureIdObserver> observer_;
238 };
239
240 // TODO(bugs.webrtc.org/13725): Enable on android when flakiness fixed.
241 #if defined(WEBRTC_ANDROID)
242 #define MAYBE_TemporalLayers DISABLED_TemporalLayers
243 #else
244 #define MAYBE_TemporalLayers TemporalLayers
245 #endif
246
247 INSTANTIATE_TEST_SUITE_P(MAYBE_TemporalLayers,
248 PictureIdTest,
249 ::testing::ValuesIn(kNumTemporalLayers));
250
SetupEncoder(VideoEncoderFactory * encoder_factory,const std::string & payload_name)251 void PictureIdTest::SetupEncoder(VideoEncoderFactory* encoder_factory,
252 const std::string& payload_name) {
253 observer_.reset(
254 new PictureIdObserver(PayloadStringToCodecType(payload_name)));
255
256 SendTask(
257 task_queue(), [this, encoder_factory, payload_name]() {
258 CreateCalls();
259
260 send_transport_.reset(new test::PacketTransport(
261 task_queue(), sender_call_.get(), observer_.get(),
262 test::PacketTransport::kSender, payload_type_map_,
263 std::make_unique<FakeNetworkPipe>(
264 Clock::GetRealTimeClock(),
265 std::make_unique<SimulatedNetwork>(
266 BuiltInNetworkBehaviorConfig()))));
267
268 CreateSendConfig(kNumSimulcastStreams, 0, 0, send_transport_.get());
269 GetVideoSendConfig()->encoder_settings.encoder_factory =
270 encoder_factory;
271 GetVideoSendConfig()->rtp.payload_name = payload_name;
272 GetVideoEncoderConfig()->codec_type =
273 PayloadStringToCodecType(payload_name);
274 SetVideoEncoderConfig(/* number_of_streams */ 1);
275 });
276 }
277
SetVideoEncoderConfig(int num_streams)278 void PictureIdTest::SetVideoEncoderConfig(int num_streams) {
279 GetVideoEncoderConfig()->number_of_streams = num_streams;
280 GetVideoEncoderConfig()->max_bitrate_bps = kEncoderBitrateBps;
281
282 // Always divide the same total bitrate across all streams so that sending a
283 // single stream avoids lowering the bitrate estimate and requiring a
284 // subsequent rampup.
285 const int encoder_stream_bps = kEncoderBitrateBps / num_streams;
286 double scale_factor = 1.0;
287 for (int i = num_streams - 1; i >= 0; --i) {
288 VideoStream& stream = GetVideoEncoderConfig()->simulcast_layers[i];
289 // Reduce the min bitrate by 10% to account for overhead that might
290 // otherwise cause streams to not be enabled.
291 stream.min_bitrate_bps = static_cast<int>(encoder_stream_bps * 0.9);
292 stream.target_bitrate_bps = encoder_stream_bps;
293 stream.max_bitrate_bps = encoder_stream_bps;
294 stream.num_temporal_layers = num_temporal_layers_;
295 stream.scale_resolution_down_by = scale_factor;
296 scale_factor *= 2.0;
297 }
298 }
299
TestPictureIdContinuousAfterReconfigure(const std::vector<int> & ssrc_counts)300 void PictureIdTest::TestPictureIdContinuousAfterReconfigure(
301 const std::vector<int>& ssrc_counts) {
302 SendTask(task_queue(), [this]() {
303 CreateVideoStreams();
304 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
305
306 // Initial test with a single stream.
307 Start();
308 });
309
310 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
311
312 // Reconfigure VideoEncoder and test picture id increase.
313 // Expect continuously increasing picture id, equivalent to no gaps.
314 observer_->SetMaxExpectedPictureIdGap(0);
315 for (int ssrc_count : ssrc_counts) {
316 SetVideoEncoderConfig(ssrc_count);
317 observer_->SetExpectedSsrcs(ssrc_count);
318 observer_->ResetObservedSsrcs();
319 // Make sure the picture_id sequence is continuous on reinit and recreate.
320 SendTask(task_queue(), [this]() {
321 GetVideoSendStream()->ReconfigureVideoEncoder(
322 GetVideoEncoderConfig()->Copy());
323 });
324 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
325 }
326
327 SendTask(task_queue(), [this]() {
328 Stop();
329 DestroyStreams();
330 });
331 }
332
TestPictureIdIncreaseAfterRecreateStreams(const std::vector<int> & ssrc_counts)333 void PictureIdTest::TestPictureIdIncreaseAfterRecreateStreams(
334 const std::vector<int>& ssrc_counts) {
335 SendTask(task_queue(), [this]() {
336 CreateVideoStreams();
337 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
338
339 // Initial test with a single stream.
340 Start();
341 });
342
343 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
344
345 // Recreate VideoSendStream and test picture id increase.
346 // When the VideoSendStream is destroyed, any frames still in queue is lost
347 // with it, therefore it is expected that some frames might be lost.
348 observer_->SetMaxExpectedPictureIdGap(kMaxFramesLost);
349 for (int ssrc_count : ssrc_counts) {
350 SendTask(task_queue(), [this, &ssrc_count]() {
351 DestroyVideoSendStreams();
352
353 SetVideoEncoderConfig(ssrc_count);
354 observer_->SetExpectedSsrcs(ssrc_count);
355 observer_->ResetObservedSsrcs();
356
357 CreateVideoSendStreams();
358 GetVideoSendStream()->Start();
359 CreateFrameGeneratorCapturer(kFrameRate, kFrameMaxWidth, kFrameMaxHeight);
360 });
361
362 EXPECT_TRUE(observer_->Wait()) << "Timed out waiting for packets.";
363 }
364
365 SendTask(task_queue(), [this]() {
366 Stop();
367 DestroyStreams();
368 });
369 }
370
TEST_P(PictureIdTest,ContinuousAfterReconfigureVp8)371 TEST_P(PictureIdTest, ContinuousAfterReconfigureVp8) {
372 test::FunctionVideoEncoderFactory encoder_factory(
373 []() { return VP8Encoder::Create(); });
374 SetupEncoder(&encoder_factory, "VP8");
375 TestPictureIdContinuousAfterReconfigure({1, 3, 3, 1, 1});
376 }
377
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamVp8)378 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamVp8) {
379 test::FunctionVideoEncoderFactory encoder_factory(
380 []() { return VP8Encoder::Create(); });
381 SetupEncoder(&encoder_factory, "VP8");
382 TestPictureIdIncreaseAfterRecreateStreams({1, 3, 3, 1, 1});
383 }
384
TEST_P(PictureIdTest,ContinuousAfterStreamCountChangeVp8)385 TEST_P(PictureIdTest, ContinuousAfterStreamCountChangeVp8) {
386 test::FunctionVideoEncoderFactory encoder_factory(
387 []() { return VP8Encoder::Create(); });
388 // Make sure that the picture id is not reset if the stream count goes
389 // down and then up.
390 SetupEncoder(&encoder_factory, "VP8");
391 TestPictureIdContinuousAfterReconfigure({3, 1, 3});
392 }
393
TEST_P(PictureIdTest,ContinuousAfterReconfigureSimulcastEncoderAdapter)394 TEST_P(PictureIdTest, ContinuousAfterReconfigureSimulcastEncoderAdapter) {
395 InternalEncoderFactory internal_encoder_factory;
396 test::FunctionVideoEncoderFactory encoder_factory(
397 [&internal_encoder_factory]() {
398 return std::make_unique<SimulcastEncoderAdapter>(
399 &internal_encoder_factory, SdpVideoFormat("VP8"));
400 });
401 SetupEncoder(&encoder_factory, "VP8");
402 TestPictureIdContinuousAfterReconfigure({1, 3, 3, 1, 1});
403 }
404
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamSimulcastEncoderAdapter)405 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamSimulcastEncoderAdapter) {
406 InternalEncoderFactory internal_encoder_factory;
407 test::FunctionVideoEncoderFactory encoder_factory(
408 [&internal_encoder_factory]() {
409 return std::make_unique<SimulcastEncoderAdapter>(
410 &internal_encoder_factory, SdpVideoFormat("VP8"));
411 });
412 SetupEncoder(&encoder_factory, "VP8");
413 TestPictureIdIncreaseAfterRecreateStreams({1, 3, 3, 1, 1});
414 }
415
TEST_P(PictureIdTest,ContinuousAfterStreamCountChangeSimulcastEncoderAdapter)416 TEST_P(PictureIdTest, ContinuousAfterStreamCountChangeSimulcastEncoderAdapter) {
417 InternalEncoderFactory internal_encoder_factory;
418 test::FunctionVideoEncoderFactory encoder_factory(
419 [&internal_encoder_factory]() {
420 return std::make_unique<SimulcastEncoderAdapter>(
421 &internal_encoder_factory, SdpVideoFormat("VP8"));
422 });
423 // Make sure that the picture id is not reset if the stream count goes
424 // down and then up.
425 SetupEncoder(&encoder_factory, "VP8");
426 TestPictureIdContinuousAfterReconfigure({3, 1, 3});
427 }
428
TEST_P(PictureIdTest,IncreasingAfterRecreateStreamVp9)429 TEST_P(PictureIdTest, IncreasingAfterRecreateStreamVp9) {
430 test::FunctionVideoEncoderFactory encoder_factory(
431 []() { return VP9Encoder::Create(); });
432 SetupEncoder(&encoder_factory, "VP9");
433 TestPictureIdIncreaseAfterRecreateStreams({1, 1});
434 }
435
436 } // namespace webrtc
437