1 /*
2 * Copyright 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <algorithm>
12
13 #include "media/engine/webrtc_video_engine.h"
14 #include "modules/video_coding/svc/scalability_mode_util.h"
15 #include "rtc_base/experiments/encoder_info_settings.h"
16 #include "test/call_test.h"
17 #include "test/fake_encoder.h"
18 #include "test/field_trial.h"
19 #include "test/gtest.h"
20 #include "test/video_encoder_proxy_factory.h"
21 #include "video/config/encoder_stream_factory.h"
22
23 namespace webrtc {
24 namespace test {
25 namespace {
SetEncoderSpecific(VideoEncoderConfig * encoder_config,VideoCodecType type,size_t num_spatial_layers)26 void SetEncoderSpecific(VideoEncoderConfig* encoder_config,
27 VideoCodecType type,
28 size_t num_spatial_layers) {
29 if (type == kVideoCodecVP9) {
30 VideoCodecVP9 vp9 = VideoEncoder::GetDefaultVp9Settings();
31 vp9.numberOfSpatialLayers = num_spatial_layers;
32 encoder_config->encoder_specific_settings =
33 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
34 vp9);
35 }
36 }
37
38 struct BitrateLimits {
39 DataRate min = DataRate::Zero();
40 DataRate max = DataRate::Zero();
41 };
42
GetLayerBitrateLimits(int pixels,const VideoCodec & codec)43 BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) {
44 if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
45 for (size_t i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
46 if (codec.spatialLayers[i].width * codec.spatialLayers[i].height ==
47 pixels) {
48 return {DataRate::KilobitsPerSec(codec.spatialLayers[i].minBitrate),
49 DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate)};
50 }
51 }
52 } else {
53 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
54 if (codec.simulcastStream[i].width * codec.simulcastStream[i].height ==
55 pixels) {
56 return {DataRate::KilobitsPerSec(codec.simulcastStream[i].minBitrate),
57 DataRate::KilobitsPerSec(codec.simulcastStream[i].maxBitrate)};
58 }
59 }
60 }
61 ADD_FAILURE();
62 return BitrateLimits();
63 }
64
65 } // namespace
66
67 class ResolutionBitrateLimitsWithScalabilityModeTest : public test::CallTest {};
68
69 class ResolutionBitrateLimitsTest
70 : public test::CallTest,
71 public ::testing::WithParamInterface<std::string> {
72 public:
ResolutionBitrateLimitsTest()73 ResolutionBitrateLimitsTest() : payload_name_(GetParam()) {}
74
75 const std::string payload_name_;
76 };
77
78 INSTANTIATE_TEST_SUITE_P(PayloadName,
79 ResolutionBitrateLimitsTest,
80 ::testing::Values("VP8", "VP9"),
__anonc952da880202(const ::testing::TestParamInfo<std::string>& info) 81 [](const ::testing::TestParamInfo<std::string>& info) {
82 return info.param;
83 });
84
85 class InitEncodeTest : public test::EndToEndTest,
86 public test::FrameGeneratorCapturer::SinkWantsObserver,
87 public test::FakeEncoder {
88 public:
89 struct Bitrate {
90 const absl::optional<DataRate> min;
91 const absl::optional<DataRate> max;
92 };
93 struct TestConfig {
94 const bool active;
95 const Bitrate bitrate;
96 const absl::optional<ScalabilityMode> scalability_mode;
97 };
98 struct Expectation {
99 const uint32_t pixels = 0;
100 const Bitrate eq_bitrate;
101 const Bitrate ne_bitrate;
102 };
103
InitEncodeTest(const std::string & payload_name,const std::vector<TestConfig> & configs,const std::vector<Expectation> & expectations)104 InitEncodeTest(const std::string& payload_name,
105 const std::vector<TestConfig>& configs,
106 const std::vector<Expectation>& expectations)
107 : EndToEndTest(test::CallTest::kDefaultTimeout),
108 FakeEncoder(Clock::GetRealTimeClock()),
109 encoder_factory_(this),
110 payload_name_(payload_name),
111 configs_(configs),
112 expectations_(expectations) {}
113
OnFrameGeneratorCapturerCreated(test::FrameGeneratorCapturer * frame_generator_capturer)114 void OnFrameGeneratorCapturerCreated(
115 test::FrameGeneratorCapturer* frame_generator_capturer) override {
116 frame_generator_capturer->SetSinkWantsObserver(this);
117 // Set initial resolution.
118 frame_generator_capturer->ChangeResolution(1280, 720);
119 }
120
OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame> * sink,const rtc::VideoSinkWants & wants)121 void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
122 const rtc::VideoSinkWants& wants) override {}
123
GetNumVideoStreams() const124 size_t GetNumVideoStreams() const override {
125 return (payload_name_ == "VP9") ? 1 : configs_.size();
126 }
127
ModifyVideoConfigs(VideoSendStream::Config * send_config,std::vector<VideoReceiveStreamInterface::Config> * receive_configs,VideoEncoderConfig * encoder_config)128 void ModifyVideoConfigs(
129 VideoSendStream::Config* send_config,
130 std::vector<VideoReceiveStreamInterface::Config>* receive_configs,
131 VideoEncoderConfig* encoder_config) override {
132 webrtc::VideoEncoder::EncoderInfo encoder_info;
133 send_config->encoder_settings.encoder_factory = &encoder_factory_;
134 send_config->rtp.payload_name = payload_name_;
135 send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
136 const VideoCodecType codec_type = PayloadStringToCodecType(payload_name_);
137 encoder_config->codec_type = codec_type;
138 encoder_config->video_stream_factory =
139 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
140 payload_name_, /*max qp*/ 0, /*screencast*/ false,
141 /*screenshare enabled*/ false, encoder_info);
142 encoder_config->max_bitrate_bps = -1;
143 if (configs_.size() == 1 && configs_[0].bitrate.max)
144 encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps();
145 if (payload_name_ == "VP9") {
146 // Simulcast layers indicates which spatial layers are active.
147 encoder_config->simulcast_layers.resize(configs_.size());
148 }
149 double scale_factor = 1.0;
150 for (int i = configs_.size() - 1; i >= 0; --i) {
151 VideoStream& stream = encoder_config->simulcast_layers[i];
152 stream.active = configs_[i].active;
153 stream.scalability_mode = configs_[i].scalability_mode;
154 if (configs_[i].bitrate.min)
155 stream.min_bitrate_bps = configs_[i].bitrate.min->bps();
156 if (configs_[i].bitrate.max)
157 stream.max_bitrate_bps = configs_[i].bitrate.max->bps();
158 stream.scale_resolution_down_by = scale_factor;
159 scale_factor *= (payload_name_ == "VP9") ? 1.0 : 2.0;
160 }
161 SetEncoderSpecific(encoder_config, codec_type, configs_.size());
162 }
163
InitEncode(const VideoCodec * codec,const Settings & settings)164 int32_t InitEncode(const VideoCodec* codec,
165 const Settings& settings) override {
166 for (const auto& expected : expectations_) {
167 BitrateLimits limits = GetLayerBitrateLimits(expected.pixels, *codec);
168 if (expected.eq_bitrate.min)
169 EXPECT_EQ(*expected.eq_bitrate.min, limits.min);
170 if (expected.eq_bitrate.max)
171 EXPECT_EQ(*expected.eq_bitrate.max, limits.max);
172 EXPECT_NE(expected.ne_bitrate.min, limits.min);
173 EXPECT_NE(expected.ne_bitrate.max, limits.max);
174 }
175 observation_complete_.Set();
176 return 0;
177 }
178
GetEncoderInfo() const179 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
180 EncoderInfo info = FakeEncoder::GetEncoderInfo();
181 if (!encoder_info_override_.resolution_bitrate_limits().empty()) {
182 info.resolution_bitrate_limits =
183 encoder_info_override_.resolution_bitrate_limits();
184 }
185 return info;
186 }
187
PerformTest()188 void PerformTest() override {
189 ASSERT_TRUE(Wait()) << "Timed out while waiting for InitEncode() call.";
190 }
191
192 private:
193 test::VideoEncoderProxyFactory encoder_factory_;
194 const std::string payload_name_;
195 const std::vector<TestConfig> configs_;
196 const std::vector<Expectation> expectations_;
197 const LibvpxVp8EncoderInfoSettings encoder_info_override_;
198 };
199
TEST_P(ResolutionBitrateLimitsTest,LimitsApplied)200 TEST_P(ResolutionBitrateLimitsTest, LimitsApplied) {
201 webrtc::test::ScopedFieldTrials field_trials(
202 "WebRTC-GetEncoderInfoOverride/"
203 "frame_size_pixels:921600,"
204 "min_start_bitrate_bps:0,"
205 "min_bitrate_bps:32000,"
206 "max_bitrate_bps:3333000/");
207
208 InitEncodeTest test(payload_name_, {{.active = true}},
209 // Expectations:
210 {{.pixels = 1280 * 720,
211 .eq_bitrate = {DataRate::KilobitsPerSec(32),
212 DataRate::KilobitsPerSec(3333)}}});
213 RunBaseTest(&test);
214 }
215
TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,OneStreamLimitsAppliedForOneSpatialLayer)216 TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
217 OneStreamLimitsAppliedForOneSpatialLayer) {
218 webrtc::test::ScopedFieldTrials field_trials(
219 "WebRTC-GetEncoderInfoOverride/"
220 "frame_size_pixels:921600,"
221 "min_start_bitrate_bps:0,"
222 "min_bitrate_bps:32000,"
223 "max_bitrate_bps:3333000/");
224
225 InitEncodeTest test(
226 "VP9", {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}},
227 // Expectations:
228 {{.pixels = 1280 * 720,
229 .eq_bitrate = {DataRate::KilobitsPerSec(32),
230 DataRate::KilobitsPerSec(3333)}}});
231 RunBaseTest(&test);
232 }
233
TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,OneStreamLimitsNotAppliedForMultipleSpatialLayers)234 TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
235 OneStreamLimitsNotAppliedForMultipleSpatialLayers) {
236 webrtc::test::ScopedFieldTrials field_trials(
237 "WebRTC-GetEncoderInfoOverride/"
238 "frame_size_pixels:230400|921600,"
239 "min_start_bitrate_bps:0|0,"
240 "min_bitrate_bps:21000|32000,"
241 "max_bitrate_bps:2222000|3333000/");
242
243 InitEncodeTest test(
244 "VP9", {{.active = true, .scalability_mode = ScalabilityMode::kL2T1}},
245 // Expectations:
246 {{.pixels = 640 * 360,
247 .ne_bitrate = {DataRate::KilobitsPerSec(31),
248 DataRate::KilobitsPerSec(2222)}},
249 {.pixels = 1280 * 720,
250 .ne_bitrate = {DataRate::KilobitsPerSec(32),
251 DataRate::KilobitsPerSec(3333)}}});
252 RunBaseTest(&test);
253 }
254
TEST_P(ResolutionBitrateLimitsTest,EncodingsApplied)255 TEST_P(ResolutionBitrateLimitsTest, EncodingsApplied) {
256 InitEncodeTest test(payload_name_,
257 {{.active = true,
258 .bitrate = {DataRate::KilobitsPerSec(22),
259 DataRate::KilobitsPerSec(3555)}}},
260 // Expectations:
261 {{.pixels = 1280 * 720,
262 .eq_bitrate = {DataRate::KilobitsPerSec(22),
263 DataRate::KilobitsPerSec(3555)}}});
264 RunBaseTest(&test);
265 }
266
TEST_P(ResolutionBitrateLimitsTest,IntersectionApplied)267 TEST_P(ResolutionBitrateLimitsTest, IntersectionApplied) {
268 webrtc::test::ScopedFieldTrials field_trials(
269 "WebRTC-GetEncoderInfoOverride/"
270 "frame_size_pixels:921600,"
271 "min_start_bitrate_bps:0,"
272 "min_bitrate_bps:32000,"
273 "max_bitrate_bps:3333000/");
274
275 InitEncodeTest test(payload_name_,
276 {{.active = true,
277 .bitrate = {DataRate::KilobitsPerSec(22),
278 DataRate::KilobitsPerSec(1555)}}},
279 // Expectations:
280 {{.pixels = 1280 * 720,
281 .eq_bitrate = {DataRate::KilobitsPerSec(32),
282 DataRate::KilobitsPerSec(1555)}}});
283 RunBaseTest(&test);
284 }
285
TEST_P(ResolutionBitrateLimitsTest,LimitsAppliedMiddleActive)286 TEST_P(ResolutionBitrateLimitsTest, LimitsAppliedMiddleActive) {
287 webrtc::test::ScopedFieldTrials field_trials(
288 "WebRTC-GetEncoderInfoOverride/"
289 "frame_size_pixels:230400|921600,"
290 "min_start_bitrate_bps:0|0,"
291 "min_bitrate_bps:21000|32000,"
292 "max_bitrate_bps:2222000|3333000/");
293
294 InitEncodeTest test(payload_name_,
295 {{.active = false}, {.active = true}, {.active = false}},
296 // Expectations:
297 {{.pixels = 640 * 360,
298 .eq_bitrate = {DataRate::KilobitsPerSec(21),
299 DataRate::KilobitsPerSec(2222)}}});
300 RunBaseTest(&test);
301 }
302
TEST_P(ResolutionBitrateLimitsTest,IntersectionAppliedMiddleActive)303 TEST_P(ResolutionBitrateLimitsTest, IntersectionAppliedMiddleActive) {
304 webrtc::test::ScopedFieldTrials field_trials(
305 "WebRTC-GetEncoderInfoOverride/"
306 "frame_size_pixels:230400|921600,"
307 "min_start_bitrate_bps:0|0,"
308 "min_bitrate_bps:31000|32000,"
309 "max_bitrate_bps:2222000|3333000/");
310
311 InitEncodeTest test(payload_name_,
312 {{.active = false},
313 {.active = true,
314 .bitrate = {DataRate::KilobitsPerSec(30),
315 DataRate::KilobitsPerSec(1555)}},
316 {.active = false}},
317 // Expectations:
318 {{.pixels = 640 * 360,
319 .eq_bitrate = {DataRate::KilobitsPerSec(31),
320 DataRate::KilobitsPerSec(1555)}}});
321 RunBaseTest(&test);
322 }
323
TEST_P(ResolutionBitrateLimitsTest,DefaultLimitsAppliedMiddleActive)324 TEST_P(ResolutionBitrateLimitsTest, DefaultLimitsAppliedMiddleActive) {
325 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
326 kDefaultSinglecastLimits360p =
327 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
328 PayloadStringToCodecType(payload_name_), 640 * 360);
329
330 InitEncodeTest test(
331 payload_name_, {{.active = false}, {.active = true}, {.active = false}},
332 // Expectations:
333 {{.pixels = 640 * 360,
334 .eq_bitrate = {
335 DataRate::BitsPerSec(kDefaultSinglecastLimits360p->min_bitrate_bps),
336 DataRate::BitsPerSec(
337 kDefaultSinglecastLimits360p->max_bitrate_bps)}}});
338 RunBaseTest(&test);
339 }
340
TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,DefaultLimitsAppliedForOneSpatialLayer)341 TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
342 DefaultLimitsAppliedForOneSpatialLayer) {
343 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
344 kDefaultSinglecastLimits720p =
345 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
346 PayloadStringToCodecType("VP9"), 1280 * 720);
347
348 InitEncodeTest test(
349 "VP9",
350 {{.active = true, .scalability_mode = ScalabilityMode::kL1T3},
351 {.active = false}},
352 // Expectations:
353 {{.pixels = 1280 * 720,
354 .eq_bitrate = {
355 DataRate::BitsPerSec(kDefaultSinglecastLimits720p->min_bitrate_bps),
356 DataRate::BitsPerSec(
357 kDefaultSinglecastLimits720p->max_bitrate_bps)}}});
358 RunBaseTest(&test);
359 }
360
TEST_P(ResolutionBitrateLimitsTest,LimitsAppliedHighestActive)361 TEST_P(ResolutionBitrateLimitsTest, LimitsAppliedHighestActive) {
362 webrtc::test::ScopedFieldTrials field_trials(
363 "WebRTC-GetEncoderInfoOverride/"
364 "frame_size_pixels:230400|921600,"
365 "min_start_bitrate_bps:0|0,"
366 "min_bitrate_bps:31000|32000,"
367 "max_bitrate_bps:2222000|3333000/");
368
369 InitEncodeTest test(payload_name_,
370 {{.active = false}, {.active = false}, {.active = true}},
371 // Expectations:
372 {{.pixels = 1280 * 720,
373 .eq_bitrate = {DataRate::KilobitsPerSec(32),
374 DataRate::KilobitsPerSec(3333)}}});
375 RunBaseTest(&test);
376 }
377
TEST_P(ResolutionBitrateLimitsTest,IntersectionAppliedHighestActive)378 TEST_P(ResolutionBitrateLimitsTest, IntersectionAppliedHighestActive) {
379 webrtc::test::ScopedFieldTrials field_trials(
380 "WebRTC-GetEncoderInfoOverride/"
381 "frame_size_pixels:230400|921600,"
382 "min_start_bitrate_bps:0|0,"
383 "min_bitrate_bps:31000|32000,"
384 "max_bitrate_bps:2222000|3333000/");
385
386 InitEncodeTest test(payload_name_,
387 {{.active = false},
388 {.active = false},
389 {.active = true,
390 .bitrate = {DataRate::KilobitsPerSec(30),
391 DataRate::KilobitsPerSec(1555)}}},
392 // Expectations:
393 {{.pixels = 1280 * 720,
394 .eq_bitrate = {DataRate::KilobitsPerSec(32),
395 DataRate::KilobitsPerSec(1555)}}});
396 RunBaseTest(&test);
397 }
398
TEST_P(ResolutionBitrateLimitsTest,LimitsNotAppliedLowestActive)399 TEST_P(ResolutionBitrateLimitsTest, LimitsNotAppliedLowestActive) {
400 webrtc::test::ScopedFieldTrials field_trials(
401 "WebRTC-GetEncoderInfoOverride/"
402 "frame_size_pixels:230400|921600,"
403 "min_start_bitrate_bps:0|0,"
404 "min_bitrate_bps:31000|32000,"
405 "max_bitrate_bps:2222000|3333000/");
406
407 InitEncodeTest test(payload_name_, {{.active = true}, {.active = false}},
408 // Expectations:
409 {{.pixels = 640 * 360,
410 .ne_bitrate = {DataRate::KilobitsPerSec(31),
411 DataRate::KilobitsPerSec(2222)}},
412 {.pixels = 1280 * 720,
413 .ne_bitrate = {DataRate::KilobitsPerSec(32),
414 DataRate::KilobitsPerSec(3333)}}});
415 RunBaseTest(&test);
416 }
417
TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,LimitsAppliedForVp9OneSpatialLayer)418 TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
419 LimitsAppliedForVp9OneSpatialLayer) {
420 webrtc::test::ScopedFieldTrials field_trials(
421 "WebRTC-GetEncoderInfoOverride/"
422 "frame_size_pixels:230400|921600,"
423 "min_start_bitrate_bps:0|0,"
424 "min_bitrate_bps:31000|32000,"
425 "max_bitrate_bps:2222000|3333000/");
426
427 InitEncodeTest test(
428 "VP9",
429 {{.active = true, .scalability_mode = ScalabilityMode::kL1T1},
430 {.active = false}},
431 // Expectations:
432 {{.pixels = 1280 * 720,
433 .eq_bitrate = {DataRate::KilobitsPerSec(32),
434 DataRate::KilobitsPerSec(3333)}}});
435 RunBaseTest(&test);
436 }
437
TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,LimitsNotAppliedForVp9MultipleSpatialLayers)438 TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
439 LimitsNotAppliedForVp9MultipleSpatialLayers) {
440 webrtc::test::ScopedFieldTrials field_trials(
441 "WebRTC-GetEncoderInfoOverride/"
442 "frame_size_pixels:230400|921600,"
443 "min_start_bitrate_bps:0|0,"
444 "min_bitrate_bps:31000|32000,"
445 "max_bitrate_bps:2222000|3333000/");
446
447 InitEncodeTest test(
448 "VP9",
449 {{.active = true, .scalability_mode = ScalabilityMode::kL2T1},
450 {.active = false}},
451 // Expectations:
452 {{.pixels = 640 * 360,
453 .ne_bitrate = {DataRate::KilobitsPerSec(31),
454 DataRate::KilobitsPerSec(2222)}},
455 {.pixels = 1280 * 720,
456 .ne_bitrate = {DataRate::KilobitsPerSec(32),
457 DataRate::KilobitsPerSec(3333)}}});
458 RunBaseTest(&test);
459 }
460
TEST_P(ResolutionBitrateLimitsTest,LimitsNotAppliedSimulcast)461 TEST_P(ResolutionBitrateLimitsTest, LimitsNotAppliedSimulcast) {
462 webrtc::test::ScopedFieldTrials field_trials(
463 "WebRTC-GetEncoderInfoOverride/"
464 "frame_size_pixels:230400|921600,"
465 "min_start_bitrate_bps:0|0,"
466 "min_bitrate_bps:31000|32000,"
467 "max_bitrate_bps:2222000|3333000/");
468
469 InitEncodeTest test(payload_name_, {{.active = true}, {.active = true}},
470 // Expectations:
471 {{.pixels = 640 * 360,
472 .ne_bitrate = {DataRate::KilobitsPerSec(31),
473 DataRate::KilobitsPerSec(2222)}},
474 {.pixels = 1280 * 720,
475 .ne_bitrate = {DataRate::KilobitsPerSec(32),
476 DataRate::KilobitsPerSec(3333)}}});
477 RunBaseTest(&test);
478 }
479
480 } // namespace test
481 } // namespace webrtc
482