1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "absl/memory/memory.h"
12 #include "api/test/create_frame_generator.h"
13 #include "api/test/frame_generator_interface.h"
14 #include "api/test/mock_video_encoder.h"
15 #include "api/video/color_space.h"
16 #include "api/video/i420_buffer.h"
17 #include "api/video_codecs/video_encoder.h"
18 #include "api/video_codecs/vp9_profile.h"
19 #include "common_video/libyuv/include/webrtc_libyuv.h"
20 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
21 #include "modules/video_coding/codecs/interface/libvpx_interface.h"
22 #include "modules/video_coding/codecs/interface/mock_libvpx_interface.h"
23 #include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
24 #include "modules/video_coding/codecs/test/video_codec_unittest.h"
25 #include "modules/video_coding/codecs/vp9/include/vp9.h"
26 #include "modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h"
27 #include "modules/video_coding/codecs/vp9/svc_config.h"
28 #include "rtc_base/strings/string_builder.h"
29 #include "test/explicit_key_value_config.h"
30 #include "test/field_trial.h"
31 #include "test/gmock.h"
32 #include "test/gtest.h"
33 #include "test/mappable_native_buffer.h"
34 #include "test/video_codec_settings.h"
35
36 namespace webrtc {
37 namespace {
38
39 using ::testing::_;
40 using ::testing::A;
41 using ::testing::AllOf;
42 using ::testing::An;
43 using ::testing::AnyNumber;
44 using ::testing::ByRef;
45 using ::testing::DoAll;
46 using ::testing::Each;
47 using ::testing::ElementsAre;
48 using ::testing::ElementsAreArray;
49 using ::testing::Field;
50 using ::testing::IsEmpty;
51 using ::testing::Mock;
52 using ::testing::NiceMock;
53 using ::testing::Return;
54 using ::testing::SafeMatcherCast;
55 using ::testing::SaveArgPointee;
56 using ::testing::SetArgPointee;
57 using ::testing::SizeIs;
58 using ::testing::TypedEq;
59 using ::testing::UnorderedElementsAreArray;
60 using ::testing::WithArg;
61 using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
62 using FramerateFractions =
63 absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
64
65 constexpr size_t kWidth = 1280;
66 constexpr size_t kHeight = 720;
67
68 const VideoEncoder::Capabilities kCapabilities(false);
69 const VideoEncoder::Settings kSettings(kCapabilities,
70 /*number_of_cores=*/1,
71 /*max_payload_size=*/0);
72
DefaultCodecSettings()73 VideoCodec DefaultCodecSettings() {
74 VideoCodec codec_settings;
75 webrtc::test::CodecSettings(kVideoCodecVP9, &codec_settings);
76 codec_settings.width = kWidth;
77 codec_settings.height = kHeight;
78 codec_settings.VP9()->numberOfTemporalLayers = 1;
79 codec_settings.VP9()->numberOfSpatialLayers = 1;
80 return codec_settings;
81 }
82
ConfigureSvc(VideoCodec & codec_settings,int num_spatial_layers,int num_temporal_layers=1)83 void ConfigureSvc(VideoCodec& codec_settings,
84 int num_spatial_layers,
85 int num_temporal_layers = 1) {
86 codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers;
87 codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers;
88 codec_settings.SetFrameDropEnabled(false);
89
90 std::vector<SpatialLayer> layers = GetSvcConfig(
91 codec_settings.width, codec_settings.height, codec_settings.maxFramerate,
92 /*first_active_layer=*/0, num_spatial_layers, num_temporal_layers, false);
93 for (size_t i = 0; i < layers.size(); ++i) {
94 codec_settings.spatialLayers[i] = layers[i];
95 }
96 }
97
98 } // namespace
99
100 class TestVp9Impl : public VideoCodecUnitTest {
101 protected:
CreateEncoder()102 std::unique_ptr<VideoEncoder> CreateEncoder() override {
103 return VP9Encoder::Create();
104 }
105
CreateDecoder()106 std::unique_ptr<VideoDecoder> CreateDecoder() override {
107 return VP9Decoder::Create();
108 }
109
ModifyCodecSettings(VideoCodec * codec_settings)110 void ModifyCodecSettings(VideoCodec* codec_settings) override {
111 webrtc::test::CodecSettings(kVideoCodecVP9, codec_settings);
112 codec_settings->width = kWidth;
113 codec_settings->height = kHeight;
114 codec_settings->VP9()->numberOfTemporalLayers = 1;
115 codec_settings->VP9()->numberOfSpatialLayers = 1;
116 }
117 };
118
119 class TestVp9ImplForPixelFormat
120 : public TestVp9Impl,
121 public ::testing::WithParamInterface<
122 test::FrameGeneratorInterface::OutputType> {
123 protected:
SetUp()124 void SetUp() override {
125 input_frame_generator_ = test::CreateSquareFrameGenerator(
126 kWidth, kHeight, GetParam(), absl::optional<int>());
127 TestVp9Impl::SetUp();
128 }
129 };
130
131 // Disabled on ios as flake, see https://crbug.com/webrtc/7057
132 #if defined(WEBRTC_IOS)
TEST_P(TestVp9ImplForPixelFormat,DISABLED_EncodeDecode)133 TEST_P(TestVp9ImplForPixelFormat, DISABLED_EncodeDecode) {
134 #else
135 TEST_P(TestVp9ImplForPixelFormat, EncodeDecode) {
136 #endif
137 VideoFrame input_frame = NextInputFrame();
138 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
139 EncodedImage encoded_frame;
140 CodecSpecificInfo codec_specific_info;
141 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
142 // First frame should be a key frame.
143 encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
144 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
145 std::unique_ptr<VideoFrame> decoded_frame;
146 absl::optional<uint8_t> decoded_qp;
147 ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
148 ASSERT_TRUE(decoded_frame);
149 EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
150
151 const ColorSpace color_space = *decoded_frame->color_space();
152 EXPECT_EQ(ColorSpace::PrimaryID::kUnspecified, color_space.primaries());
153 EXPECT_EQ(ColorSpace::TransferID::kUnspecified, color_space.transfer());
154 EXPECT_EQ(ColorSpace::MatrixID::kUnspecified, color_space.matrix());
155 EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range());
156 EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified,
157 color_space.chroma_siting_horizontal());
158 EXPECT_EQ(ColorSpace::ChromaSiting::kUnspecified,
159 color_space.chroma_siting_vertical());
160 }
161
162 TEST_P(TestVp9ImplForPixelFormat, EncodeNativeBuffer) {
163 VideoFrame input_frame = NextInputFrame();
164 // Replace the input frame with a fake native buffer of the same size and
165 // underlying pixel format. Do not allow ToI420() for non-I420 buffers,
166 // ensuring zero-conversion.
167 input_frame = test::CreateMappableNativeFrame(
168 input_frame.ntp_time_ms(), input_frame.video_frame_buffer()->type(),
169 input_frame.width(), input_frame.height());
170 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
171 EncodedImage encoded_frame;
172 CodecSpecificInfo codec_specific_info;
173 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
174
175 // After encoding, we would expect a single mapping to have happened.
176 rtc::scoped_refptr<test::MappableNativeBuffer> mappable_buffer =
177 test::GetMappableNativeBufferFromVideoFrame(input_frame);
178 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_buffers =
179 mappable_buffer->GetMappedFramedBuffers();
180 ASSERT_EQ(mapped_buffers.size(), 1u);
181 EXPECT_EQ(mapped_buffers[0]->type(), mappable_buffer->mappable_type());
182 EXPECT_EQ(mapped_buffers[0]->width(), input_frame.width());
183 EXPECT_EQ(mapped_buffers[0]->height(), input_frame.height());
184 EXPECT_FALSE(mappable_buffer->DidConvertToI420());
185 }
186
187 TEST_P(TestVp9ImplForPixelFormat, DecodedColorSpaceFromBitstream) {
188 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
189 EncodedImage encoded_frame;
190 CodecSpecificInfo codec_specific_info;
191 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
192
193 // Encoded frame without explicit color space information.
194 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
195 std::unique_ptr<VideoFrame> decoded_frame;
196 absl::optional<uint8_t> decoded_qp;
197 ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
198 ASSERT_TRUE(decoded_frame);
199 // Color space present from encoded bitstream.
200 ASSERT_TRUE(decoded_frame->color_space());
201 // No HDR metadata present.
202 EXPECT_FALSE(decoded_frame->color_space()->hdr_metadata());
203 }
204
205 TEST_P(TestVp9ImplForPixelFormat, DecodedQpEqualsEncodedQp) {
206 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
207 EncodedImage encoded_frame;
208 CodecSpecificInfo codec_specific_info;
209 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
210 // First frame should be a key frame.
211 encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
212 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
213 std::unique_ptr<VideoFrame> decoded_frame;
214 absl::optional<uint8_t> decoded_qp;
215 ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
216 ASSERT_TRUE(decoded_frame);
217 ASSERT_TRUE(decoded_qp);
218 EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
219 }
220
221 TEST_F(TestVp9Impl, SwitchInputPixelFormatsWithoutReconfigure) {
222 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
223 EncodedImage encoded_frame;
224 CodecSpecificInfo codec_specific_info;
225 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
226
227 // Change the input frame type from I420 to NV12, encoding should still work.
228 input_frame_generator_ = test::CreateSquareFrameGenerator(
229 kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12,
230 absl::optional<int>());
231 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
232 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
233
234 // Flipping back to I420, encoding should still work.
235 input_frame_generator_ = test::CreateSquareFrameGenerator(
236 kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420,
237 absl::optional<int>());
238 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
239 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
240 }
241
242 TEST(Vp9ImplTest, ParserQpEqualsEncodedQp) {
243 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
244 VideoCodec codec_settings = DefaultCodecSettings();
245 encoder->InitEncode(&codec_settings, kSettings);
246
247 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
248 EncodedVideoFrameProducer(*encoder)
249 .SetNumInputFrames(1)
250 .SetResolution({kWidth, kHeight})
251 .Encode();
252 ASSERT_THAT(frames, SizeIs(1));
253 const auto& encoded_frame = frames.front().encoded_image;
254 int qp = 0;
255 ASSERT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
256 EXPECT_EQ(encoded_frame.qp_, qp);
257 }
258
259 TEST(Vp9ImplTest, EncodeAttachesTemplateStructureWithSvcController) {
260 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
261 VideoCodec codec_settings = DefaultCodecSettings();
262 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
263 WEBRTC_VIDEO_CODEC_OK);
264
265 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
266 EncodedVideoFrameProducer(*encoder)
267 .SetNumInputFrames(2)
268 .SetResolution({kWidth, kHeight})
269 .Encode();
270
271 ASSERT_THAT(frames, SizeIs(2));
272 EXPECT_TRUE(frames[0].codec_specific_info.template_structure);
273 EXPECT_TRUE(frames[0].codec_specific_info.generic_frame_info);
274
275 EXPECT_FALSE(frames[1].codec_specific_info.template_structure);
276 EXPECT_TRUE(frames[1].codec_specific_info.generic_frame_info);
277 }
278
279 TEST(Vp9ImplTest, EncoderWith2TemporalLayers) {
280 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
281 VideoCodec codec_settings = DefaultCodecSettings();
282 codec_settings.VP9()->numberOfTemporalLayers = 2;
283 // Tl0PidIdx is only used in non-flexible mode.
284 codec_settings.VP9()->flexibleMode = false;
285 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
286 WEBRTC_VIDEO_CODEC_OK);
287
288 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
289 EncodedVideoFrameProducer(*encoder)
290 .SetNumInputFrames(4)
291 .SetResolution({kWidth, kHeight})
292 .Encode();
293
294 ASSERT_THAT(frames, SizeIs(4));
295 EXPECT_EQ(frames[0].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
296 EXPECT_EQ(frames[1].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
297 EXPECT_EQ(frames[2].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
298 EXPECT_EQ(frames[3].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
299 }
300
301 TEST(Vp9ImplTest, EncodeTemporalLayersWithSvcController) {
302 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
303 VideoCodec codec_settings = DefaultCodecSettings();
304 codec_settings.VP9()->numberOfTemporalLayers = 2;
305 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
306 WEBRTC_VIDEO_CODEC_OK);
307
308 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
309 EncodedVideoFrameProducer(*encoder)
310 .SetNumInputFrames(4)
311 .SetResolution({kWidth, kHeight})
312 .Encode();
313
314 ASSERT_THAT(frames, SizeIs(4));
315 EXPECT_EQ(frames[0].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
316 EXPECT_EQ(frames[1].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
317 EXPECT_EQ(frames[2].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
318 EXPECT_EQ(frames[3].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
319 // Verify codec agnostic part
320 ASSERT_TRUE(frames[0].codec_specific_info.generic_frame_info);
321 ASSERT_TRUE(frames[1].codec_specific_info.generic_frame_info);
322 ASSERT_TRUE(frames[2].codec_specific_info.generic_frame_info);
323 ASSERT_TRUE(frames[3].codec_specific_info.generic_frame_info);
324 EXPECT_EQ(frames[0].codec_specific_info.generic_frame_info->temporal_id, 0);
325 EXPECT_EQ(frames[1].codec_specific_info.generic_frame_info->temporal_id, 1);
326 EXPECT_EQ(frames[2].codec_specific_info.generic_frame_info->temporal_id, 0);
327 EXPECT_EQ(frames[3].codec_specific_info.generic_frame_info->temporal_id, 1);
328 }
329
330 TEST(Vp9ImplTest, EncoderWith2SpatialLayers) {
331 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
332 VideoCodec codec_settings = DefaultCodecSettings();
333 codec_settings.VP9()->numberOfSpatialLayers = 2;
334 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
335 WEBRTC_VIDEO_CODEC_OK);
336
337 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
338 EncodedVideoFrameProducer(*encoder)
339 .SetNumInputFrames(1)
340 .SetResolution({kWidth, kHeight})
341 .Encode();
342
343 ASSERT_THAT(frames, SizeIs(2));
344 EXPECT_EQ(frames[0].encoded_image.SpatialIndex(), 0);
345 EXPECT_EQ(frames[1].encoded_image.SpatialIndex(), 1);
346 }
347
348 TEST(Vp9ImplTest, EncodeSpatialLayersWithSvcController) {
349 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
350 VideoCodec codec_settings = DefaultCodecSettings();
351 codec_settings.VP9()->numberOfSpatialLayers = 2;
352 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
353 WEBRTC_VIDEO_CODEC_OK);
354
355 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
356 EncodedVideoFrameProducer(*encoder)
357 .SetNumInputFrames(2)
358 .SetResolution({kWidth, kHeight})
359 .Encode();
360
361 ASSERT_THAT(frames, SizeIs(4));
362 EXPECT_EQ(frames[0].encoded_image.SpatialIndex(), 0);
363 EXPECT_EQ(frames[1].encoded_image.SpatialIndex(), 1);
364 EXPECT_EQ(frames[2].encoded_image.SpatialIndex(), 0);
365 EXPECT_EQ(frames[3].encoded_image.SpatialIndex(), 1);
366 // Verify codec agnostic part
367 ASSERT_TRUE(frames[0].codec_specific_info.generic_frame_info);
368 ASSERT_TRUE(frames[1].codec_specific_info.generic_frame_info);
369 ASSERT_TRUE(frames[2].codec_specific_info.generic_frame_info);
370 ASSERT_TRUE(frames[3].codec_specific_info.generic_frame_info);
371 EXPECT_EQ(frames[0].codec_specific_info.generic_frame_info->spatial_id, 0);
372 EXPECT_EQ(frames[1].codec_specific_info.generic_frame_info->spatial_id, 1);
373 EXPECT_EQ(frames[2].codec_specific_info.generic_frame_info->spatial_id, 0);
374 EXPECT_EQ(frames[3].codec_specific_info.generic_frame_info->spatial_id, 1);
375 }
376
377 TEST_F(TestVp9Impl, EncoderExplicitLayering) {
378 // Override default settings.
379 codec_settings_.VP9()->numberOfTemporalLayers = 1;
380 codec_settings_.VP9()->numberOfSpatialLayers = 2;
381
382 codec_settings_.width = 960;
383 codec_settings_.height = 540;
384 codec_settings_.spatialLayers[0].minBitrate = 200;
385 codec_settings_.spatialLayers[0].maxBitrate = 500;
386 codec_settings_.spatialLayers[0].targetBitrate =
387 (codec_settings_.spatialLayers[0].minBitrate +
388 codec_settings_.spatialLayers[0].maxBitrate) /
389 2;
390 codec_settings_.spatialLayers[0].active = true;
391
392 codec_settings_.spatialLayers[1].minBitrate = 400;
393 codec_settings_.spatialLayers[1].maxBitrate = 1500;
394 codec_settings_.spatialLayers[1].targetBitrate =
395 (codec_settings_.spatialLayers[1].minBitrate +
396 codec_settings_.spatialLayers[1].maxBitrate) /
397 2;
398 codec_settings_.spatialLayers[1].active = true;
399
400 codec_settings_.spatialLayers[0].width = codec_settings_.width / 2;
401 codec_settings_.spatialLayers[0].height = codec_settings_.height / 2;
402 codec_settings_.spatialLayers[0].maxFramerate = codec_settings_.maxFramerate;
403 codec_settings_.spatialLayers[1].width = codec_settings_.width;
404 codec_settings_.spatialLayers[1].height = codec_settings_.height;
405 codec_settings_.spatialLayers[1].maxFramerate = codec_settings_.maxFramerate;
406
407 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
408 encoder_->InitEncode(&codec_settings_, kSettings));
409
410 // Ensure it fails if scaling factors in horz/vert dimentions are different.
411 codec_settings_.spatialLayers[0].width = codec_settings_.width;
412 codec_settings_.spatialLayers[0].height = codec_settings_.height / 2;
413 codec_settings_.spatialLayers[1].width = codec_settings_.width;
414 codec_settings_.spatialLayers[1].height = codec_settings_.height;
415 EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_PARAMETER,
416 encoder_->InitEncode(&codec_settings_, kSettings));
417
418 // Ensure it fails if scaling factor is not power of two.
419 codec_settings_.spatialLayers[0].width = codec_settings_.width / 3;
420 codec_settings_.spatialLayers[0].height = codec_settings_.height / 3;
421 codec_settings_.spatialLayers[1].width = codec_settings_.width;
422 codec_settings_.spatialLayers[1].height = codec_settings_.height;
423 EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_PARAMETER,
424 encoder_->InitEncode(&codec_settings_, kSettings));
425 }
426
427 TEST_F(TestVp9Impl, EnableDisableSpatialLayers) {
428 // Configure encoder to produce N spatial layers. Encode frames of layer 0
429 // then enable layer 1 and encode more frames and so on until layer N-1.
430 // Then disable layers one by one in the same way.
431 // Note: bit rate allocation is high to avoid frame dropping due to rate
432 // control, the encoder should always produce a frame. A dropped
433 // frame indicates a problem and the test will fail.
434 const size_t num_spatial_layers = 3;
435 const size_t num_frames_to_encode = 5;
436
437 ConfigureSvc(codec_settings_, num_spatial_layers);
438 codec_settings_.SetFrameDropEnabled(true);
439
440 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
441 encoder_->InitEncode(&codec_settings_, kSettings));
442
443 VideoBitrateAllocation bitrate_allocation;
444 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
445 // Allocate high bit rate to avoid frame dropping due to rate control.
446 bitrate_allocation.SetBitrate(
447 sl_idx, 0,
448 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
449 encoder_->SetRates(VideoEncoder::RateControlParameters(
450 bitrate_allocation, codec_settings_.maxFramerate));
451
452 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
453 SetWaitForEncodedFramesThreshold(sl_idx + 1);
454 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
455 encoder_->Encode(NextInputFrame(), nullptr));
456 std::vector<EncodedImage> encoded_frame;
457 std::vector<CodecSpecificInfo> codec_specific_info;
458 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
459 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
460 frame_num == 0);
461 }
462 }
463
464 for (size_t i = 0; i < num_spatial_layers - 1; ++i) {
465 const size_t sl_idx = num_spatial_layers - i - 1;
466 bitrate_allocation.SetBitrate(sl_idx, 0, 0);
467 encoder_->SetRates(VideoEncoder::RateControlParameters(
468 bitrate_allocation, codec_settings_.maxFramerate));
469
470 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
471 SetWaitForEncodedFramesThreshold(sl_idx);
472 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
473 encoder_->Encode(NextInputFrame(), nullptr));
474 std::vector<EncodedImage> encoded_frame;
475 std::vector<CodecSpecificInfo> codec_specific_info;
476 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
477 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
478 frame_num == 0);
479 }
480 }
481 }
482
483 TEST(Vp9ImplTest, EnableDisableSpatialLayersWithSvcController) {
484 const int num_spatial_layers = 3;
485 // Configure encoder to produce 3 spatial layers. Encode frames of layer 0
486 // then enable layer 1 and encode more frames and so on.
487 // Then disable layers one by one in the same way.
488 // Note: bit rate allocation is high to avoid frame dropping due to rate
489 // control, the encoder should always produce a frame. A dropped
490 // frame indicates a problem and the test will fail.
491 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
492 VideoCodec codec_settings = DefaultCodecSettings();
493 ConfigureSvc(codec_settings, num_spatial_layers);
494 codec_settings.SetFrameDropEnabled(true);
495 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
496 WEBRTC_VIDEO_CODEC_OK);
497
498 EncodedVideoFrameProducer producer(*encoder);
499 producer.SetResolution({kWidth, kHeight});
500
501 // Encode a key frame to validate all other frames are delta frames.
502 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
503 producer.SetNumInputFrames(1).Encode();
504 ASSERT_THAT(frames, Not(IsEmpty()));
505 EXPECT_TRUE(frames[0].codec_specific_info.template_structure);
506
507 const size_t num_frames_to_encode = 5;
508
509 VideoBitrateAllocation bitrate_allocation;
510 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
511 // Allocate high bit rate to avoid frame dropping due to rate control.
512 bitrate_allocation.SetBitrate(
513 sl_idx, 0,
514 codec_settings.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
515 encoder->SetRates(VideoEncoder::RateControlParameters(
516 bitrate_allocation, codec_settings.maxFramerate));
517
518 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
519 // With (sl_idx+1) spatial layers expect (sl_idx+1) frames per input frame.
520 ASSERT_THAT(frames, SizeIs(num_frames_to_encode * (sl_idx + 1)));
521 for (size_t i = 0; i < frames.size(); ++i) {
522 EXPECT_TRUE(frames[i].codec_specific_info.generic_frame_info);
523 EXPECT_FALSE(frames[i].codec_specific_info.template_structure);
524 }
525 }
526
527 for (int sl_idx = num_spatial_layers - 1; sl_idx > 0; --sl_idx) {
528 bitrate_allocation.SetBitrate(sl_idx, 0, 0);
529 encoder->SetRates(VideoEncoder::RateControlParameters(
530 bitrate_allocation, codec_settings.maxFramerate));
531
532 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
533 // With `sl_idx` spatial layer disabled, there are `sl_idx` spatial layers
534 // left.
535 ASSERT_THAT(frames, SizeIs(num_frames_to_encode * sl_idx));
536 for (size_t i = 0; i < frames.size(); ++i) {
537 EXPECT_TRUE(frames[i].codec_specific_info.generic_frame_info);
538 EXPECT_FALSE(frames[i].codec_specific_info.template_structure);
539 }
540 }
541 }
542
543 MATCHER_P2(GenericLayerIs, spatial_id, temporal_id, "") {
544 if (arg.codec_specific_info.generic_frame_info == absl::nullopt) {
545 *result_listener << " miss generic_frame_info";
546 return false;
547 }
548 const auto& layer = *arg.codec_specific_info.generic_frame_info;
549 if (layer.spatial_id != spatial_id || layer.temporal_id != temporal_id) {
550 *result_listener << " frame from layer (" << layer.spatial_id << ", "
551 << layer.temporal_id << ")";
552 return false;
553 }
554 return true;
555 }
556
557 TEST(Vp9ImplTest, SpatialUpswitchNotAtGOFBoundary) {
558 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
559 VideoCodec codec_settings = DefaultCodecSettings();
560 ConfigureSvc(codec_settings, /*num_spatial_layers=*/3,
561 /*num_temporal_layers=*/3);
562 codec_settings.SetFrameDropEnabled(true);
563 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
564 WEBRTC_VIDEO_CODEC_OK);
565
566 EncodedVideoFrameProducer producer(*encoder);
567 producer.SetResolution({kWidth, kHeight});
568
569 // Disable all but spatial_layer = 0;
570 VideoBitrateAllocation bitrate_allocation;
571 int layer_bitrate_bps = codec_settings.spatialLayers[0].targetBitrate * 1000;
572 bitrate_allocation.SetBitrate(0, 0, layer_bitrate_bps);
573 bitrate_allocation.SetBitrate(0, 1, layer_bitrate_bps);
574 bitrate_allocation.SetBitrate(0, 2, layer_bitrate_bps);
575 encoder->SetRates(VideoEncoder::RateControlParameters(
576 bitrate_allocation, codec_settings.maxFramerate));
577 EXPECT_THAT(producer.SetNumInputFrames(3).Encode(),
578 ElementsAre(GenericLayerIs(0, 0), GenericLayerIs(0, 2),
579 GenericLayerIs(0, 1)));
580
581 // Upswitch to spatial_layer = 1
582 layer_bitrate_bps = codec_settings.spatialLayers[1].targetBitrate * 1000;
583 bitrate_allocation.SetBitrate(1, 0, layer_bitrate_bps);
584 bitrate_allocation.SetBitrate(1, 1, layer_bitrate_bps);
585 bitrate_allocation.SetBitrate(1, 2, layer_bitrate_bps);
586 encoder->SetRates(VideoEncoder::RateControlParameters(
587 bitrate_allocation, codec_settings.maxFramerate));
588 // Expect upswitch doesn't happen immediately since there is no S1 frame that
589 // S1T2 frame can reference.
590 EXPECT_THAT(producer.SetNumInputFrames(1).Encode(),
591 ElementsAre(GenericLayerIs(0, 2)));
592 // Expect spatial upswitch happens now, at T0 frame.
593 EXPECT_THAT(producer.SetNumInputFrames(1).Encode(),
594 ElementsAre(GenericLayerIs(0, 0), GenericLayerIs(1, 0)));
595 }
596 // TODO(bugs.webrtc.org/13442) Enable once a forward fix has landed in WebRTC.
597 TEST_F(TestVp9Impl, DISABLED_DisableEnableBaseLayerTriggersKeyFrame) {
598 // Configure encoder to produce N spatial layers. Encode frames for all
599 // layers. Then disable all but the last layer. Then reenable all back again.
600 test::ScopedFieldTrials override_field_trials(
601 "WebRTC-Vp9ExternalRefCtrl/Enabled/");
602 const size_t num_spatial_layers = 3;
603 const size_t num_temporal_layers = 3;
604 // Must not be multiple of temporal period to exercise all code paths.
605 const size_t num_frames_to_encode = 5;
606
607 ConfigureSvc(codec_settings_, num_spatial_layers, num_temporal_layers);
608 codec_settings_.SetFrameDropEnabled(false);
609 codec_settings_.VP9()->flexibleMode = false;
610 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOnKeyPic;
611 codec_settings_.mode = VideoCodecMode::kRealtimeVideo;
612
613 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
614 encoder_->InitEncode(&codec_settings_, kSettings));
615
616 VideoBitrateAllocation bitrate_allocation;
617 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
618 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
619 // Allocate high bit rate to avoid frame dropping due to rate control.
620 bitrate_allocation.SetBitrate(
621 sl_idx, tl_idx,
622 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
623 }
624 }
625 encoder_->SetRates(VideoEncoder::RateControlParameters(
626 bitrate_allocation, codec_settings_.maxFramerate));
627
628 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
629 SetWaitForEncodedFramesThreshold(num_spatial_layers);
630 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
631 encoder_->Encode(NextInputFrame(), nullptr));
632 std::vector<EncodedImage> encoded_frame;
633 std::vector<CodecSpecificInfo> codec_specific_info;
634 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
635 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
636 frame_num == 0);
637 }
638
639 // Disable all but top layer.
640 for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
641 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
642 bitrate_allocation.SetBitrate(sl_idx, tl_idx, 0);
643 }
644 }
645 encoder_->SetRates(VideoEncoder::RateControlParameters(
646 bitrate_allocation, codec_settings_.maxFramerate));
647
648 bool seen_ss_data = false;
649 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
650 SetWaitForEncodedFramesThreshold(1);
651 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
652 encoder_->Encode(NextInputFrame(), nullptr));
653 std::vector<EncodedImage> encoded_frame;
654 std::vector<CodecSpecificInfo> codec_specific_info;
655 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
656 // SS available immediatly after switching on base temporal layer.
657 if (seen_ss_data) {
658 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
659 false);
660 } else {
661 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
662 codec_specific_info[0].codecSpecific.VP9.temporal_idx == 0);
663 seen_ss_data |=
664 codec_specific_info[0].codecSpecific.VP9.ss_data_available;
665 }
666 // No key-frames generated for disabling layers.
667 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
668 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 2);
669 }
670 EXPECT_TRUE(seen_ss_data);
671
672 // Force key-frame.
673 std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
674 SetWaitForEncodedFramesThreshold(1);
675 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
676 encoder_->Encode(NextInputFrame(), &frame_types));
677 std::vector<EncodedImage> encoded_frame;
678 std::vector<CodecSpecificInfo> codec_specific_info;
679 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
680 // Key-frame should be produced.
681 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey);
682 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 2);
683
684 // Encode some more frames.
685 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
686 SetWaitForEncodedFramesThreshold(1);
687 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
688 encoder_->Encode(NextInputFrame(), nullptr));
689 std::vector<EncodedImage> encoded_frame;
690 std::vector<CodecSpecificInfo> codec_specific_info;
691 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
692 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
693 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 2);
694 }
695
696 // Enable the second layer back.
697 // Allocate high bit rate to avoid frame dropping due to rate control.
698 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
699 bitrate_allocation.SetBitrate(
700 1, tl_idx, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2);
701 }
702 encoder_->SetRates(VideoEncoder::RateControlParameters(
703 bitrate_allocation, codec_settings_.maxFramerate));
704
705 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
706 SetWaitForEncodedFramesThreshold(2);
707 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
708 encoder_->Encode(NextInputFrame(), nullptr));
709 std::vector<EncodedImage> encoded_frame;
710 std::vector<CodecSpecificInfo> codec_specific_info;
711 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
712 ASSERT_EQ(encoded_frame.size(), 2u);
713 // SS available immediatly after switching on.
714 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
715 frame_num == 0);
716 // Keyframe should be generated when enabling lower layers.
717 const VideoFrameType expected_type = frame_num == 0
718 ? VideoFrameType::kVideoFrameKey
719 : VideoFrameType::kVideoFrameDelta;
720 EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
721 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 1);
722 EXPECT_EQ(encoded_frame[1].SpatialIndex().value_or(-1), 2);
723 }
724
725 // Enable the first layer back.
726 // Allocate high bit rate to avoid frame dropping due to rate control.
727 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
728 bitrate_allocation.SetBitrate(
729 0, tl_idx, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2);
730 }
731 encoder_->SetRates(VideoEncoder::RateControlParameters(
732 bitrate_allocation, codec_settings_.maxFramerate));
733
734 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
735 SetWaitForEncodedFramesThreshold(num_spatial_layers);
736 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
737 encoder_->Encode(NextInputFrame(), nullptr));
738 std::vector<EncodedImage> encoded_frame;
739 std::vector<CodecSpecificInfo> codec_specific_info;
740 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
741 ASSERT_EQ(encoded_frame.size(), 3u);
742 // SS available immediatly after switching on.
743 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
744 frame_num == 0);
745 // Keyframe should be generated when enabling lower layers.
746 const VideoFrameType expected_type = frame_num == 0
747 ? VideoFrameType::kVideoFrameKey
748 : VideoFrameType::kVideoFrameDelta;
749 EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
750 }
751 }
752 // TODO(bugs.webrtc.org/13442) Enable once a forward fix has landed in WebRTC.
753 TEST(Vp9ImplTest,
754 DISABLED_DisableEnableBaseLayerWithSvcControllerTriggersKeyFrame) {
755 // Configure encoder to produce N spatial layers. Encode frames for all
756 // layers. Then disable all but the last layer. Then reenable all back again.
757 const size_t num_spatial_layers = 3;
758 const size_t num_temporal_layers = 3;
759 // Must not be multiple of temporal period to exercise all code paths.
760 const size_t num_frames_to_encode = 5;
761
762 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
763 VideoCodec codec_settings = DefaultCodecSettings();
764 ConfigureSvc(codec_settings, num_spatial_layers, num_temporal_layers);
765 codec_settings.SetFrameDropEnabled(false);
766 codec_settings.VP9()->flexibleMode = false;
767 codec_settings.VP9()->interLayerPred = InterLayerPredMode::kOnKeyPic;
768 codec_settings.mode = VideoCodecMode::kRealtimeVideo;
769
770 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
771 WEBRTC_VIDEO_CODEC_OK);
772
773 VideoBitrateAllocation bitrate_allocation;
774 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
775 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
776 // Allocate high bit rate to avoid frame dropping due to rate control.
777 bitrate_allocation.SetBitrate(
778 sl_idx, tl_idx,
779 codec_settings.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
780 }
781 }
782 encoder->SetRates(VideoEncoder::RateControlParameters(
783 bitrate_allocation, codec_settings.maxFramerate));
784
785 EncodedVideoFrameProducer producer(*encoder);
786 producer.SetResolution({kWidth, kHeight});
787
788 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
789 producer.SetNumInputFrames(num_frames_to_encode).Encode();
790 ASSERT_THAT(frames, SizeIs(num_frames_to_encode * num_spatial_layers));
791
792 // Disable all but top spatial layer.
793 for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
794 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
795 bitrate_allocation.SetBitrate(sl_idx, tl_idx, 0);
796 }
797 }
798 encoder->SetRates(VideoEncoder::RateControlParameters(
799 bitrate_allocation, codec_settings.maxFramerate));
800
801 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
802 EXPECT_THAT(frames, SizeIs(num_frames_to_encode));
803 for (const auto& frame : frames) {
804 // Expect no key-frames generated.
805 EXPECT_FALSE(frame.codec_specific_info.template_structure);
806 ASSERT_TRUE(frame.codec_specific_info.generic_frame_info);
807 EXPECT_EQ(frame.codec_specific_info.generic_frame_info->spatial_id, 2);
808 }
809
810 frames = producer.ForceKeyFrame().SetNumInputFrames(1).Encode();
811 ASSERT_THAT(frames, SizeIs(1));
812 // Key-frame should be produced.
813 EXPECT_EQ(frames[0].encoded_image._frameType, VideoFrameType::kVideoFrameKey);
814 ASSERT_TRUE(frames[0].codec_specific_info.template_structure);
815 ASSERT_TRUE(frames[0].codec_specific_info.generic_frame_info);
816 EXPECT_EQ(frames[0].codec_specific_info.generic_frame_info->spatial_id, 2);
817
818 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
819 ASSERT_THAT(frames, SizeIs(num_frames_to_encode));
820 for (const auto& frame : frames) {
821 EXPECT_EQ(frame.encoded_image._frameType, VideoFrameType::kVideoFrameDelta);
822 EXPECT_FALSE(frame.codec_specific_info.template_structure);
823 ASSERT_TRUE(frame.codec_specific_info.generic_frame_info);
824 EXPECT_EQ(frame.codec_specific_info.generic_frame_info->spatial_id, 2);
825 }
826
827 // Enable the second layer back.
828 // Allocate high bit rate to avoid frame dropping due to rate control.
829 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
830 bitrate_allocation.SetBitrate(
831 1, tl_idx, codec_settings.spatialLayers[0].targetBitrate * 1000 * 2);
832 }
833 encoder->SetRates(VideoEncoder::RateControlParameters(
834 bitrate_allocation, codec_settings.maxFramerate));
835
836 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
837 ASSERT_THAT(frames, SizeIs(num_frames_to_encode * 2));
838 EXPECT_EQ(frames[0].encoded_image._frameType, VideoFrameType::kVideoFrameKey);
839 EXPECT_TRUE(frames[0].codec_specific_info.template_structure);
840 ASSERT_TRUE(frames[0].codec_specific_info.generic_frame_info);
841 EXPECT_EQ(frames[0].codec_specific_info.generic_frame_info->spatial_id, 1);
842 for (size_t i = 1; i < frames.size(); ++i) {
843 EXPECT_EQ(frames[i].encoded_image._frameType,
844 VideoFrameType::kVideoFrameDelta);
845 EXPECT_FALSE(frames[i].codec_specific_info.template_structure);
846 ASSERT_TRUE(frames[i].codec_specific_info.generic_frame_info);
847 EXPECT_EQ(frames[i].codec_specific_info.generic_frame_info->spatial_id,
848 1 + static_cast<int>(i % 2));
849 }
850
851 // Enable the first layer back.
852 // Allocate high bit rate to avoid frame dropping due to rate control.
853 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
854 bitrate_allocation.SetBitrate(
855 0, tl_idx, codec_settings.spatialLayers[1].targetBitrate * 1000 * 2);
856 }
857 encoder->SetRates(VideoEncoder::RateControlParameters(
858 bitrate_allocation, codec_settings.maxFramerate));
859
860 frames = producer.SetNumInputFrames(num_frames_to_encode).Encode();
861 ASSERT_THAT(frames, SizeIs(num_frames_to_encode * 3));
862 EXPECT_TRUE(frames[0].codec_specific_info.template_structure);
863 ASSERT_TRUE(frames[0].codec_specific_info.generic_frame_info);
864 EXPECT_EQ(frames[0].codec_specific_info.generic_frame_info->spatial_id, 0);
865 for (size_t i = 1; i < frames.size(); ++i) {
866 EXPECT_FALSE(frames[i].codec_specific_info.template_structure);
867 ASSERT_TRUE(frames[i].codec_specific_info.generic_frame_info);
868 EXPECT_EQ(frames[i].codec_specific_info.generic_frame_info->spatial_id,
869 static_cast<int>(i % 3));
870 }
871 }
872
873 TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrameForScreenshare) {
874 // Configure encoder to produce N spatial layers. Encode frames for all
875 // layers. Then disable all but the last layer. Then reenable all back again.
876 const size_t num_spatial_layers = 3;
877 const size_t num_frames_to_encode = 5;
878
879 ConfigureSvc(codec_settings_, num_spatial_layers);
880 codec_settings_.SetFrameDropEnabled(false);
881 codec_settings_.mode = VideoCodecMode::kScreensharing;
882 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
883 codec_settings_.VP9()->flexibleMode = true;
884
885 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
886 encoder_->InitEncode(&codec_settings_, kSettings));
887
888 VideoBitrateAllocation bitrate_allocation;
889 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
890 // Allocate high bit rate to avoid frame dropping due to rate control.
891 bitrate_allocation.SetBitrate(
892 sl_idx, 0,
893 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
894 }
895 encoder_->SetRates(VideoEncoder::RateControlParameters(
896 bitrate_allocation, codec_settings_.maxFramerate));
897
898 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
899 SetWaitForEncodedFramesThreshold(num_spatial_layers);
900 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
901 encoder_->Encode(NextInputFrame(), nullptr));
902 std::vector<EncodedImage> encoded_frame;
903 std::vector<CodecSpecificInfo> codec_specific_info;
904 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
905 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
906 frame_num == 0);
907 }
908
909 // Disable all but top layer.
910 for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
911 bitrate_allocation.SetBitrate(sl_idx, 0, 0);
912 }
913 encoder_->SetRates(VideoEncoder::RateControlParameters(
914 bitrate_allocation, codec_settings_.maxFramerate));
915
916 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
917 SetWaitForEncodedFramesThreshold(1);
918 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
919 encoder_->Encode(NextInputFrame(), nullptr));
920 std::vector<EncodedImage> encoded_frame;
921 std::vector<CodecSpecificInfo> codec_specific_info;
922 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
923 // SS available immediatly after switching off.
924 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
925 frame_num == 0);
926 // No key-frames generated for disabling layers.
927 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
928 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 2);
929 }
930
931 // Force key-frame.
932 std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
933 SetWaitForEncodedFramesThreshold(1);
934 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
935 encoder_->Encode(NextInputFrame(), &frame_types));
936 std::vector<EncodedImage> encoded_frame;
937 std::vector<CodecSpecificInfo> codec_specific_info;
938 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
939 // Key-frame should be produced.
940 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey);
941
942 // Enable the second layer back.
943 // Allocate high bit rate to avoid frame dropping due to rate control.
944 bitrate_allocation.SetBitrate(
945 1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2);
946 encoder_->SetRates(VideoEncoder::RateControlParameters(
947 bitrate_allocation, codec_settings_.maxFramerate));
948
949 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
950 SetWaitForEncodedFramesThreshold(2);
951 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
952 encoder_->Encode(NextInputFrame(), nullptr));
953 std::vector<EncodedImage> encoded_frame;
954 std::vector<CodecSpecificInfo> codec_specific_info;
955 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
956 ASSERT_EQ(encoded_frame.size(), 2u);
957 // SS available immediatly after switching on.
958 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
959 frame_num == 0);
960 // Keyframe should be generated when enabling lower layers.
961 const VideoFrameType expected_type = frame_num == 0
962 ? VideoFrameType::kVideoFrameKey
963 : VideoFrameType::kVideoFrameDelta;
964 EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
965 EXPECT_EQ(encoded_frame[0].SpatialIndex().value_or(-1), 1);
966 EXPECT_EQ(encoded_frame[1].SpatialIndex().value_or(-1), 2);
967 }
968
969 // Enable the first layer back.
970 // Allocate high bit rate to avoid frame dropping due to rate control.
971 bitrate_allocation.SetBitrate(
972 0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2);
973 encoder_->SetRates(VideoEncoder::RateControlParameters(
974 bitrate_allocation, codec_settings_.maxFramerate));
975
976 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
977 SetWaitForEncodedFramesThreshold(num_spatial_layers);
978 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
979 encoder_->Encode(NextInputFrame(), nullptr));
980 std::vector<EncodedImage> encoded_frame;
981 std::vector<CodecSpecificInfo> codec_specific_info;
982 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
983 ASSERT_EQ(encoded_frame.size(), 3u);
984 // SS available immediatly after switching on.
985 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
986 frame_num == 0);
987 // Keyframe should be generated when enabling lower layers.
988 const VideoFrameType expected_type = frame_num == 0
989 ? VideoFrameType::kVideoFrameKey
990 : VideoFrameType::kVideoFrameDelta;
991 EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
992 }
993 }
994
995 TEST_F(TestVp9Impl, EndOfPicture) {
996 const size_t num_spatial_layers = 2;
997 ConfigureSvc(codec_settings_, num_spatial_layers);
998
999 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1000 encoder_->InitEncode(&codec_settings_, kSettings));
1001
1002 // Encode both base and upper layers. Check that end-of-superframe flag is
1003 // set on upper layer frame but not on base layer frame.
1004 VideoBitrateAllocation bitrate_allocation;
1005 bitrate_allocation.SetBitrate(
1006 0, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000);
1007 bitrate_allocation.SetBitrate(
1008 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000);
1009 encoder_->SetRates(VideoEncoder::RateControlParameters(
1010 bitrate_allocation, codec_settings_.maxFramerate));
1011 SetWaitForEncodedFramesThreshold(2);
1012 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1013
1014 std::vector<EncodedImage> frames;
1015 std::vector<CodecSpecificInfo> codec_specific;
1016 ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific));
1017 EXPECT_FALSE(codec_specific[0].end_of_picture);
1018 EXPECT_TRUE(codec_specific[1].end_of_picture);
1019
1020 // Encode only base layer. Check that end-of-superframe flag is
1021 // set on base layer frame.
1022 bitrate_allocation.SetBitrate(1, 0, 0);
1023 encoder_->SetRates(VideoEncoder::RateControlParameters(
1024 bitrate_allocation, codec_settings_.maxFramerate));
1025 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1026 encoder_->InitEncode(&codec_settings_, kSettings));
1027
1028 SetWaitForEncodedFramesThreshold(1);
1029 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1030
1031 ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific));
1032 EXPECT_FALSE(frames[0].SpatialIndex());
1033 EXPECT_TRUE(codec_specific[0].end_of_picture);
1034 }
1035
1036 TEST_F(TestVp9Impl, InterLayerPred) {
1037 const size_t num_spatial_layers = 2;
1038 ConfigureSvc(codec_settings_, num_spatial_layers);
1039 codec_settings_.SetFrameDropEnabled(false);
1040
1041 VideoBitrateAllocation bitrate_allocation;
1042 for (size_t i = 0; i < num_spatial_layers; ++i) {
1043 bitrate_allocation.SetBitrate(
1044 i, 0, codec_settings_.spatialLayers[i].targetBitrate * 1000);
1045 }
1046
1047 const std::vector<InterLayerPredMode> inter_layer_pred_modes = {
1048 InterLayerPredMode::kOff, InterLayerPredMode::kOn,
1049 InterLayerPredMode::kOnKeyPic};
1050
1051 for (const InterLayerPredMode inter_layer_pred : inter_layer_pred_modes) {
1052 codec_settings_.VP9()->interLayerPred = inter_layer_pred;
1053 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1054 encoder_->InitEncode(&codec_settings_, kSettings));
1055
1056 encoder_->SetRates(VideoEncoder::RateControlParameters(
1057 bitrate_allocation, codec_settings_.maxFramerate));
1058
1059 SetWaitForEncodedFramesThreshold(2);
1060 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1061 encoder_->Encode(NextInputFrame(), nullptr));
1062
1063 std::vector<EncodedImage> frames;
1064 std::vector<CodecSpecificInfo> codec_specific;
1065 ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific));
1066
1067 // Key frame.
1068 ASSERT_EQ(frames[0].SpatialIndex(), 0);
1069 ASSERT_FALSE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted);
1070 EXPECT_FALSE(codec_specific[0].codecSpecific.VP9.inter_layer_predicted);
1071 EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred,
1072 inter_layer_pred == InterLayerPredMode::kOff);
1073 EXPECT_TRUE(codec_specific[0].codecSpecific.VP9.ss_data_available);
1074
1075 ASSERT_EQ(frames[1].SpatialIndex(), 1);
1076 ASSERT_FALSE(codec_specific[1].codecSpecific.VP9.inter_pic_predicted);
1077 EXPECT_EQ(codec_specific[1].codecSpecific.VP9.inter_layer_predicted,
1078 inter_layer_pred == InterLayerPredMode::kOn ||
1079 inter_layer_pred == InterLayerPredMode::kOnKeyPic);
1080 EXPECT_EQ(codec_specific[1].codecSpecific.VP9.ss_data_available,
1081 inter_layer_pred == InterLayerPredMode::kOff);
1082 EXPECT_TRUE(
1083 codec_specific[1].codecSpecific.VP9.non_ref_for_inter_layer_pred);
1084
1085 // Delta frame.
1086 SetWaitForEncodedFramesThreshold(2);
1087 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1088 encoder_->Encode(NextInputFrame(), nullptr));
1089 ASSERT_TRUE(WaitForEncodedFrames(&frames, &codec_specific));
1090
1091 ASSERT_EQ(frames[0].SpatialIndex(), 0);
1092 ASSERT_TRUE(codec_specific[0].codecSpecific.VP9.inter_pic_predicted);
1093 EXPECT_FALSE(codec_specific[0].codecSpecific.VP9.inter_layer_predicted);
1094 EXPECT_EQ(codec_specific[0].codecSpecific.VP9.non_ref_for_inter_layer_pred,
1095 inter_layer_pred != InterLayerPredMode::kOn);
1096 EXPECT_FALSE(codec_specific[0].codecSpecific.VP9.ss_data_available);
1097
1098 ASSERT_EQ(frames[1].SpatialIndex(), 1);
1099 ASSERT_TRUE(codec_specific[1].codecSpecific.VP9.inter_pic_predicted);
1100 EXPECT_EQ(codec_specific[1].codecSpecific.VP9.inter_layer_predicted,
1101 inter_layer_pred == InterLayerPredMode::kOn);
1102 EXPECT_TRUE(
1103 codec_specific[1].codecSpecific.VP9.non_ref_for_inter_layer_pred);
1104 EXPECT_FALSE(codec_specific[1].codecSpecific.VP9.ss_data_available);
1105 }
1106 }
1107
1108 TEST_F(TestVp9Impl,
1109 EnablingUpperLayerTriggersKeyFrameIfInterLayerPredIsDisabled) {
1110 const size_t num_spatial_layers = 3;
1111 const size_t num_frames_to_encode = 2;
1112
1113 ConfigureSvc(codec_settings_, num_spatial_layers);
1114 codec_settings_.SetFrameDropEnabled(false);
1115
1116 const std::vector<InterLayerPredMode> inter_layer_pred_modes = {
1117 InterLayerPredMode::kOff, InterLayerPredMode::kOn,
1118 InterLayerPredMode::kOnKeyPic};
1119
1120 for (const InterLayerPredMode inter_layer_pred : inter_layer_pred_modes) {
1121 codec_settings_.VP9()->interLayerPred = inter_layer_pred;
1122 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1123 encoder_->InitEncode(&codec_settings_, kSettings));
1124
1125 VideoBitrateAllocation bitrate_allocation;
1126 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1127 bitrate_allocation.SetBitrate(
1128 sl_idx, 0,
1129 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
1130 encoder_->SetRates(VideoEncoder::RateControlParameters(
1131 bitrate_allocation, codec_settings_.maxFramerate));
1132
1133 for (size_t frame_num = 0; frame_num < num_frames_to_encode;
1134 ++frame_num) {
1135 SetWaitForEncodedFramesThreshold(sl_idx + 1);
1136 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1137 encoder_->Encode(NextInputFrame(), nullptr));
1138 std::vector<EncodedImage> encoded_frame;
1139 std::vector<CodecSpecificInfo> codec_specific_info;
1140 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1141
1142 const bool is_first_upper_layer_frame = (sl_idx > 0 && frame_num == 0);
1143 if (is_first_upper_layer_frame) {
1144 if (inter_layer_pred == InterLayerPredMode::kOn) {
1145 EXPECT_EQ(encoded_frame[0]._frameType,
1146 VideoFrameType::kVideoFrameDelta);
1147 } else {
1148 EXPECT_EQ(encoded_frame[0]._frameType,
1149 VideoFrameType::kVideoFrameKey);
1150 }
1151 } else if (sl_idx == 0 && frame_num == 0) {
1152 EXPECT_EQ(encoded_frame[0]._frameType,
1153 VideoFrameType::kVideoFrameKey);
1154 } else {
1155 for (size_t i = 0; i <= sl_idx; ++i) {
1156 EXPECT_EQ(encoded_frame[i]._frameType,
1157 VideoFrameType::kVideoFrameDelta);
1158 }
1159 }
1160 }
1161 }
1162 }
1163 }
1164
1165 TEST_F(TestVp9Impl,
1166 EnablingUpperLayerUnsetsInterPicPredictedInInterlayerPredModeOn) {
1167 const size_t num_spatial_layers = 3;
1168 const size_t num_frames_to_encode = 2;
1169
1170 ConfigureSvc(codec_settings_, num_spatial_layers);
1171 codec_settings_.SetFrameDropEnabled(false);
1172 codec_settings_.VP9()->flexibleMode = false;
1173
1174 const std::vector<InterLayerPredMode> inter_layer_pred_modes = {
1175 InterLayerPredMode::kOff, InterLayerPredMode::kOn,
1176 InterLayerPredMode::kOnKeyPic};
1177
1178 for (const InterLayerPredMode inter_layer_pred : inter_layer_pred_modes) {
1179 codec_settings_.VP9()->interLayerPred = inter_layer_pred;
1180 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1181 encoder_->InitEncode(&codec_settings_, kSettings));
1182
1183 VideoBitrateAllocation bitrate_allocation;
1184 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1185 bitrate_allocation.SetBitrate(
1186 sl_idx, 0,
1187 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
1188 encoder_->SetRates(VideoEncoder::RateControlParameters(
1189 bitrate_allocation, codec_settings_.maxFramerate));
1190
1191 for (size_t frame_num = 0; frame_num < num_frames_to_encode;
1192 ++frame_num) {
1193 SetWaitForEncodedFramesThreshold(sl_idx + 1);
1194 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1195 encoder_->Encode(NextInputFrame(), nullptr));
1196 std::vector<EncodedImage> encoded_frame;
1197 std::vector<CodecSpecificInfo> codec_specific_info;
1198 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1199
1200 ASSERT_EQ(codec_specific_info.size(), sl_idx + 1);
1201
1202 for (size_t i = 0; i <= sl_idx; ++i) {
1203 const bool is_keyframe =
1204 encoded_frame[0]._frameType == VideoFrameType::kVideoFrameKey;
1205 const bool is_first_upper_layer_frame =
1206 (i == sl_idx && frame_num == 0);
1207 // Interframe references are there, unless it's a keyframe,
1208 // or it's a first activated frame in a upper layer
1209 const bool expect_no_references =
1210 is_keyframe || (is_first_upper_layer_frame &&
1211 inter_layer_pred == InterLayerPredMode::kOn);
1212 EXPECT_EQ(
1213 codec_specific_info[i].codecSpecific.VP9.inter_pic_predicted,
1214 !expect_no_references);
1215 }
1216 }
1217 }
1218 }
1219 }
1220
1221 TEST_F(TestVp9Impl, EnablingDisablingUpperLayerInTheSameGof) {
1222 const size_t num_spatial_layers = 2;
1223 const size_t num_temporal_layers = 2;
1224
1225 ConfigureSvc(codec_settings_, num_spatial_layers, num_temporal_layers);
1226 codec_settings_.SetFrameDropEnabled(false);
1227 codec_settings_.VP9()->flexibleMode = false;
1228
1229 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1230 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1231 encoder_->InitEncode(&codec_settings_, kSettings));
1232
1233 VideoBitrateAllocation bitrate_allocation;
1234
1235 // Enable both spatial and both temporal layers.
1236 bitrate_allocation.SetBitrate(
1237 0, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 / 2);
1238 bitrate_allocation.SetBitrate(
1239 0, 1, codec_settings_.spatialLayers[0].targetBitrate * 1000 / 2);
1240 bitrate_allocation.SetBitrate(
1241 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1242 bitrate_allocation.SetBitrate(
1243 1, 1, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1244 encoder_->SetRates(VideoEncoder::RateControlParameters(
1245 bitrate_allocation, codec_settings_.maxFramerate));
1246
1247 std::vector<EncodedImage> encoded_frame;
1248 std::vector<CodecSpecificInfo> codec_specific_info;
1249
1250 // Encode 3 frames.
1251 for (int i = 0; i < 3; ++i) {
1252 SetWaitForEncodedFramesThreshold(2);
1253 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1254 encoder_->Encode(NextInputFrame(), nullptr));
1255 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1256 ASSERT_EQ(codec_specific_info.size(), 2u);
1257 }
1258
1259 // Disable SL1 layer.
1260 bitrate_allocation.SetBitrate(1, 0, 0);
1261 bitrate_allocation.SetBitrate(1, 1, 0);
1262 encoder_->SetRates(VideoEncoder::RateControlParameters(
1263 bitrate_allocation, codec_settings_.maxFramerate));
1264
1265 // Encode 1 frame.
1266 SetWaitForEncodedFramesThreshold(1);
1267 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1268 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1269 ASSERT_EQ(codec_specific_info.size(), 1u);
1270 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
1271 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 1);
1272 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.inter_pic_predicted, true);
1273
1274 // Enable SL1 layer.
1275 bitrate_allocation.SetBitrate(
1276 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1277 bitrate_allocation.SetBitrate(
1278 1, 1, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1279 encoder_->SetRates(VideoEncoder::RateControlParameters(
1280 bitrate_allocation, codec_settings_.maxFramerate));
1281
1282 // Encode 1 frame.
1283 SetWaitForEncodedFramesThreshold(2);
1284 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1285 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1286 ASSERT_EQ(codec_specific_info.size(), 2u);
1287 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
1288 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 0);
1289 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.inter_pic_predicted, true);
1290 EXPECT_EQ(codec_specific_info[1].codecSpecific.VP9.inter_pic_predicted, true);
1291 }
1292
1293 TEST_F(TestVp9Impl, EnablingDisablingUpperLayerAccrossGof) {
1294 const size_t num_spatial_layers = 2;
1295 const size_t num_temporal_layers = 2;
1296
1297 ConfigureSvc(codec_settings_, num_spatial_layers, num_temporal_layers);
1298 codec_settings_.SetFrameDropEnabled(false);
1299 codec_settings_.VP9()->flexibleMode = false;
1300
1301 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1302 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1303 encoder_->InitEncode(&codec_settings_, kSettings));
1304
1305 VideoBitrateAllocation bitrate_allocation;
1306
1307 // Enable both spatial and both temporal layers.
1308 bitrate_allocation.SetBitrate(
1309 0, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 / 2);
1310 bitrate_allocation.SetBitrate(
1311 0, 1, codec_settings_.spatialLayers[0].targetBitrate * 1000 / 2);
1312 bitrate_allocation.SetBitrate(
1313 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1314 bitrate_allocation.SetBitrate(
1315 1, 1, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1316 encoder_->SetRates(VideoEncoder::RateControlParameters(
1317 bitrate_allocation, codec_settings_.maxFramerate));
1318
1319 std::vector<EncodedImage> encoded_frame;
1320 std::vector<CodecSpecificInfo> codec_specific_info;
1321
1322 // Encode 3 frames.
1323 for (int i = 0; i < 3; ++i) {
1324 SetWaitForEncodedFramesThreshold(2);
1325 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1326 encoder_->Encode(NextInputFrame(), nullptr));
1327 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1328 ASSERT_EQ(codec_specific_info.size(), 2u);
1329 }
1330
1331 // Disable SL1 layer.
1332 bitrate_allocation.SetBitrate(1, 0, 0);
1333 bitrate_allocation.SetBitrate(1, 1, 0);
1334 encoder_->SetRates(VideoEncoder::RateControlParameters(
1335 bitrate_allocation, codec_settings_.maxFramerate));
1336
1337 // Encode 11 frames. More than Gof length 2, and odd to end at TL1 frame.
1338 for (int i = 0; i < 11; ++i) {
1339 SetWaitForEncodedFramesThreshold(1);
1340 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1341 encoder_->Encode(NextInputFrame(), nullptr));
1342 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1343 ASSERT_EQ(codec_specific_info.size(), 1u);
1344 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
1345 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 1 - i % 2);
1346 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.inter_pic_predicted,
1347 true);
1348 }
1349
1350 // Enable SL1 layer.
1351 bitrate_allocation.SetBitrate(
1352 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1353 bitrate_allocation.SetBitrate(
1354 1, 1, codec_settings_.spatialLayers[1].targetBitrate * 1000 / 2);
1355 encoder_->SetRates(VideoEncoder::RateControlParameters(
1356 bitrate_allocation, codec_settings_.maxFramerate));
1357
1358 // Encode 1 frame.
1359 SetWaitForEncodedFramesThreshold(2);
1360 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1361 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
1362 ASSERT_EQ(codec_specific_info.size(), 2u);
1363 EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
1364 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 0);
1365 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.inter_pic_predicted, true);
1366 EXPECT_EQ(codec_specific_info[1].codecSpecific.VP9.inter_pic_predicted,
1367 false);
1368 }
1369
1370 TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) {
1371 const size_t num_spatial_layers = 3;
1372 // Chosen by hand, the 2nd frame is dropped with configured per-layer max
1373 // framerate.
1374 const size_t num_frames_to_encode_before_drop = 1;
1375
1376 codec_settings_.maxFramerate = 30;
1377 ConfigureSvc(codec_settings_, num_spatial_layers);
1378 codec_settings_.spatialLayers[0].maxFramerate = 5.0;
1379 // use 30 for the SL 1 instead of 10, so even if SL 0 frame is dropped due to
1380 // framerate capping we would still get back at least a middle layer. It
1381 // simplifies the test.
1382 codec_settings_.spatialLayers[1].maxFramerate = 30.0;
1383 codec_settings_.spatialLayers[2].maxFramerate = 30.0;
1384 codec_settings_.SetFrameDropEnabled(false);
1385 codec_settings_.mode = VideoCodecMode::kScreensharing;
1386 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1387 codec_settings_.VP9()->flexibleMode = true;
1388 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1389 encoder_->InitEncode(&codec_settings_, kSettings));
1390
1391 // Enable all but the last layer.
1392 VideoBitrateAllocation bitrate_allocation;
1393 for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
1394 bitrate_allocation.SetBitrate(
1395 sl_idx, 0, codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
1396 }
1397 encoder_->SetRates(VideoEncoder::RateControlParameters(
1398 bitrate_allocation, codec_settings_.maxFramerate));
1399
1400 // Encode enough frames to force drop due to framerate capping.
1401 for (size_t frame_num = 0; frame_num < num_frames_to_encode_before_drop;
1402 ++frame_num) {
1403 SetWaitForEncodedFramesThreshold(num_spatial_layers - 1);
1404 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1405 encoder_->Encode(NextInputFrame(), nullptr));
1406 std::vector<EncodedImage> encoded_frames;
1407 std::vector<CodecSpecificInfo> codec_specific_info;
1408 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1409 }
1410
1411 // Enable the last layer.
1412 bitrate_allocation.SetBitrate(
1413 num_spatial_layers - 1, 0,
1414 codec_settings_.spatialLayers[num_spatial_layers - 1].targetBitrate *
1415 1000);
1416 encoder_->SetRates(VideoEncoder::RateControlParameters(
1417 bitrate_allocation, codec_settings_.maxFramerate));
1418
1419 // All layers are encoded, even though frame dropping should happen.
1420 SetWaitForEncodedFramesThreshold(num_spatial_layers);
1421 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1422 // Now all 3 layers should be encoded.
1423 std::vector<EncodedImage> encoded_frames;
1424 std::vector<CodecSpecificInfo> codec_specific_info;
1425 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1426 EXPECT_EQ(encoded_frames.size(), 3u);
1427 // Scalability structure has to be triggered.
1428 EXPECT_TRUE(codec_specific_info[0].codecSpecific.VP9.ss_data_available);
1429 }
1430
1431 TEST_F(TestVp9Impl, ScreenshareFrameDropping) {
1432 const int num_spatial_layers = 3;
1433 const int num_frames_to_detect_drops = 2;
1434
1435 codec_settings_.maxFramerate = 30;
1436 ConfigureSvc(codec_settings_, num_spatial_layers);
1437 // use 30 for the SL0 and SL1 because it simplifies the test.
1438 codec_settings_.spatialLayers[0].maxFramerate = 30.0;
1439 codec_settings_.spatialLayers[1].maxFramerate = 30.0;
1440 codec_settings_.spatialLayers[2].maxFramerate = 30.0;
1441 codec_settings_.SetFrameDropEnabled(true);
1442 codec_settings_.mode = VideoCodecMode::kScreensharing;
1443 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1444 codec_settings_.VP9()->flexibleMode = true;
1445 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1446 encoder_->InitEncode(&codec_settings_, kSettings));
1447
1448 // Enable all but the last layer.
1449 VideoBitrateAllocation bitrate_allocation;
1450 // Very low bitrate for the lowest spatial layer to ensure rate-control drops.
1451 bitrate_allocation.SetBitrate(0, 0, 1000);
1452 bitrate_allocation.SetBitrate(
1453 1, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000);
1454 // Disable highest layer.
1455 bitrate_allocation.SetBitrate(2, 0, 0);
1456
1457 encoder_->SetRates(VideoEncoder::RateControlParameters(
1458 bitrate_allocation, codec_settings_.maxFramerate));
1459
1460 bool frame_dropped = false;
1461 // Encode enough frames to force drop due to rate-control.
1462 for (size_t frame_num = 0; frame_num < num_frames_to_detect_drops;
1463 ++frame_num) {
1464 SetWaitForEncodedFramesThreshold(1);
1465 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1466 encoder_->Encode(NextInputFrame(), nullptr));
1467 std::vector<EncodedImage> encoded_frames;
1468 std::vector<CodecSpecificInfo> codec_specific_info;
1469 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1470 EXPECT_LE(encoded_frames.size(), 2u);
1471 EXPECT_GE(encoded_frames.size(), 1u);
1472 if (encoded_frames.size() == 1) {
1473 frame_dropped = true;
1474 // Dropped frame is on the SL0.
1475 EXPECT_EQ(encoded_frames[0].SpatialIndex(), 1);
1476 }
1477 }
1478 EXPECT_TRUE(frame_dropped);
1479
1480 // Enable the last layer.
1481 bitrate_allocation.SetBitrate(
1482 2, 0, codec_settings_.spatialLayers[2].targetBitrate * 1000);
1483 encoder_->SetRates(VideoEncoder::RateControlParameters(
1484 bitrate_allocation, codec_settings_.maxFramerate));
1485 SetWaitForEncodedFramesThreshold(1);
1486 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1487 std::vector<EncodedImage> encoded_frames;
1488 std::vector<CodecSpecificInfo> codec_specific_info;
1489 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1490 // No drop allowed.
1491 EXPECT_EQ(encoded_frames.size(), 3u);
1492
1493 // Verify that frame-dropping is re-enabled back.
1494 frame_dropped = false;
1495 // Encode enough frames to force drop due to rate-control.
1496 for (size_t frame_num = 0; frame_num < num_frames_to_detect_drops;
1497 ++frame_num) {
1498 SetWaitForEncodedFramesThreshold(1);
1499 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1500 encoder_->Encode(NextInputFrame(), nullptr));
1501 std::vector<EncodedImage> encoded_frames;
1502 std::vector<CodecSpecificInfo> codec_specific_info;
1503 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1504 EXPECT_LE(encoded_frames.size(), 3u);
1505 EXPECT_GE(encoded_frames.size(), 2u);
1506 if (encoded_frames.size() == 2) {
1507 frame_dropped = true;
1508 // Dropped frame is on the SL0.
1509 EXPECT_EQ(encoded_frames[0].SpatialIndex(), 1);
1510 EXPECT_EQ(encoded_frames[1].SpatialIndex(), 2);
1511 }
1512 }
1513 EXPECT_TRUE(frame_dropped);
1514 }
1515
1516 TEST_F(TestVp9Impl, RemovingLayerIsNotDelayedInScreenshareAndAddsSsInfo) {
1517 const size_t num_spatial_layers = 3;
1518 // Chosen by hand, the 2nd frame is dropped with configured per-layer max
1519 // framerate.
1520 const size_t num_frames_to_encode_before_drop = 1;
1521 // Chosen by hand, exactly 5 frames are dropped for input fps=30 and max
1522 // framerate = 5.
1523 const size_t num_dropped_frames = 5;
1524
1525 codec_settings_.maxFramerate = 30;
1526 ConfigureSvc(codec_settings_, num_spatial_layers);
1527 codec_settings_.spatialLayers[0].maxFramerate = 5.0;
1528 // use 30 for the SL 1 instead of 5, so even if SL 0 frame is dropped due to
1529 // framerate capping we would still get back at least a middle layer. It
1530 // simplifies the test.
1531 codec_settings_.spatialLayers[1].maxFramerate = 30.0;
1532 codec_settings_.spatialLayers[2].maxFramerate = 30.0;
1533 codec_settings_.SetFrameDropEnabled(false);
1534 codec_settings_.mode = VideoCodecMode::kScreensharing;
1535 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1536 codec_settings_.VP9()->flexibleMode = true;
1537 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1538 encoder_->InitEncode(&codec_settings_, kSettings));
1539
1540 // All layers are enabled from the start.
1541 VideoBitrateAllocation bitrate_allocation;
1542 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1543 bitrate_allocation.SetBitrate(
1544 sl_idx, 0, codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
1545 }
1546 encoder_->SetRates(VideoEncoder::RateControlParameters(
1547 bitrate_allocation, codec_settings_.maxFramerate));
1548
1549 // Encode enough frames to force drop due to framerate capping.
1550 for (size_t frame_num = 0; frame_num < num_frames_to_encode_before_drop;
1551 ++frame_num) {
1552 SetWaitForEncodedFramesThreshold(num_spatial_layers);
1553 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1554 encoder_->Encode(NextInputFrame(), nullptr));
1555 std::vector<EncodedImage> encoded_frames;
1556 std::vector<CodecSpecificInfo> codec_specific_info;
1557 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1558 }
1559
1560 // Now the first layer should not have frames in it.
1561 for (size_t frame_num = 0; frame_num < num_dropped_frames - 2; ++frame_num) {
1562 SetWaitForEncodedFramesThreshold(2);
1563 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1564 encoder_->Encode(NextInputFrame(), nullptr));
1565 // First layer is dropped due to frame rate cap. The last layer should not
1566 // be enabled yet.
1567 std::vector<EncodedImage> encoded_frames;
1568 std::vector<CodecSpecificInfo> codec_specific_info;
1569 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1570 // First layer is skipped.
1571 EXPECT_EQ(encoded_frames[0].SpatialIndex().value_or(-1), 1);
1572 }
1573
1574 // Disable the last layer.
1575 bitrate_allocation.SetBitrate(num_spatial_layers - 1, 0, 0);
1576 encoder_->SetRates(VideoEncoder::RateControlParameters(
1577 bitrate_allocation, codec_settings_.maxFramerate));
1578
1579 // Still expected to drop first layer. Last layer has to be disable also.
1580 for (size_t frame_num = num_dropped_frames - 2;
1581 frame_num < num_dropped_frames; ++frame_num) {
1582 // Expect back one frame.
1583 SetWaitForEncodedFramesThreshold(1);
1584 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1585 encoder_->Encode(NextInputFrame(), nullptr));
1586 // First layer is dropped due to frame rate cap. The last layer should not
1587 // be enabled yet.
1588 std::vector<EncodedImage> encoded_frames;
1589 std::vector<CodecSpecificInfo> codec_specific_info;
1590 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1591 // First layer is skipped.
1592 EXPECT_EQ(encoded_frames[0].SpatialIndex().value_or(-1), 1);
1593 // No SS data on non-base spatial layer.
1594 EXPECT_FALSE(codec_specific_info[0].codecSpecific.VP9.ss_data_available);
1595 }
1596
1597 SetWaitForEncodedFramesThreshold(2);
1598 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1599 std::vector<EncodedImage> encoded_frames;
1600 std::vector<CodecSpecificInfo> codec_specific_info;
1601 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1602 // First layer is not skipped now.
1603 EXPECT_EQ(encoded_frames[0].SpatialIndex().value_or(-1), 0);
1604 // SS data should be present.
1605 EXPECT_TRUE(codec_specific_info[0].codecSpecific.VP9.ss_data_available);
1606 }
1607
1608 TEST_F(TestVp9Impl, DisableNewLayerInVideoDelaysSsInfoTillTL0) {
1609 const size_t num_spatial_layers = 3;
1610 const size_t num_temporal_layers = 2;
1611 // Chosen by hand, the 2nd frame is dropped with configured per-layer max
1612 // framerate.
1613 ConfigureSvc(codec_settings_, num_spatial_layers, num_temporal_layers);
1614 codec_settings_.SetFrameDropEnabled(false);
1615 codec_settings_.mode = VideoCodecMode::kRealtimeVideo;
1616 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOnKeyPic;
1617 codec_settings_.VP9()->flexibleMode = false;
1618 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1619 encoder_->InitEncode(&codec_settings_, kSettings));
1620
1621 // Enable all the layers.
1622 VideoBitrateAllocation bitrate_allocation;
1623 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1624 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
1625 bitrate_allocation.SetBitrate(
1626 sl_idx, tl_idx,
1627 codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 /
1628 num_temporal_layers);
1629 }
1630 }
1631 encoder_->SetRates(VideoEncoder::RateControlParameters(
1632 bitrate_allocation, codec_settings_.maxFramerate));
1633
1634 std::vector<EncodedImage> encoded_frames;
1635 std::vector<CodecSpecificInfo> codec_specific_info;
1636
1637 // Encode one TL0 frame
1638 SetWaitForEncodedFramesThreshold(num_spatial_layers);
1639 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1640 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1641 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 0u);
1642
1643 // Disable the last layer.
1644 for (size_t tl_idx = 0; tl_idx < num_temporal_layers; ++tl_idx) {
1645 bitrate_allocation.SetBitrate(num_spatial_layers - 1, tl_idx, 0);
1646 }
1647 encoder_->SetRates(VideoEncoder::RateControlParameters(
1648 bitrate_allocation, codec_settings_.maxFramerate));
1649
1650 // Next is TL1 frame. The last layer is disabled immediately, but SS structure
1651 // is not provided here.
1652 SetWaitForEncodedFramesThreshold(num_spatial_layers - 1);
1653 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1654 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1655 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 1u);
1656 EXPECT_FALSE(codec_specific_info[0].codecSpecific.VP9.ss_data_available);
1657
1658 // Next is TL0 frame, which should have delayed SS structure.
1659 SetWaitForEncodedFramesThreshold(num_spatial_layers - 1);
1660 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1661 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
1662 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.temporal_idx, 0u);
1663 EXPECT_TRUE(codec_specific_info[0].codecSpecific.VP9.ss_data_available);
1664 EXPECT_TRUE(codec_specific_info[0]
1665 .codecSpecific.VP9.spatial_layer_resolution_present);
1666 EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.num_spatial_layers,
1667 num_spatial_layers - 1);
1668 }
1669
1670 TEST_F(TestVp9Impl,
1671 LowLayerMarkedAsRefIfHighLayerNotEncodedAndInterLayerPredIsEnabled) {
1672 ConfigureSvc(codec_settings_, 3);
1673 codec_settings_.SetFrameDropEnabled(false);
1674 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
1675
1676 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1677 encoder_->InitEncode(&codec_settings_, kSettings));
1678
1679 VideoBitrateAllocation bitrate_allocation;
1680 bitrate_allocation.SetBitrate(
1681 0, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000);
1682 encoder_->SetRates(VideoEncoder::RateControlParameters(
1683 bitrate_allocation, codec_settings_.maxFramerate));
1684 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1685 EncodedImage encoded_frame;
1686 CodecSpecificInfo codec_info;
1687 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_info));
1688 EXPECT_TRUE(codec_info.codecSpecific.VP9.ss_data_available);
1689 EXPECT_FALSE(codec_info.codecSpecific.VP9.non_ref_for_inter_layer_pred);
1690 }
1691
1692 TEST_F(TestVp9Impl, ScalabilityStructureIsAvailableInFlexibleMode) {
1693 codec_settings_.VP9()->flexibleMode = true;
1694 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1695 encoder_->InitEncode(&codec_settings_, kSettings));
1696
1697 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
1698 EncodedImage encoded_frame;
1699 CodecSpecificInfo codec_specific_info;
1700 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
1701 EXPECT_TRUE(codec_specific_info.codecSpecific.VP9.ss_data_available);
1702 }
1703
1704 TEST_F(TestVp9Impl, Profile0PreferredPixelFormats) {
1705 EXPECT_THAT(encoder_->GetEncoderInfo().preferred_pixel_formats,
1706 testing::UnorderedElementsAre(VideoFrameBuffer::Type::kNV12,
1707 VideoFrameBuffer::Type::kI420));
1708 }
1709
1710 TEST_F(TestVp9Impl, EncoderInfoWithoutResolutionBitrateLimits) {
1711 EXPECT_TRUE(encoder_->GetEncoderInfo().resolution_bitrate_limits.empty());
1712 }
1713
1714 TEST_F(TestVp9Impl, EncoderInfoWithBitrateLimitsFromFieldTrial) {
1715 test::ScopedFieldTrials field_trials(
1716 "WebRTC-VP9-GetEncoderInfoOverride/"
1717 "frame_size_pixels:123|456|789,"
1718 "min_start_bitrate_bps:11000|22000|33000,"
1719 "min_bitrate_bps:44000|55000|66000,"
1720 "max_bitrate_bps:77000|88000|99000/");
1721 SetUp();
1722
1723 EXPECT_THAT(
1724 encoder_->GetEncoderInfo().resolution_bitrate_limits,
1725 ::testing::ElementsAre(
1726 VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000},
1727 VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000},
1728 VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000}));
1729 }
1730
1731 TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) {
1732 const uint8_t kNumSpatialLayers = 3;
1733 const uint8_t kNumTemporalLayers = 3;
1734
1735 codec_settings_.maxFramerate = 30;
1736 codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
1737 codec_settings_.VP9()->numberOfTemporalLayers = kNumTemporalLayers;
1738
1739 for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
1740 codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
1741 codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
1742 codec_settings_.spatialLayers[sl_idx].minBitrate =
1743 codec_settings_.startBitrate;
1744 codec_settings_.spatialLayers[sl_idx].maxBitrate =
1745 codec_settings_.startBitrate;
1746 codec_settings_.spatialLayers[sl_idx].targetBitrate =
1747 codec_settings_.startBitrate;
1748 codec_settings_.spatialLayers[sl_idx].active = true;
1749 codec_settings_.spatialLayers[sl_idx].maxFramerate =
1750 codec_settings_.maxFramerate;
1751 }
1752
1753 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1754 encoder_->InitEncode(&codec_settings_, kSettings));
1755
1756 FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
1757 expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
1758 expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
1759 expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
1760 expected_fps_allocation[1] = expected_fps_allocation[0];
1761 expected_fps_allocation[2] = expected_fps_allocation[0];
1762 EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
1763 ElementsAreArray(expected_fps_allocation));
1764 }
1765
1766 TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) {
1767 const uint8_t kNumSpatialLayers = 3;
1768
1769 codec_settings_.maxFramerate = 30;
1770 codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
1771 codec_settings_.VP9()->numberOfTemporalLayers = 1;
1772 codec_settings_.VP9()->flexibleMode = true;
1773
1774 VideoEncoder::RateControlParameters rate_params;
1775 for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
1776 codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
1777 codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
1778 codec_settings_.spatialLayers[sl_idx].minBitrate =
1779 codec_settings_.startBitrate;
1780 codec_settings_.spatialLayers[sl_idx].maxBitrate =
1781 codec_settings_.startBitrate;
1782 codec_settings_.spatialLayers[sl_idx].targetBitrate =
1783 codec_settings_.startBitrate;
1784 codec_settings_.spatialLayers[sl_idx].active = true;
1785 // Force different frame rates for different layers, to verify that total
1786 // fraction is correct.
1787 codec_settings_.spatialLayers[sl_idx].maxFramerate =
1788 codec_settings_.maxFramerate / (kNumSpatialLayers - sl_idx);
1789 rate_params.bitrate.SetBitrate(sl_idx, 0,
1790 codec_settings_.startBitrate * 1000);
1791 }
1792 rate_params.bandwidth_allocation =
1793 DataRate::BitsPerSec(rate_params.bitrate.get_sum_bps());
1794 rate_params.framerate_fps = codec_settings_.maxFramerate;
1795
1796 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1797 encoder_->InitEncode(&codec_settings_, kSettings));
1798
1799 // No temporal layers allowed when spatial layers have different fps targets.
1800 FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
1801 expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 3);
1802 expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2);
1803 expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction);
1804 EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
1805 ::testing::ElementsAreArray(expected_fps_allocation));
1806
1807 // SetRates with current fps does not alter outcome.
1808 encoder_->SetRates(rate_params);
1809 EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
1810 ::testing::ElementsAreArray(expected_fps_allocation));
1811
1812 // Higher fps than the codec wants, should still not affect outcome.
1813 rate_params.framerate_fps *= 2;
1814 encoder_->SetRates(rate_params);
1815 EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
1816 ::testing::ElementsAreArray(expected_fps_allocation));
1817 }
1818
1819 class Vp9ImplWithLayeringTest
1820 : public ::testing::TestWithParam<std::tuple<int, int, bool>> {
1821 protected:
1822 Vp9ImplWithLayeringTest()
1823 : num_spatial_layers_(std::get<0>(GetParam())),
1824 num_temporal_layers_(std::get<1>(GetParam())),
1825 override_field_trials_(std::get<2>(GetParam())
1826 ? "WebRTC-Vp9ExternalRefCtrl/Enabled/"
1827 : "") {}
1828
1829 const uint8_t num_spatial_layers_;
1830 const uint8_t num_temporal_layers_;
1831 const test::ScopedFieldTrials override_field_trials_;
1832 };
1833
1834 TEST_P(Vp9ImplWithLayeringTest, FlexibleMode) {
1835 // In flexible mode encoder wrapper obtains actual list of references from
1836 // encoder and writes it into RTP payload descriptor. Check that reference
1837 // list in payload descriptor matches the predefined one, which is used
1838 // in non-flexible mode.
1839 std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
1840 VideoCodec codec_settings = DefaultCodecSettings();
1841 codec_settings.VP9()->flexibleMode = true;
1842 codec_settings.SetFrameDropEnabled(false);
1843 codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers_;
1844 codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers_;
1845 EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
1846 WEBRTC_VIDEO_CODEC_OK);
1847
1848 GofInfoVP9 gof;
1849 if (num_temporal_layers_ == 1) {
1850 gof.SetGofInfoVP9(kTemporalStructureMode1);
1851 } else if (num_temporal_layers_ == 2) {
1852 gof.SetGofInfoVP9(kTemporalStructureMode2);
1853 } else if (num_temporal_layers_ == 3) {
1854 gof.SetGofInfoVP9(kTemporalStructureMode3);
1855 }
1856
1857 // Encode at least (num_frames_in_gof + 1) frames to verify references
1858 // of non-key frame with gof_idx = 0.
1859 int num_input_frames = gof.num_frames_in_gof + 1;
1860 std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
1861 EncodedVideoFrameProducer(*encoder)
1862 .SetNumInputFrames(num_input_frames)
1863 .SetResolution({kWidth, kHeight})
1864 .Encode();
1865 ASSERT_THAT(frames, SizeIs(num_input_frames * num_spatial_layers_));
1866
1867 for (size_t i = 0; i < frames.size(); ++i) {
1868 const EncodedVideoFrameProducer::EncodedFrame& frame = frames[i];
1869 const size_t picture_idx = i / num_spatial_layers_;
1870 const size_t gof_idx = picture_idx % gof.num_frames_in_gof;
1871
1872 const CodecSpecificInfoVP9& vp9 =
1873 frame.codec_specific_info.codecSpecific.VP9;
1874 EXPECT_EQ(frame.encoded_image.SpatialIndex(),
1875 num_spatial_layers_ == 1
1876 ? absl::nullopt
1877 : absl::optional<int>(i % num_spatial_layers_))
1878 << "Frame " << i;
1879 EXPECT_EQ(vp9.temporal_idx, num_temporal_layers_ == 1
1880 ? kNoTemporalIdx
1881 : gof.temporal_idx[gof_idx])
1882 << "Frame " << i;
1883 EXPECT_EQ(vp9.temporal_up_switch, gof.temporal_up_switch[gof_idx])
1884 << "Frame " << i;
1885 if (picture_idx == 0) {
1886 EXPECT_EQ(vp9.num_ref_pics, 0) << "Frame " << i;
1887 } else {
1888 EXPECT_THAT(rtc::MakeArrayView(vp9.p_diff, vp9.num_ref_pics),
1889 UnorderedElementsAreArray(gof.pid_diff[gof_idx],
1890 gof.num_ref_pics[gof_idx]))
1891 << "Frame " << i;
1892 }
1893 }
1894 }
1895
1896 INSTANTIATE_TEST_SUITE_P(All,
1897 Vp9ImplWithLayeringTest,
1898 ::testing::Combine(::testing::Values(1, 2, 3),
1899 ::testing::Values(1, 2, 3),
1900 ::testing::Bool()));
1901
1902 class TestVp9ImplFrameDropping : public TestVp9Impl {
1903 protected:
1904 void ModifyCodecSettings(VideoCodec* codec_settings) override {
1905 webrtc::test::CodecSettings(kVideoCodecVP9, codec_settings);
1906 // We need to encode quite a lot of frames in this test. Use low resolution
1907 // to reduce execution time.
1908 codec_settings->width = 64;
1909 codec_settings->height = 64;
1910 codec_settings->mode = VideoCodecMode::kScreensharing;
1911 }
1912 };
1913
1914 TEST_F(TestVp9ImplFrameDropping, PreEncodeFrameDropping) {
1915 const size_t num_frames_to_encode = 100;
1916 const float input_framerate_fps = 30.0;
1917 const float video_duration_secs = num_frames_to_encode / input_framerate_fps;
1918 const float expected_framerate_fps = 5.0f;
1919 const float max_abs_framerate_error_fps = expected_framerate_fps * 0.1f;
1920
1921 codec_settings_.maxFramerate = static_cast<uint32_t>(expected_framerate_fps);
1922 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1923 encoder_->InitEncode(&codec_settings_, kSettings));
1924
1925 VideoFrame input_frame = NextInputFrame();
1926 for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
1927 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
1928 const size_t timestamp = input_frame.timestamp() +
1929 kVideoPayloadTypeFrequency / input_framerate_fps;
1930 input_frame.set_timestamp(static_cast<uint32_t>(timestamp));
1931 }
1932
1933 const size_t num_encoded_frames = GetNumEncodedFrames();
1934 const float encoded_framerate_fps = num_encoded_frames / video_duration_secs;
1935 EXPECT_NEAR(encoded_framerate_fps, expected_framerate_fps,
1936 max_abs_framerate_error_fps);
1937 }
1938
1939 TEST_F(TestVp9ImplFrameDropping, DifferentFrameratePerSpatialLayer) {
1940 // Assign different frame rate to spatial layers and check that result frame
1941 // rate is close to the assigned one.
1942 const uint8_t num_spatial_layers = 3;
1943 const float input_framerate_fps = 30.0;
1944 const size_t video_duration_secs = 3;
1945 const size_t num_input_frames = video_duration_secs * input_framerate_fps;
1946
1947 codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers;
1948 codec_settings_.SetFrameDropEnabled(false);
1949 codec_settings_.VP9()->flexibleMode = true;
1950
1951 VideoBitrateAllocation bitrate_allocation;
1952 for (uint8_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1953 // Frame rate increases from low to high layer.
1954 const uint32_t framerate_fps = 10 * (sl_idx + 1);
1955
1956 codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
1957 codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
1958 codec_settings_.spatialLayers[sl_idx].maxFramerate = framerate_fps;
1959 codec_settings_.spatialLayers[sl_idx].minBitrate =
1960 codec_settings_.startBitrate;
1961 codec_settings_.spatialLayers[sl_idx].maxBitrate =
1962 codec_settings_.startBitrate;
1963 codec_settings_.spatialLayers[sl_idx].targetBitrate =
1964 codec_settings_.startBitrate;
1965 codec_settings_.spatialLayers[sl_idx].active = true;
1966
1967 bitrate_allocation.SetBitrate(
1968 sl_idx, 0, codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
1969 }
1970
1971 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
1972 encoder_->InitEncode(&codec_settings_, kSettings));
1973
1974 encoder_->SetRates(VideoEncoder::RateControlParameters(
1975 bitrate_allocation, codec_settings_.maxFramerate));
1976
1977 VideoFrame input_frame = NextInputFrame();
1978 for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) {
1979 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
1980 const size_t timestamp = input_frame.timestamp() +
1981 kVideoPayloadTypeFrequency / input_framerate_fps;
1982 input_frame.set_timestamp(static_cast<uint32_t>(timestamp));
1983 }
1984
1985 std::vector<EncodedImage> encoded_frames;
1986 std::vector<CodecSpecificInfo> codec_infos;
1987 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_infos));
1988
1989 std::vector<size_t> num_encoded_frames(num_spatial_layers, 0);
1990 for (EncodedImage& encoded_frame : encoded_frames) {
1991 ++num_encoded_frames[encoded_frame.SpatialIndex().value_or(0)];
1992 }
1993
1994 for (uint8_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
1995 const float layer_target_framerate_fps =
1996 codec_settings_.spatialLayers[sl_idx].maxFramerate;
1997 const float layer_output_framerate_fps =
1998 static_cast<float>(num_encoded_frames[sl_idx]) / video_duration_secs;
1999 const float max_framerate_error_fps = layer_target_framerate_fps * 0.1f;
2000 EXPECT_NEAR(layer_output_framerate_fps, layer_target_framerate_fps,
2001 max_framerate_error_fps);
2002 }
2003 }
2004
2005 class TestVp9ImplProfile2 : public TestVp9Impl {
2006 protected:
2007 void SetUp() override {
2008 // Profile 2 might not be available on some platforms until
2009 // https://bugs.chromium.org/p/webm/issues/detail?id=1544 is solved.
2010 bool profile_2_is_supported = false;
2011 for (const auto& codec : SupportedVP9Codecs()) {
2012 if (ParseSdpForVP9Profile(codec.parameters)
2013 .value_or(VP9Profile::kProfile0) == VP9Profile::kProfile2) {
2014 profile_2_is_supported = true;
2015 }
2016 }
2017 if (!profile_2_is_supported)
2018 return;
2019
2020 TestVp9Impl::SetUp();
2021 input_frame_generator_ = test::CreateSquareFrameGenerator(
2022 codec_settings_.width, codec_settings_.height,
2023 test::FrameGeneratorInterface::OutputType::kI010,
2024 absl::optional<int>());
2025 }
2026
2027 std::unique_ptr<VideoEncoder> CreateEncoder() override {
2028 cricket::VideoCodec profile2_codec;
2029 profile2_codec.SetParam(kVP9FmtpProfileId,
2030 VP9ProfileToString(VP9Profile::kProfile2));
2031 return VP9Encoder::Create(profile2_codec);
2032 }
2033
2034 std::unique_ptr<VideoDecoder> CreateDecoder() override {
2035 return VP9Decoder::Create();
2036 }
2037 };
2038
2039 TEST_F(TestVp9ImplProfile2, EncodeDecode) {
2040 if (!encoder_)
2041 return;
2042
2043 VideoFrame input_frame = NextInputFrame();
2044 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
2045 EncodedImage encoded_frame;
2046 CodecSpecificInfo codec_specific_info;
2047 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
2048 // First frame should be a key frame.
2049 encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
2050 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, 0));
2051 std::unique_ptr<VideoFrame> decoded_frame;
2052 absl::optional<uint8_t> decoded_qp;
2053 ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
2054 ASSERT_TRUE(decoded_frame);
2055
2056 // TODO(emircan): Add PSNR for different color depths.
2057 EXPECT_GT(I420PSNR(*input_frame.video_frame_buffer()->ToI420(),
2058 *decoded_frame->video_frame_buffer()->ToI420()),
2059 31);
2060 }
2061
2062 TEST_F(TestVp9Impl, EncodeWithDynamicRate) {
2063 // Configured dynamic rate field trial and re-create the encoder.
2064 test::ScopedFieldTrials field_trials(
2065 "WebRTC-VideoRateControl/vp9_dynamic_rate:true/");
2066 SetUp();
2067
2068 // Set 300kbps target with 100% headroom.
2069 VideoEncoder::RateControlParameters params;
2070 params.bandwidth_allocation = DataRate::BitsPerSec(300000);
2071 params.bitrate.SetBitrate(0, 0, params.bandwidth_allocation.bps());
2072 params.framerate_fps = 30.0;
2073
2074 encoder_->SetRates(params);
2075 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
2076 EncodedImage encoded_frame;
2077 CodecSpecificInfo codec_specific_info;
2078 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
2079
2080 // Set no headroom and encode again.
2081 params.bandwidth_allocation = DataRate::Zero();
2082 encoder_->SetRates(params);
2083 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
2084 ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
2085 }
2086
2087 TEST_F(TestVp9Impl, ReenablingUpperLayerAfterKFWithInterlayerPredIsEnabled) {
2088 const size_t num_spatial_layers = 2;
2089 const int num_frames_to_encode = 10;
2090 codec_settings_.VP9()->flexibleMode = true;
2091 codec_settings_.SetFrameDropEnabled(false);
2092 codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers;
2093 codec_settings_.VP9()->numberOfTemporalLayers = 1;
2094 codec_settings_.VP9()->interLayerPred = InterLayerPredMode::kOn;
2095 // Force low frame-rate, so all layers are present for all frames.
2096 codec_settings_.maxFramerate = 5;
2097
2098 ConfigureSvc(codec_settings_, num_spatial_layers);
2099
2100 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
2101 encoder_->InitEncode(&codec_settings_, kSettings));
2102
2103 VideoBitrateAllocation bitrate_allocation;
2104 for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
2105 bitrate_allocation.SetBitrate(
2106 sl_idx, 0, codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000);
2107 }
2108 encoder_->SetRates(VideoEncoder::RateControlParameters(
2109 bitrate_allocation, codec_settings_.maxFramerate));
2110
2111 std::vector<EncodedImage> encoded_frames;
2112 std::vector<CodecSpecificInfo> codec_specific;
2113
2114 for (int i = 0; i < num_frames_to_encode; ++i) {
2115 SetWaitForEncodedFramesThreshold(num_spatial_layers);
2116 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
2117 encoder_->Encode(NextInputFrame(), nullptr));
2118 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific));
2119 EXPECT_EQ(encoded_frames.size(), num_spatial_layers);
2120 }
2121
2122 // Disable the last layer.
2123 bitrate_allocation.SetBitrate(num_spatial_layers - 1, 0, 0);
2124 encoder_->SetRates(VideoEncoder::RateControlParameters(
2125 bitrate_allocation, codec_settings_.maxFramerate));
2126
2127 for (int i = 0; i < num_frames_to_encode; ++i) {
2128 SetWaitForEncodedFramesThreshold(num_spatial_layers - 1);
2129 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
2130 encoder_->Encode(NextInputFrame(), nullptr));
2131 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific));
2132 EXPECT_EQ(encoded_frames.size(), num_spatial_layers - 1);
2133 }
2134
2135 std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
2136
2137 // Force a key-frame with the last layer still disabled.
2138 SetWaitForEncodedFramesThreshold(num_spatial_layers - 1);
2139 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
2140 encoder_->Encode(NextInputFrame(), &frame_types));
2141 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific));
2142 EXPECT_EQ(encoded_frames.size(), num_spatial_layers - 1);
2143 ASSERT_EQ(encoded_frames[0]._frameType, VideoFrameType::kVideoFrameKey);
2144
2145 // Re-enable the last layer.
2146 bitrate_allocation.SetBitrate(
2147 num_spatial_layers - 1, 0,
2148 codec_settings_.spatialLayers[num_spatial_layers - 1].targetBitrate *
2149 1000);
2150 encoder_->SetRates(VideoEncoder::RateControlParameters(
2151 bitrate_allocation, codec_settings_.maxFramerate));
2152
2153 SetWaitForEncodedFramesThreshold(num_spatial_layers);
2154 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
2155 ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific));
2156 EXPECT_EQ(encoded_frames.size(), num_spatial_layers);
2157 EXPECT_EQ(encoded_frames[0]._frameType, VideoFrameType::kVideoFrameDelta);
2158 }
2159
2160 TEST_F(TestVp9Impl, HandlesEmptyDecoderConfigure) {
2161 std::unique_ptr<VideoDecoder> decoder = CreateDecoder();
2162 // Check that default settings are ok for decoder.
2163 EXPECT_TRUE(decoder->Configure({}));
2164 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder->Release());
2165 }
2166
2167 INSTANTIATE_TEST_SUITE_P(
2168 TestVp9ImplForPixelFormat,
2169 TestVp9ImplForPixelFormat,
2170 ::testing::Values(test::FrameGeneratorInterface::OutputType::kI420,
2171 test::FrameGeneratorInterface::OutputType::kNV12),
2172 [](const auto& info) {
2173 return test::FrameGeneratorInterface::OutputTypeToString(info.param);
2174 });
2175
2176 // Helper function to populate an vpx_image_t instance with dimensions and
2177 // potential image data.
2178 std::function<vpx_image_t*(vpx_image_t*,
2179 vpx_img_fmt_t,
2180 unsigned int,
2181 unsigned int,
2182 unsigned int,
2183 unsigned char* img_data)>
2184 GetWrapImageFunction(vpx_image_t* img) {
2185 return [img](vpx_image_t* /*img*/, vpx_img_fmt_t fmt, unsigned int d_w,
2186 unsigned int d_h, unsigned int /*stride_align*/,
2187 unsigned char* img_data) {
2188 img->fmt = fmt;
2189 img->d_w = d_w;
2190 img->d_h = d_h;
2191 img->img_data = img_data;
2192 return img;
2193 };
2194 }
2195
2196 TEST(Vp9SpeedSettingsTrialsTest, NoSvcUsesGlobalSpeedFromTl0InLayerConfig) {
2197 // TL0 speed 8 at >= 480x270, 5 if below that.
2198 test::ExplicitKeyValueConfig trials(
2199 "WebRTC-VP9-PerformanceFlags/"
2200 "use_per_layer_speed,"
2201 "min_pixel_count:0|129600,"
2202 "base_layer_speed:4|8,"
2203 "high_layer_speed:5|9,"
2204 "deblock_mode:1|0/");
2205
2206 // Keep a raw pointer for EXPECT calls and the like. Ownership is otherwise
2207 // passed on to LibvpxVp9Encoder.
2208 auto* const vpx = new NiceMock<MockLibvpxInterface>();
2209 LibvpxVp9Encoder encoder(cricket::VideoCodec(),
2210 absl::WrapUnique<LibvpxInterface>(vpx), trials);
2211
2212 VideoCodec settings = DefaultCodecSettings();
2213 settings.width = 480;
2214 settings.height = 270;
2215 vpx_image_t img;
2216
2217 ON_CALL(*vpx, img_wrap).WillByDefault(GetWrapImageFunction(&img));
2218 ON_CALL(*vpx, codec_enc_config_default)
2219 .WillByDefault(DoAll(WithArg<1>([](vpx_codec_enc_cfg_t* cfg) {
2220 memset(cfg, 0, sizeof(vpx_codec_enc_cfg_t));
2221 }),
2222 Return(VPX_CODEC_OK)));
2223 EXPECT_CALL(*vpx, codec_control(_, _, An<int>())).Times(AnyNumber());
2224
2225 EXPECT_CALL(*vpx, codec_control(_, VP9E_SET_SVC_PARAMETERS,
2226 A<vpx_svc_extra_cfg_t*>()))
2227 .Times(0);
2228
2229 EXPECT_CALL(*vpx, codec_control(_, VP8E_SET_CPUUSED, TypedEq<int>(8)));
2230 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings));
2231
2232 encoder.Release();
2233 settings.width = 352;
2234 settings.height = 216;
2235
2236 EXPECT_CALL(*vpx, codec_control(_, VP8E_SET_CPUUSED, TypedEq<int>(4)));
2237 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings));
2238 }
2239
2240 TEST(Vp9SpeedSettingsTrialsTest,
2241 NoPerLayerFlagUsesGlobalSpeedFromTopLayerInConfig) {
2242 // TL0 speed 8 at >= 480x270, 5 if below that.
2243 test::ExplicitKeyValueConfig trials(
2244 "WebRTC-VP9-PerformanceFlags/"
2245 "min_pixel_count:0|129600,"
2246 "base_layer_speed:4|8,"
2247 "high_layer_speed:5|9,"
2248 "deblock_mode:1|0/");
2249
2250 // Keep a raw pointer for EXPECT calls and the like. Ownership is otherwise
2251 // passed on to LibvpxVp9Encoder.
2252 auto* const vpx = new NiceMock<MockLibvpxInterface>();
2253 LibvpxVp9Encoder encoder(cricket::VideoCodec(),
2254 absl::WrapUnique<LibvpxInterface>(vpx), trials);
2255
2256 VideoCodec settings = DefaultCodecSettings();
2257 settings.width = 480;
2258 settings.height = 270;
2259 ConfigureSvc(settings, 2, 3);
2260 vpx_image_t img;
2261
2262 ON_CALL(*vpx, img_wrap).WillByDefault(GetWrapImageFunction(&img));
2263 ON_CALL(*vpx, codec_enc_config_default)
2264 .WillByDefault(DoAll(WithArg<1>([](vpx_codec_enc_cfg_t* cfg) {
2265 memset(cfg, 0, sizeof(vpx_codec_enc_cfg_t));
2266 }),
2267 Return(VPX_CODEC_OK)));
2268 EXPECT_CALL(*vpx, codec_control(_, _, An<int>())).Times(AnyNumber());
2269
2270 // Speed settings not populated when 'use_per_layer_speed' flag is absent.
2271 EXPECT_CALL(*vpx,
2272 codec_control(
2273 _, VP9E_SET_SVC_PARAMETERS,
2274 SafeMatcherCast<vpx_svc_extra_cfg_t*>(AllOf(
2275 Field(&vpx_svc_extra_cfg_t::speed_per_layer, Each(0)),
2276 Field(&vpx_svc_extra_cfg_t::loopfilter_ctrl, Each(0))))))
2277 .Times(2);
2278
2279 EXPECT_CALL(*vpx, codec_control(_, VP8E_SET_CPUUSED, TypedEq<int>(8)));
2280 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings));
2281
2282 encoder.Release();
2283 settings.width = 476;
2284 settings.height = 268;
2285 settings.spatialLayers[0].width = settings.width / 2;
2286 settings.spatialLayers[0].height = settings.height / 2;
2287 settings.spatialLayers[1].width = settings.width;
2288 settings.spatialLayers[1].height = settings.height;
2289
2290 EXPECT_CALL(*vpx, codec_control(_, VP8E_SET_CPUUSED, TypedEq<int>(4)));
2291 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings));
2292 }
2293
2294 TEST(Vp9SpeedSettingsTrialsTest, DefaultPerLayerFlagsWithSvc) {
2295 // Per-temporal and spatial layer speed settings:
2296 // SL0: TL0 = speed 5, TL1/TL2 = speed 8.
2297 // SL1/2: TL0 = speed 7, TL1/TL2 = speed 8.
2298 // Deblocking-mode per spatial layer:
2299 // SL0: mode 1, SL1/2: mode 0.
2300 test::ExplicitKeyValueConfig trials(
2301 "WebRTC-VP9-PerformanceFlags/"
2302 "use_per_layer_speed,"
2303 "min_pixel_count:0|129600,"
2304 "base_layer_speed:5|7,"
2305 "high_layer_speed:8|8,"
2306 "deblock_mode:1|0/");
2307
2308 // Keep a raw pointer for EXPECT calls and the like. Ownership is otherwise
2309 // passed on to LibvpxVp9Encoder.
2310 auto* const vpx = new NiceMock<MockLibvpxInterface>();
2311 LibvpxVp9Encoder encoder(cricket::VideoCodec(),
2312 absl::WrapUnique<LibvpxInterface>(vpx), trials);
2313
2314 VideoCodec settings = DefaultCodecSettings();
2315 constexpr int kNumSpatialLayers = 3;
2316 constexpr int kNumTemporalLayers = 3;
2317 ConfigureSvc(settings, kNumSpatialLayers, kNumTemporalLayers);
2318 VideoBitrateAllocation bitrate_allocation;
2319 for (int si = 0; si < kNumSpatialLayers; ++si) {
2320 for (int ti = 0; ti < kNumTemporalLayers; ++ti) {
2321 uint32_t bitrate_bps =
2322 settings.spatialLayers[si].targetBitrate * 1'000 / kNumTemporalLayers;
2323 bitrate_allocation.SetBitrate(si, ti, bitrate_bps);
2324 }
2325 }
2326 vpx_image_t img;
2327
2328 // Speed settings per spatial layer, for TL0.
2329 const int kBaseTlSpeed[VPX_MAX_LAYERS] = {5, 7, 7};
2330 // Speed settings per spatial layer, for TL1, TL2.
2331 const int kHighTlSpeed[VPX_MAX_LAYERS] = {8, 8, 8};
2332 // Loopfilter settings are handled within libvpx, so this array is valid for
2333 // both TL0 and higher.
2334 const int kLoopFilter[VPX_MAX_LAYERS] = {1, 0, 0};
2335
2336 ON_CALL(*vpx, img_wrap).WillByDefault(GetWrapImageFunction(&img));
2337 ON_CALL(*vpx, codec_enc_init)
2338 .WillByDefault(WithArg<0>([](vpx_codec_ctx_t* ctx) {
2339 memset(ctx, 0, sizeof(*ctx));
2340 return VPX_CODEC_OK;
2341 }));
2342 ON_CALL(*vpx, codec_enc_config_default)
2343 .WillByDefault(DoAll(WithArg<1>([](vpx_codec_enc_cfg_t* cfg) {
2344 memset(cfg, 0, sizeof(vpx_codec_enc_cfg_t));
2345 }),
2346 Return(VPX_CODEC_OK)));
2347 EXPECT_CALL(
2348 *vpx, codec_control(_, VP9E_SET_SVC_PARAMETERS,
2349 SafeMatcherCast<vpx_svc_extra_cfg_t*>(
2350 AllOf(Field(&vpx_svc_extra_cfg_t::speed_per_layer,
2351 ElementsAreArray(kBaseTlSpeed)),
2352 Field(&vpx_svc_extra_cfg_t::loopfilter_ctrl,
2353 ElementsAreArray(kLoopFilter))))));
2354
2355 // Capture the callback into the vp9 wrapper.
2356 vpx_codec_priv_output_cx_pkt_cb_pair_t callback_pointer = {};
2357 EXPECT_CALL(*vpx, codec_control(_, VP9E_REGISTER_CX_CALLBACK, A<void*>()))
2358 .WillOnce(WithArg<2>([&](void* cbp) {
2359 callback_pointer =
2360 *reinterpret_cast<vpx_codec_priv_output_cx_pkt_cb_pair_t*>(cbp);
2361 return VPX_CODEC_OK;
2362 }));
2363
2364 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings));
2365
2366 encoder.SetRates(VideoEncoder::RateControlParameters(bitrate_allocation,
2367 settings.maxFramerate));
2368
2369 MockEncodedImageCallback callback;
2370 encoder.RegisterEncodeCompleteCallback(&callback);
2371 auto frame_generator = test::CreateSquareFrameGenerator(
2372 kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420, 10);
2373 Mock::VerifyAndClearExpectations(vpx);
2374
2375 uint8_t data[1] = {0};
2376 vpx_codec_cx_pkt encoded_data = {};
2377 encoded_data.data.frame.buf = &data;
2378 encoded_data.data.frame.sz = 1;
2379
2380 const auto kImageOk =
2381 EncodedImageCallback::Result(EncodedImageCallback::Result::OK);
2382
2383 int spatial_id = 0;
2384 int temporal_id = 0;
2385 EXPECT_CALL(*vpx,
2386 codec_control(_, VP9E_SET_SVC_LAYER_ID, A<vpx_svc_layer_id_t*>()))
2387 .Times(AnyNumber());
2388 EXPECT_CALL(*vpx,
2389 codec_control(_, VP9E_GET_SVC_LAYER_ID, A<vpx_svc_layer_id_t*>()))
2390 .WillRepeatedly(WithArg<2>([&](vpx_svc_layer_id_t* layer_id) {
2391 layer_id->spatial_layer_id = spatial_id;
2392 layer_id->temporal_layer_id = temporal_id;
2393 return VPX_CODEC_OK;
2394 }));
2395 vpx_svc_ref_frame_config_t stored_refs = {};
2396 ON_CALL(*vpx, codec_control(_, VP9E_SET_SVC_REF_FRAME_CONFIG,
2397 A<vpx_svc_ref_frame_config_t*>()))
2398 .WillByDefault(
2399 DoAll(SaveArgPointee<2>(&stored_refs), Return(VPX_CODEC_OK)));
2400 ON_CALL(*vpx, codec_control(_, VP9E_GET_SVC_REF_FRAME_CONFIG,
2401 A<vpx_svc_ref_frame_config_t*>()))
2402 .WillByDefault(
2403 DoAll(SetArgPointee<2>(ByRef(stored_refs)), Return(VPX_CODEC_OK)));
2404
2405 // First frame is keyframe.
2406 encoded_data.data.frame.flags = VPX_FRAME_IS_KEY;
2407
2408 // Default 3-layer temporal pattern: 0-2-1-2, then repeat and do two more.
2409 for (int ti : {0, 2, 1, 2, 0, 2}) {
2410 EXPECT_CALL(*vpx, codec_encode).WillOnce(Return(VPX_CODEC_OK));
2411 // No update expected if flags haven't changed, and they change we we move
2412 // between base temporal layer and non-base temporal layer.
2413 if ((ti > 0) != (temporal_id > 0)) {
2414 EXPECT_CALL(*vpx, codec_control(
2415 _, VP9E_SET_SVC_PARAMETERS,
2416 SafeMatcherCast<vpx_svc_extra_cfg_t*>(AllOf(
2417 Field(&vpx_svc_extra_cfg_t::speed_per_layer,
2418 ElementsAreArray(ti == 0 ? kBaseTlSpeed
2419 : kHighTlSpeed)),
2420 Field(&vpx_svc_extra_cfg_t::loopfilter_ctrl,
2421 ElementsAreArray(kLoopFilter))))));
2422 } else {
2423 EXPECT_CALL(*vpx, codec_control(_, VP9E_SET_SVC_PARAMETERS,
2424 A<vpx_svc_extra_cfg_t*>()))
2425 .Times(0);
2426 }
2427
2428 VideoFrame frame =
2429 VideoFrame::Builder()
2430 .set_video_frame_buffer(frame_generator->NextFrame().buffer)
2431 .build();
2432 encoder.Encode(frame, nullptr);
2433
2434 temporal_id = ti;
2435 for (int si = 0; si < kNumSpatialLayers; ++si) {
2436 spatial_id = si;
2437
2438 EXPECT_CALL(callback, OnEncodedImage).WillOnce(Return(kImageOk));
2439 callback_pointer.output_cx_pkt(&encoded_data, callback_pointer.user_priv);
2440 }
2441
2442 encoded_data.data.frame.flags = 0; // Following frames are delta frames.
2443 }
2444 }
2445
2446 } // namespace webrtc
2447