1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "modules/video_coding/svc/scalability_structure_key_svc.h"
11
12 #include <vector>
13
14 #include "api/array_view.h"
15 #include "api/transport/rtp/dependency_descriptor.h"
16 #include "common_video/generic_frame_descriptor/generic_frame_info.h"
17 #include "modules/video_coding/svc/scalability_structure_test_helpers.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20
21 namespace webrtc {
22 namespace {
23
24 using ::testing::ElementsAre;
25 using ::testing::IsEmpty;
26 using ::testing::SizeIs;
27
TEST(ScalabilityStructureL3T3KeyTest,SkipingT1FrameOnOneSpatialLayerKeepsStructureValid)28 TEST(ScalabilityStructureL3T3KeyTest,
29 SkipingT1FrameOnOneSpatialLayerKeepsStructureValid) {
30 ScalabilityStructureL3T3Key structure;
31 ScalabilityStructureWrapper wrapper(structure);
32 std::vector<GenericFrameInfo> frames;
33
34 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/3, /*s1=*/3));
35 wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
36 EXPECT_THAT(frames, SizeIs(4));
37 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/3, /*s1=*/1));
38 wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
39 EXPECT_THAT(frames, SizeIs(5));
40 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/3, /*s1=*/3));
41 wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
42 ASSERT_THAT(frames, SizeIs(7));
43
44 EXPECT_EQ(frames[0].temporal_id, 0);
45 EXPECT_EQ(frames[1].temporal_id, 0);
46 EXPECT_EQ(frames[2].temporal_id, 2);
47 EXPECT_EQ(frames[3].temporal_id, 2);
48 EXPECT_EQ(frames[4].temporal_id, 1);
49 EXPECT_EQ(frames[5].temporal_id, 2);
50 EXPECT_EQ(frames[6].temporal_id, 2);
51 EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
52 }
53
TEST(ScalabilityStructureL3T3KeyTest,SkipT1FrameByEncoderKeepsReferencesValid)54 TEST(ScalabilityStructureL3T3KeyTest,
55 SkipT1FrameByEncoderKeepsReferencesValid) {
56 std::vector<GenericFrameInfo> frames;
57 ScalabilityStructureL3T3Key structure;
58 ScalabilityStructureWrapper wrapper(structure);
59
60 // 1st 2 temporal units (T0 and T2)
61 wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
62 // Simulate T1 frame dropped by the encoder,
63 // i.e. retrieve config, but skip calling OnEncodeDone.
64 structure.NextFrameConfig(/*restart=*/false);
65 // one more temporal unit.
66 wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
67
68 EXPECT_THAT(frames, SizeIs(9));
69 EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
70 }
71
TEST(ScalabilityStructureL3T3KeyTest,SkippingFrameReusePreviousFrameConfiguration)72 TEST(ScalabilityStructureL3T3KeyTest,
73 SkippingFrameReusePreviousFrameConfiguration) {
74 std::vector<GenericFrameInfo> frames;
75 ScalabilityStructureL3T3Key structure;
76 ScalabilityStructureWrapper wrapper(structure);
77
78 // 1st 2 temporal units (T0 and T2)
79 wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
80 ASSERT_THAT(frames, SizeIs(6));
81 ASSERT_EQ(frames[0].temporal_id, 0);
82 ASSERT_EQ(frames[3].temporal_id, 2);
83
84 // Simulate a frame dropped by the encoder,
85 // i.e. retrieve config, but skip calling OnEncodeDone.
86 structure.NextFrameConfig(/*restart=*/false);
87 // two more temporal unit, expect temporal pattern continues
88 wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
89 ASSERT_THAT(frames, SizeIs(12));
90 // Expect temporal pattern continues as if there were no dropped frames.
91 EXPECT_EQ(frames[6].temporal_id, 1);
92 EXPECT_EQ(frames[9].temporal_id, 2);
93 }
94
TEST(ScalabilityStructureL3T3KeyTest,SkippingKeyFrameTriggersNewKeyFrame)95 TEST(ScalabilityStructureL3T3KeyTest, SkippingKeyFrameTriggersNewKeyFrame) {
96 std::vector<GenericFrameInfo> frames;
97 ScalabilityStructureL3T3Key structure;
98 ScalabilityStructureWrapper wrapper(structure);
99
100 // Ask for a key frame config, but do not return any frames
101 structure.NextFrameConfig(/*restart=*/false);
102
103 // Ask for more frames, expect they start with a key frame.
104 wrapper.GenerateFrames(/*num_temporal_units=*/2, frames);
105 ASSERT_THAT(frames, SizeIs(6));
106 ASSERT_EQ(frames[0].temporal_id, 0);
107 ASSERT_EQ(frames[3].temporal_id, 2);
108 EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
109 }
110
TEST(ScalabilityStructureL3T3KeyTest,SkippingT2FrameAndDisablingT2LayerProduceT1AsNextFrame)111 TEST(ScalabilityStructureL3T3KeyTest,
112 SkippingT2FrameAndDisablingT2LayerProduceT1AsNextFrame) {
113 std::vector<GenericFrameInfo> frames;
114 ScalabilityStructureL3T3Key structure;
115 ScalabilityStructureWrapper wrapper(structure);
116
117 wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
118 // Ask for next (T2) frame config, but do not return any frames
119 auto config = structure.NextFrameConfig(/*restart=*/false);
120 ASSERT_THAT(config, Not(IsEmpty()));
121 ASSERT_EQ(config.front().TemporalId(), 2);
122
123 // Disable T2 layer,
124 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2, /*s2=*/2));
125 // Expect instead of reusing unused config, T1 config is generated.
126 config = structure.NextFrameConfig(/*restart=*/false);
127 ASSERT_THAT(config, Not(IsEmpty()));
128 EXPECT_EQ(config.front().TemporalId(), 1);
129 }
130
TEST(ScalabilityStructureL3T3KeyTest,EnableT2LayerWhileProducingT1Frame)131 TEST(ScalabilityStructureL3T3KeyTest, EnableT2LayerWhileProducingT1Frame) {
132 std::vector<GenericFrameInfo> frames;
133 ScalabilityStructureL3T3Key structure;
134 ScalabilityStructureWrapper wrapper(structure);
135
136 // Disable T2 layer,
137 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2, /*s2=*/2));
138
139 // Generate the key frame.
140 wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
141 ASSERT_THAT(frames, SizeIs(3));
142 EXPECT_EQ(frames[0].temporal_id, 0);
143
144 // Ask for next (T1) frame config, but do not return any frames yet.
145 auto config = structure.NextFrameConfig(/*restart=*/false);
146 ASSERT_THAT(config, Not(IsEmpty()));
147 ASSERT_EQ(config.front().TemporalId(), 1);
148
149 // Reenable T2 layer.
150 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/3, /*s1=*/3, /*s2=*/3));
151
152 // Finish encoding previously requested config.
153 for (auto layer_config : config) {
154 GenericFrameInfo info = structure.OnEncodeDone(layer_config);
155 EXPECT_EQ(info.temporal_id, 1);
156 frames.push_back(info);
157 }
158 ASSERT_THAT(frames, SizeIs(6));
159
160 // Generate more frames, expect T2 pattern resumes.
161 wrapper.GenerateFrames(/*num_temporal_units=*/4, frames);
162 ASSERT_THAT(frames, SizeIs(18));
163 EXPECT_EQ(frames[6].temporal_id, 2);
164 EXPECT_EQ(frames[9].temporal_id, 0);
165 EXPECT_EQ(frames[12].temporal_id, 2);
166 EXPECT_EQ(frames[15].temporal_id, 1);
167
168 EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
169 }
170
TEST(ScalabilityStructureL3T3KeyTest,ReenablingSpatialLayerBeforeMissedT0FrameDoesntTriggerAKeyFrame)171 TEST(ScalabilityStructureL3T3KeyTest,
172 ReenablingSpatialLayerBeforeMissedT0FrameDoesntTriggerAKeyFrame) {
173 ScalabilityStructureL3T3Key structure;
174 ScalabilityStructureWrapper wrapper(structure);
175 std::vector<GenericFrameInfo> frames;
176
177 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2));
178 wrapper.GenerateFrames(1, frames);
179 EXPECT_THAT(frames, SizeIs(2));
180 // Drop a spatial layer.
181 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/0));
182 wrapper.GenerateFrames(1, frames);
183 EXPECT_THAT(frames, SizeIs(3));
184 // Reenable a spatial layer before T0 frame is encoded.
185 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2));
186 wrapper.GenerateFrames(1, frames);
187 EXPECT_THAT(frames, SizeIs(5));
188
189 EXPECT_EQ(frames[0].temporal_id, 0);
190 EXPECT_EQ(frames[1].temporal_id, 0);
191 EXPECT_EQ(frames[2].temporal_id, 1);
192 EXPECT_EQ(frames[3].temporal_id, 0);
193 EXPECT_EQ(frames[4].temporal_id, 0);
194 EXPECT_THAT(frames[3].frame_diffs, SizeIs(1));
195 EXPECT_THAT(frames[4].frame_diffs, SizeIs(1));
196 EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
197 }
198
TEST(ScalabilityStructureL3T3KeyTest,ReenablingSpatialLayerTriggersKeyFrame)199 TEST(ScalabilityStructureL3T3KeyTest, ReenablingSpatialLayerTriggersKeyFrame) {
200 ScalabilityStructureL3T3Key structure;
201 ScalabilityStructureWrapper wrapper(structure);
202 std::vector<GenericFrameInfo> frames;
203
204 // Start with all spatial layers enabled.
205 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2, /*s2=*/2));
206 wrapper.GenerateFrames(3, frames);
207 EXPECT_THAT(frames, SizeIs(9));
208 // Drop a spatial layer. Two remaining spatial layers should just continue.
209 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/0, /*s2=*/2));
210 wrapper.GenerateFrames(2, frames);
211 EXPECT_THAT(frames, SizeIs(13));
212 // Reenable spatial layer, expect a full restart.
213 structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/2, /*s1=*/2, /*s2=*/2));
214 wrapper.GenerateFrames(1, frames);
215 ASSERT_THAT(frames, SizeIs(16));
216
217 // First 3 temporal units with all spatial layers enabled.
218 EXPECT_EQ(frames[0].temporal_id, 0);
219 EXPECT_EQ(frames[3].temporal_id, 1);
220 EXPECT_EQ(frames[6].temporal_id, 0);
221 // 2 temporal units with spatial layer 1 disabled.
222 EXPECT_EQ(frames[9].spatial_id, 0);
223 EXPECT_EQ(frames[9].temporal_id, 1);
224 EXPECT_EQ(frames[10].spatial_id, 2);
225 EXPECT_EQ(frames[10].temporal_id, 1);
226 // T0 frames were encoded while spatial layer 1 is disabled.
227 EXPECT_EQ(frames[11].spatial_id, 0);
228 EXPECT_EQ(frames[11].temporal_id, 0);
229 EXPECT_EQ(frames[12].spatial_id, 2);
230 EXPECT_EQ(frames[12].temporal_id, 0);
231 // Key frame to reenable spatial layer 1.
232 EXPECT_THAT(frames[13].frame_diffs, IsEmpty());
233 EXPECT_THAT(frames[14].frame_diffs, ElementsAre(1));
234 EXPECT_THAT(frames[15].frame_diffs, ElementsAre(1));
235 EXPECT_EQ(frames[13].temporal_id, 0);
236 EXPECT_EQ(frames[14].temporal_id, 0);
237 EXPECT_EQ(frames[15].temporal_id, 0);
238 auto all_frames = rtc::MakeArrayView(frames.data(), frames.size());
239 EXPECT_TRUE(wrapper.FrameReferencesAreValid(all_frames.subview(0, 13)));
240 // Frames starting from the frame#13 should not reference any earlier frames.
241 EXPECT_TRUE(wrapper.FrameReferencesAreValid(all_frames.subview(13)));
242 }
243
244 } // namespace
245 } // namespace webrtc
246