1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/video_coding/rtp_vp8_ref_finder.h"
12
13 #include <utility>
14 #include <vector>
15
16 #include "modules/video_coding/frame_object.h"
17 #include "test/gmock.h"
18 #include "test/gtest.h"
19
20 using ::testing::Contains;
21 using ::testing::Eq;
22 using ::testing::Matcher;
23 using ::testing::Matches;
24 using ::testing::SizeIs;
25 using ::testing::UnorderedElementsAreArray;
26
27 namespace webrtc {
28 namespace {
29
30 MATCHER_P2(HasIdAndRefs, id, refs, "") {
31 return Matches(Eq(id))(arg->Id()) &&
32 Matches(UnorderedElementsAreArray(refs))(
33 rtc::ArrayView<int64_t>(arg->references, arg->num_references));
34 }
35
36 Matcher<const std::vector<std::unique_ptr<EncodedFrame>>&>
HasFrameWithIdAndRefs(int64_t frame_id,const std::vector<int64_t> & refs)37 HasFrameWithIdAndRefs(int64_t frame_id, const std::vector<int64_t>& refs) {
38 return Contains(HasIdAndRefs(frame_id, refs));
39 }
40
41 class Frame {
42 public:
AsKeyFrame(bool is_keyframe=true)43 Frame& AsKeyFrame(bool is_keyframe = true) {
44 is_keyframe_ = is_keyframe;
45 return *this;
46 }
47
Pid(int pid)48 Frame& Pid(int pid) {
49 picture_id_ = pid;
50 return *this;
51 }
52
Tid(int tid)53 Frame& Tid(int tid) {
54 temporal_id_ = tid;
55 return *this;
56 }
57
Tl0(int tl0)58 Frame& Tl0(int tl0) {
59 tl0_idx_ = tl0;
60 return *this;
61 }
62
AsSync(bool is_sync=true)63 Frame& AsSync(bool is_sync = true) {
64 sync = is_sync;
65 return *this;
66 }
67
operator std::unique_ptr<RtpFrameObject>()68 operator std::unique_ptr<RtpFrameObject>() {
69 RTPVideoHeaderVP8 vp8_header{};
70 vp8_header.pictureId = *picture_id_;
71 vp8_header.temporalIdx = *temporal_id_;
72 vp8_header.tl0PicIdx = *tl0_idx_;
73 vp8_header.layerSync = sync;
74
75 RTPVideoHeader video_header;
76 video_header.frame_type = is_keyframe_ ? VideoFrameType::kVideoFrameKey
77 : VideoFrameType::kVideoFrameDelta;
78 video_header.video_type_header = vp8_header;
79 // clang-format off
80 return std::make_unique<RtpFrameObject>(
81 /*seq_num_start=*/0,
82 /*seq_num_end=*/0,
83 /*markerBit=*/true,
84 /*times_nacked=*/0,
85 /*first_packet_received_time=*/0,
86 /*last_packet_received_time=*/0,
87 /*rtp_timestamp=*/0,
88 /*ntp_time_ms=*/0,
89 VideoSendTiming(),
90 /*payload_type=*/0,
91 kVideoCodecVP8,
92 kVideoRotation_0,
93 VideoContentType::UNSPECIFIED,
94 video_header,
95 /*color_space=*/absl::nullopt,
96 RtpPacketInfos(),
97 EncodedImageBuffer::Create(/*size=*/0));
98 // clang-format on
99 }
100
101 private:
102 bool is_keyframe_ = false;
103 absl::optional<int> picture_id_;
104 absl::optional<int> temporal_id_;
105 absl::optional<int> tl0_idx_;
106 bool sync = false;
107 };
108
109 } // namespace
110
111 class RtpVp8RefFinderTest : public ::testing::Test {
112 protected:
RtpVp8RefFinderTest()113 RtpVp8RefFinderTest() : ref_finder_(std::make_unique<RtpVp8RefFinder>()) {}
114
Insert(std::unique_ptr<RtpFrameObject> frame)115 void Insert(std::unique_ptr<RtpFrameObject> frame) {
116 for (auto& f : ref_finder_->ManageFrame(std::move(frame))) {
117 frames_.push_back(std::move(f));
118 }
119 }
120
121 std::unique_ptr<RtpVp8RefFinder> ref_finder_;
122 std::vector<std::unique_ptr<EncodedFrame>> frames_;
123 };
124
TEST_F(RtpVp8RefFinderTest,Vp8RepeatedFrame_0)125 TEST_F(RtpVp8RefFinderTest, Vp8RepeatedFrame_0) {
126 Insert(Frame().Pid(0).Tid(0).Tl0(1).AsKeyFrame());
127 Insert(Frame().Pid(1).Tid(0).Tl0(2));
128 Insert(Frame().Pid(1).Tid(0).Tl0(2));
129
130 EXPECT_THAT(frames_, SizeIs(2));
131 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
132 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
133 }
134
TEST_F(RtpVp8RefFinderTest,Vp8RepeatedFrameLayerSync_01)135 TEST_F(RtpVp8RefFinderTest, Vp8RepeatedFrameLayerSync_01) {
136 Insert(Frame().Pid(0).Tid(0).Tl0(1).AsKeyFrame());
137 Insert(Frame().Pid(1).Tid(1).Tl0(1).AsSync());
138 Insert(Frame().Pid(1).Tid(1).Tl0(1).AsSync());
139
140 EXPECT_THAT(frames_, SizeIs(2));
141 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
142 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
143 }
144
TEST_F(RtpVp8RefFinderTest,Vp8RepeatedFrame_01)145 TEST_F(RtpVp8RefFinderTest, Vp8RepeatedFrame_01) {
146 Insert(Frame().Pid(0).Tid(0).Tl0(1).AsKeyFrame());
147 Insert(Frame().Pid(1).Tid(0).Tl0(2).AsSync());
148 Insert(Frame().Pid(2).Tid(0).Tl0(3));
149 Insert(Frame().Pid(3).Tid(0).Tl0(4));
150 Insert(Frame().Pid(3).Tid(0).Tl0(4));
151
152 EXPECT_THAT(frames_, SizeIs(4));
153 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
154 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
155 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {1}));
156 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {2}));
157 }
158
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayers_0)159 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayers_0) {
160 Insert(Frame().Pid(0).Tid(0).Tl0(1).AsKeyFrame());
161 Insert(Frame().Pid(1).Tid(0).Tl0(2));
162
163 EXPECT_THAT(frames_, SizeIs(2));
164 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
165 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
166 }
167
TEST_F(RtpVp8RefFinderTest,Vp8DuplicateTl1Frames)168 TEST_F(RtpVp8RefFinderTest, Vp8DuplicateTl1Frames) {
169 Insert(Frame().Pid(0).Tid(0).Tl0(0).AsKeyFrame());
170 Insert(Frame().Pid(1).Tid(1).Tl0(0).AsSync());
171 Insert(Frame().Pid(2).Tid(0).Tl0(1));
172 Insert(Frame().Pid(3).Tid(1).Tl0(1));
173 Insert(Frame().Pid(3).Tid(1).Tl0(1));
174 Insert(Frame().Pid(4).Tid(0).Tl0(2));
175 Insert(Frame().Pid(5).Tid(1).Tl0(2));
176
177 EXPECT_THAT(frames_, SizeIs(6));
178 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
179 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
180 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
181 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {1, 2}));
182 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {2}));
183 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {3, 4}));
184 }
185
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayersReordering_0)186 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayersReordering_0) {
187 Insert(Frame().Pid(1).Tid(0).Tl0(2));
188 Insert(Frame().Pid(0).Tid(0).Tl0(1).AsKeyFrame());
189 Insert(Frame().Pid(3).Tid(0).Tl0(4));
190 Insert(Frame().Pid(2).Tid(0).Tl0(3));
191 Insert(Frame().Pid(5).Tid(0).Tl0(6));
192 Insert(Frame().Pid(6).Tid(0).Tl0(7));
193 Insert(Frame().Pid(4).Tid(0).Tl0(5));
194
195 EXPECT_THAT(frames_, SizeIs(7));
196 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
197 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
198 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {1}));
199 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {2}));
200 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {3}));
201 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {4}));
202 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(6, {5}));
203 }
204
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayers_01)205 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayers_01) {
206 Insert(Frame().Pid(0).Tid(0).Tl0(255).AsKeyFrame());
207 Insert(Frame().Pid(1).Tid(1).Tl0(255).AsSync());
208 Insert(Frame().Pid(2).Tid(0).Tl0(0));
209 Insert(Frame().Pid(3).Tid(1).Tl0(0));
210
211 EXPECT_THAT(frames_, SizeIs(4));
212 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
213 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
214 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
215 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {1, 2}));
216 }
217
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayersReordering_01)218 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayersReordering_01) {
219 Insert(Frame().Pid(1).Tid(1).Tl0(255).AsSync());
220 Insert(Frame().Pid(0).Tid(0).Tl0(255).AsKeyFrame());
221 Insert(Frame().Pid(3).Tid(1).Tl0(0));
222 Insert(Frame().Pid(5).Tid(1).Tl0(1));
223 Insert(Frame().Pid(2).Tid(0).Tl0(0));
224 Insert(Frame().Pid(4).Tid(0).Tl0(1));
225 Insert(Frame().Pid(6).Tid(0).Tl0(2));
226 Insert(Frame().Pid(7).Tid(1).Tl0(2));
227
228 EXPECT_THAT(frames_, SizeIs(8));
229 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
230 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
231 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
232 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {1, 2}));
233 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {2}));
234 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {3, 4}));
235 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(6, {4}));
236 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(7, {5, 6}));
237 }
238
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayers_0212)239 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayers_0212) {
240 Insert(Frame().Pid(0).Tid(0).Tl0(55).AsKeyFrame());
241 Insert(Frame().Pid(1).Tid(2).Tl0(55).AsSync());
242 Insert(Frame().Pid(2).Tid(1).Tl0(55).AsSync());
243 Insert(Frame().Pid(3).Tid(2).Tl0(55));
244 Insert(Frame().Pid(4).Tid(0).Tl0(56));
245 Insert(Frame().Pid(5).Tid(2).Tl0(56));
246 Insert(Frame().Pid(6).Tid(1).Tl0(56));
247 Insert(Frame().Pid(7).Tid(2).Tl0(56));
248 Insert(Frame().Pid(8).Tid(0).Tl0(57));
249 Insert(Frame().Pid(9).Tid(2).Tl0(57).AsSync());
250 Insert(Frame().Pid(10).Tid(1).Tl0(57).AsSync());
251 Insert(Frame().Pid(11).Tid(2).Tl0(57));
252
253 EXPECT_THAT(frames_, SizeIs(12));
254 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
255 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
256 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
257 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {0, 1, 2}));
258 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {0}));
259 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {2, 3, 4}));
260 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(6, {2, 4}));
261 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(7, {4, 5, 6}));
262 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(8, {4}));
263 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(9, {8}));
264 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(10, {8}));
265 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(11, {8, 9, 10}));
266 }
267
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayersMissingFrame_0212)268 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayersMissingFrame_0212) {
269 Insert(Frame().Pid(0).Tid(0).Tl0(55).AsKeyFrame());
270 Insert(Frame().Pid(2).Tid(1).Tl0(55).AsSync());
271 Insert(Frame().Pid(3).Tid(2).Tl0(55));
272
273 EXPECT_THAT(frames_, SizeIs(2));
274 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
275 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
276 }
277
278 // Test with 3 temporal layers in a 0212 pattern.
TEST_F(RtpVp8RefFinderTest,Vp8TemporalLayersReordering_0212)279 TEST_F(RtpVp8RefFinderTest, Vp8TemporalLayersReordering_0212) {
280 Insert(Frame().Pid(127).Tid(2).Tl0(55).AsSync());
281 Insert(Frame().Pid(126).Tid(0).Tl0(55).AsKeyFrame());
282 Insert(Frame().Pid(128).Tid(1).Tl0(55).AsSync());
283 Insert(Frame().Pid(130).Tid(0).Tl0(56));
284 Insert(Frame().Pid(131).Tid(2).Tl0(56));
285 Insert(Frame().Pid(129).Tid(2).Tl0(55));
286 Insert(Frame().Pid(133).Tid(2).Tl0(56));
287 Insert(Frame().Pid(135).Tid(2).Tl0(57).AsSync());
288 Insert(Frame().Pid(132).Tid(1).Tl0(56));
289 Insert(Frame().Pid(134).Tid(0).Tl0(57));
290 Insert(Frame().Pid(137).Tid(2).Tl0(57));
291 Insert(Frame().Pid(136).Tid(1).Tl0(57).AsSync());
292
293 EXPECT_THAT(frames_, SizeIs(12));
294 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(126, {}));
295 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(127, {126}));
296 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(128, {126}));
297 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(129, {126, 127, 128}));
298 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(130, {126}));
299 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(131, {128, 129, 130}));
300 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(132, {128, 130}));
301 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(133, {130, 131, 132}));
302 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(134, {130}));
303 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(135, {134}));
304 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(136, {134}));
305 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(137, {134, 135, 136}));
306 }
307
TEST_F(RtpVp8RefFinderTest,Vp8LayerSync)308 TEST_F(RtpVp8RefFinderTest, Vp8LayerSync) {
309 Insert(Frame().Pid(0).Tid(0).Tl0(0).AsKeyFrame());
310 Insert(Frame().Pid(1).Tid(1).Tl0(0).AsSync());
311 Insert(Frame().Pid(2).Tid(0).Tl0(1));
312 Insert(Frame().Pid(4).Tid(0).Tl0(2));
313 Insert(Frame().Pid(5).Tid(1).Tl0(2).AsSync());
314 Insert(Frame().Pid(6).Tid(0).Tl0(3));
315 Insert(Frame().Pid(7).Tid(1).Tl0(3));
316
317 EXPECT_THAT(frames_, SizeIs(7));
318 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(0, {}));
319 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {0}));
320 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {0}));
321 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {2}));
322 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {4}));
323 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(6, {4}));
324 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(7, {5, 6}));
325 }
326
TEST_F(RtpVp8RefFinderTest,Vp8Tl1SyncFrameAfterTl1Frame)327 TEST_F(RtpVp8RefFinderTest, Vp8Tl1SyncFrameAfterTl1Frame) {
328 Insert(Frame().Pid(1).Tid(0).Tl0(247).AsKeyFrame().AsSync());
329 Insert(Frame().Pid(3).Tid(0).Tl0(248));
330 Insert(Frame().Pid(4).Tid(1).Tl0(248));
331 Insert(Frame().Pid(5).Tid(1).Tl0(248).AsSync());
332
333 EXPECT_THAT(frames_, SizeIs(3));
334 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {}));
335 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {1}));
336 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {3}));
337 }
338
TEST_F(RtpVp8RefFinderTest,Vp8DetectMissingFrame_0212)339 TEST_F(RtpVp8RefFinderTest, Vp8DetectMissingFrame_0212) {
340 Insert(Frame().Pid(1).Tid(0).Tl0(1).AsKeyFrame());
341 Insert(Frame().Pid(2).Tid(2).Tl0(1).AsSync());
342 Insert(Frame().Pid(3).Tid(1).Tl0(1).AsSync());
343 Insert(Frame().Pid(4).Tid(2).Tl0(1));
344 Insert(Frame().Pid(6).Tid(2).Tl0(2));
345 Insert(Frame().Pid(7).Tid(1).Tl0(2));
346 Insert(Frame().Pid(8).Tid(2).Tl0(2));
347 Insert(Frame().Pid(5).Tid(0).Tl0(2));
348
349 EXPECT_THAT(frames_, SizeIs(8));
350 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(1, {}));
351 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(2, {1}));
352 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(3, {1}));
353 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(4, {1, 2, 3}));
354 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(5, {1}));
355 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(6, {3, 4, 5}));
356 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(7, {3, 5}));
357 EXPECT_THAT(frames_, HasFrameWithIdAndRefs(8, {5, 6, 7}));
358 }
359
TEST_F(RtpVp8RefFinderTest,StashedFramesDoNotWrapTl0Backwards)360 TEST_F(RtpVp8RefFinderTest, StashedFramesDoNotWrapTl0Backwards) {
361 Insert(Frame().Pid(0).Tid(0).Tl0(0));
362 EXPECT_THAT(frames_, SizeIs(0));
363
364 Insert(Frame().Pid(128).Tid(0).Tl0(128).AsKeyFrame());
365 EXPECT_THAT(frames_, SizeIs(1));
366 Insert(Frame().Pid(129).Tid(0).Tl0(129));
367 EXPECT_THAT(frames_, SizeIs(2));
368 }
369
370 } // namespace webrtc
371