xref: /aosp_15_r20/external/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422) 
1 /*
2  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
12 
13 #include <cstdint>
14 #include <memory>
15 
16 #include "api/video/video_bitrate_allocation.h"
17 #include "api/video_codecs/video_codec.h"
18 #include "api/video_codecs/vp8_frame_config.h"
19 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
20 #include "modules/video_coding/include/video_codec_interface.h"
21 #include "modules/video_coding/utility/simulcast_rate_allocator.h"
22 #include "test/field_trial.h"
23 #include "test/gmock.h"
24 #include "test/gtest.h"
25 #include "vpx/vp8cx.h"
26 
27 // TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration().
28 
29 namespace webrtc {
30 namespace test {
31 namespace {
32 
33 using ::testing::Each;
34 
35 enum {
36   kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
37                         VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
38   kTemporalUpdateGoldenWithoutDependency =
39       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
40       VP8_EFLAG_NO_UPD_LAST,
41   kTemporalUpdateGolden =
42       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
43   kTemporalUpdateAltrefWithoutDependency =
44       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
45       VP8_EFLAG_NO_UPD_LAST,
46   kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
47   kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
48                         VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
49   kTemporalUpdateNoneNoRefAltRef =
50       VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
51       VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
52   kTemporalUpdateNoneNoRefGolden =
53       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
54       VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
55   kTemporalUpdateNoneNoRefGoldenAltRef =
56       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
57       VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
58   kTemporalUpdateGoldenWithoutDependencyRefAltRef =
59       VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
60   kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
61   kTemporalUpdateLastRefAltRef =
62       VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
63   kTemporalUpdateLastAndGoldenRefAltRef =
64       VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
65 };
66 
67 using BufferFlags = Vp8FrameConfig::BufferFlags;
68 using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
69 
70 constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone);
71 constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast);
72 constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden);
73 constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref);
74 constexpr uint8_t kAll = kLast | kGolden | kAltref;
75 
ToVp8CodecFlags(uint8_t referenced_buffers,uint8_t updated_buffers,bool update_entropy)76 constexpr int ToVp8CodecFlags(uint8_t referenced_buffers,
77                               uint8_t updated_buffers,
78                               bool update_entropy) {
79   return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) |
80          (((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) |
81          (((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) |
82          (((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) |
83          (((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) |
84          (((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) |
85          (update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY);
86 }
87 
88 constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true);
89 
GetTemporalLayerRates(int target_bitrate_kbps,int framerate_fps,int num_temporal_layers)90 std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps,
91                                             int framerate_fps,
92                                             int num_temporal_layers) {
93   VideoCodec codec;
94   codec.codecType = VideoCodecType::kVideoCodecVP8;
95   codec.numberOfSimulcastStreams = 1;
96   codec.maxBitrate = target_bitrate_kbps;
97   codec.maxFramerate = framerate_fps;
98   codec.simulcastStream[0].targetBitrate = target_bitrate_kbps;
99   codec.simulcastStream[0].maxBitrate = target_bitrate_kbps;
100   codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers;
101   codec.simulcastStream[0].active = true;
102   SimulcastRateAllocator allocator(codec);
103   return allocator
104       .Allocate(
105           VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps))
106       .GetTemporalLayerAllocation(0);
107 }
108 
109 constexpr int kDefaultBitrateBps = 500;
110 constexpr int kDefaultFramerate = 30;
111 constexpr int kDefaultBytesPerFrame =
112     (kDefaultBitrateBps / 8) / kDefaultFramerate;
113 constexpr int kDefaultQp = 2;
114 }  // namespace
115 
116 class TemporalLayersTest : public ::testing::Test {
117  public:
118   ~TemporalLayersTest() override = default;
119 
IgnoredCodecSpecificInfo()120   CodecSpecificInfo* IgnoredCodecSpecificInfo() {
121     codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
122     return codec_specific_info_.get();
123   }
124 
125  private:
126   std::unique_ptr<CodecSpecificInfo> codec_specific_info_;
127 };
128 
129 TEST_F(TemporalLayersTest, 2Layers) {
130   constexpr int kNumLayers = 2;
131   DefaultTemporalLayers tl(kNumLayers);
132   DefaultTemporalLayersChecker checker(kNumLayers);
133   tl.OnRatesUpdated(0,
134                     GetTemporalLayerRates(kDefaultBytesPerFrame,
135                                           kDefaultFramerate, kNumLayers),
136                     kDefaultFramerate);
137   tl.UpdateConfiguration(0);
138 
139   constexpr size_t kPatternSize = 4;
140   constexpr size_t kRepetitions = 4;
141 
142   const int expected_flags[kPatternSize] = {
143       ToVp8CodecFlags(kLast, kLast, true),
144       ToVp8CodecFlags(kLast, kGolden, true),
145       ToVp8CodecFlags(kLast, kLast, true),
146       ToVp8CodecFlags(kLast | kGolden, kNone, false),
147   };
148   const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1};
149   const bool expected_layer_sync[kPatternSize] = {false, true, false, false};
150 
151   uint32_t timestamp = 0;
152   for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
153     const size_t ind = i % kPatternSize;
154     const bool is_keyframe = (i == 0);
155     CodecSpecificInfo info;
156     Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
157     EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind],
158               LibvpxVp8Encoder::EncodeFlags(tl_config))
159         << i;
160     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
161                     kDefaultQp, &info);
162     EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
163     EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
164     EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
165     EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
166     EXPECT_EQ(is_keyframe || expected_layer_sync[ind],
167               info.codecSpecific.VP8.layerSync);
168     EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
169     timestamp += 3000;
170   }
171 }
172 
173 TEST_F(TemporalLayersTest, 3Layers) {
174   constexpr int kNumLayers = 3;
175   DefaultTemporalLayers tl(kNumLayers);
176   DefaultTemporalLayersChecker checker(kNumLayers);
177   tl.OnRatesUpdated(0,
178                     GetTemporalLayerRates(kDefaultBytesPerFrame,
179                                           kDefaultFramerate, kNumLayers),
180                     kDefaultFramerate);
181   tl.UpdateConfiguration(0);
182 
183   int expected_flags[16] = {
184       kTemporalUpdateLast,
185       kTemporalUpdateNoneNoRefGoldenAltRef,
186       kTemporalUpdateGoldenWithoutDependency,
187       kTemporalUpdateNoneNoRefAltRef,
188       kTemporalUpdateLast,
189       kTemporalUpdateNoneNoRefAltRef,
190       kTemporalUpdateGolden,
191       kTemporalUpdateNoneNoRefAltRef,
192       kTemporalUpdateLast,
193       kTemporalUpdateNoneNoRefGoldenAltRef,
194       kTemporalUpdateGoldenWithoutDependency,
195       kTemporalUpdateNoneNoRefAltRef,
196       kTemporalUpdateLast,
197       kTemporalUpdateNoneNoRefAltRef,
198       kTemporalUpdateGolden,
199       kTemporalUpdateNoneNoRefAltRef,
200   };
201   int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
202                                    0, 2, 1, 2, 0, 2, 1, 2};
203 
204   bool expected_layer_sync[16] = {false, true,  true,  false, false, false,
205                                   false, false, false, true,  true,  false,
206                                   false, false, false, false};
207 
208   unsigned int timestamp = 0;
209   for (int i = 0; i < 16; ++i) {
210     const bool is_keyframe = (i == 0);
211     CodecSpecificInfo info;
212     Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
213     EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
214               LibvpxVp8Encoder::EncodeFlags(tl_config))
215         << i;
216     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
217                     kDefaultQp, &info);
218     EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
219     EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
220     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
221     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
222     EXPECT_EQ(is_keyframe || expected_layer_sync[i],
223               info.codecSpecific.VP8.layerSync);
224     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
225     timestamp += 3000;
226   }
227 }
228 
TEST_F(TemporalLayersTest,Alternative3Layers)229 TEST_F(TemporalLayersTest, Alternative3Layers) {
230   constexpr int kNumLayers = 3;
231   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
232   DefaultTemporalLayers tl(kNumLayers);
233   DefaultTemporalLayersChecker checker(kNumLayers);
234   tl.OnRatesUpdated(0,
235                     GetTemporalLayerRates(kDefaultBytesPerFrame,
236                                           kDefaultFramerate, kNumLayers),
237                     kDefaultFramerate);
238   tl.UpdateConfiguration(0);
239 
240   int expected_flags[8] = {kTemporalUpdateLast,
241                            kTemporalUpdateAltrefWithoutDependency,
242                            kTemporalUpdateGoldenWithoutDependency,
243                            kTemporalUpdateNone,
244                            kTemporalUpdateLast,
245                            kTemporalUpdateAltrefWithoutDependency,
246                            kTemporalUpdateGoldenWithoutDependency,
247                            kTemporalUpdateNone};
248   int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
249 
250   bool expected_layer_sync[8] = {false, true, true, false,
251                                  false, true, true, false};
252 
253   unsigned int timestamp = 0;
254   for (int i = 0; i < 8; ++i) {
255     const bool is_keyframe = (i == 0);
256     CodecSpecificInfo info;
257     Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
258     EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
259               LibvpxVp8Encoder::EncodeFlags(tl_config))
260         << i;
261     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
262                     kDefaultQp, &info);
263     EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
264     EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
265     EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
266     EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
267     EXPECT_EQ(is_keyframe || expected_layer_sync[i],
268               info.codecSpecific.VP8.layerSync);
269     EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
270     timestamp += 3000;
271   }
272 }
273 
TEST_F(TemporalLayersTest,SearchOrder)274 TEST_F(TemporalLayersTest, SearchOrder) {
275   constexpr int kNumLayers = 3;
276   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
277   DefaultTemporalLayers tl(kNumLayers);
278   DefaultTemporalLayersChecker checker(kNumLayers);
279   tl.OnRatesUpdated(0,
280                     GetTemporalLayerRates(kDefaultBytesPerFrame,
281                                           kDefaultFramerate, kNumLayers),
282                     kDefaultFramerate);
283   tl.UpdateConfiguration(0);
284 
285   // Use a repeating pattern of tl 0, 2, 1, 2.
286   // Tl 0, 1, 2 update last, golden, altref respectively.
287 
288   // Start with a key-frame. tl_config flags can be ignored.
289   uint32_t timestamp = 0;
290   Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
291   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
292                   IgnoredCodecSpecificInfo());
293 
294   // TL2 frame. First one only references TL0. Updates altref.
295   tl_config = tl.NextFrameConfig(0, ++timestamp);
296   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
297                   IgnoredCodecSpecificInfo());
298   EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
299   EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
300 
301   // TL1 frame. Can only reference TL0. Updated golden.
302   tl_config = tl.NextFrameConfig(0, ++timestamp);
303   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
304                   IgnoredCodecSpecificInfo());
305   EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
306   EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
307 
308   // TL2 frame. Can reference all three buffers. Golden was the last to be
309   // updated, the next to last was altref.
310   tl_config = tl.NextFrameConfig(0, ++timestamp);
311   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
312                   IgnoredCodecSpecificInfo());
313   EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
314   EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
315 }
316 
TEST_F(TemporalLayersTest,SearchOrderWithDrop)317 TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
318   constexpr int kNumLayers = 3;
319   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
320   DefaultTemporalLayers tl(kNumLayers);
321   DefaultTemporalLayersChecker checker(kNumLayers);
322   tl.OnRatesUpdated(0,
323                     GetTemporalLayerRates(kDefaultBytesPerFrame,
324                                           kDefaultFramerate, kNumLayers),
325                     kDefaultFramerate);
326   tl.UpdateConfiguration(0);
327 
328   // Use a repeating pattern of tl 0, 2, 1, 2.
329   // Tl 0, 1, 2 update last, golden, altref respectively.
330 
331   // Start with a key-frame. tl_config flags can be ignored.
332   uint32_t timestamp = 0;
333   Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
334   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
335                   IgnoredCodecSpecificInfo());
336 
337   // TL2 frame. First one only references TL0. Updates altref.
338   tl_config = tl.NextFrameConfig(0, ++timestamp);
339   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
340                   IgnoredCodecSpecificInfo());
341   EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
342   EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
343 
344   // Dropped TL1 frame. Can only reference TL0. Should have updated golden.
345   tl_config = tl.NextFrameConfig(0, ++timestamp);
346   tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
347 
348   // TL2 frame. Can normally reference all three buffers, but golden has not
349   // been populated this cycle. Altref was last to be updated, before that last.
350   tl_config = tl.NextFrameConfig(0, ++timestamp);
351   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
352                   IgnoredCodecSpecificInfo());
353   EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
354   EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
355 }
356 
TEST_F(TemporalLayersTest,DoesNotReferenceDroppedFrames)357 TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
358   constexpr int kNumLayers = 3;
359   // Use a repeating pattern of tl 0, 2, 1, 2.
360   // Tl 0, 1, 2 update last, golden, altref respectively.
361   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
362   DefaultTemporalLayers tl(kNumLayers);
363   DefaultTemporalLayersChecker checker(kNumLayers);
364   tl.OnRatesUpdated(0,
365                     GetTemporalLayerRates(kDefaultBytesPerFrame,
366                                           kDefaultFramerate, kNumLayers),
367                     kDefaultFramerate);
368   tl.UpdateConfiguration(0);
369 
370   // Start with a keyframe.
371   uint32_t timestamp = 0;
372   Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
373   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
374                   IgnoredCodecSpecificInfo());
375 
376   // Dropped TL2 frame.
377   tl_config = tl.NextFrameConfig(0, ++timestamp);
378   tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
379 
380   // Dropped TL1 frame.
381   tl_config = tl.NextFrameConfig(0, ++timestamp);
382   tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
383 
384   // TL2 frame. Can reference all three buffers, valid since golden and altref
385   // both contain the last keyframe.
386   tl_config = tl.NextFrameConfig(0, ++timestamp);
387   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
388                   IgnoredCodecSpecificInfo());
389   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
390   EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
391   EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
392 
393   // Restart of cycle!
394 
395   // TL0 base layer frame, updating and referencing last.
396   tl_config = tl.NextFrameConfig(0, ++timestamp);
397   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
398                   IgnoredCodecSpecificInfo());
399 
400   // TL2 frame, updating altref.
401   tl_config = tl.NextFrameConfig(0, ++timestamp);
402   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
403                   IgnoredCodecSpecificInfo());
404 
405   // TL1 frame, updating golden.
406   tl_config = tl.NextFrameConfig(0, ++timestamp);
407   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
408                   IgnoredCodecSpecificInfo());
409 
410   // TL2 frame. Can still reference all buffer since they have been update this
411   // cycle.
412   tl_config = tl.NextFrameConfig(0, ++timestamp);
413   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
414                   IgnoredCodecSpecificInfo());
415   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
416   EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
417   EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
418 
419   // Restart of cycle!
420 
421   // TL0 base layer frame, updating and referencing last.
422   tl_config = tl.NextFrameConfig(0, ++timestamp);
423   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
424                   IgnoredCodecSpecificInfo());
425 
426   // Dropped TL2 frame.
427   tl_config = tl.NextFrameConfig(0, ++timestamp);
428   tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
429 
430   // Dropped TL1 frame.
431   tl_config = tl.NextFrameConfig(0, ++timestamp);
432   tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
433 
434   // TL2 frame. This time golden and altref contain data from the previous cycle
435   // and cannot be referenced.
436   tl_config = tl.NextFrameConfig(0, ++timestamp);
437   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
438                   IgnoredCodecSpecificInfo());
439   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
440   EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
441   EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
442 }
443 
TEST_F(TemporalLayersTest,DoesNotReferenceUnlessGuaranteedToExist)444 TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
445   constexpr int kNumLayers = 3;
446   // Use a repeating pattern of tl 0, 2, 1, 2.
447   // Tl 0, 1 updates last, golden respectively. Altref is always last keyframe.
448   DefaultTemporalLayers tl(kNumLayers);
449   DefaultTemporalLayersChecker checker(kNumLayers);
450   tl.OnRatesUpdated(0,
451                     GetTemporalLayerRates(kDefaultBytesPerFrame,
452                                           kDefaultFramerate, kNumLayers),
453                     kDefaultFramerate);
454   tl.UpdateConfiguration(0);
455 
456   // Start with a keyframe.
457   uint32_t timestamp = 0;
458   Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
459   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
460                   IgnoredCodecSpecificInfo());
461 
462   // Do a full cycle of the pattern.
463   for (int i = 0; i < 7; ++i) {
464     tl_config = tl.NextFrameConfig(0, ++timestamp);
465     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
466                     IgnoredCodecSpecificInfo());
467   }
468 
469   // TL0 base layer frame, starting the cycle over.
470   tl_config = tl.NextFrameConfig(0, ++timestamp);
471   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
472                   IgnoredCodecSpecificInfo());
473 
474   // TL2 frame.
475   tl_config = tl.NextFrameConfig(0, ++timestamp);
476   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
477                   IgnoredCodecSpecificInfo());
478 
479   // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
480   // TL1 frame, updating golden.
481   tl_config = tl.NextFrameConfig(0, ++timestamp);
482 
483   // TL2 frame, that should be referencing golden, but we can't be certain it's
484   // not going to be dropped, so that is not allowed.
485   tl_config = tl.NextFrameConfig(0, timestamp + 1);
486   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
487   EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
488   EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
489 
490   // TL0 base layer frame.
491   tl_config = tl.NextFrameConfig(0, timestamp + 2);
492 
493   // The previous four enqueued frames finally get encoded, and the updated
494   // buffers are now OK to reference.
495   // Enqueued TL1 frame ready.
496   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
497                   IgnoredCodecSpecificInfo());
498   // Enqueued TL2 frame.
499   tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
500                   IgnoredCodecSpecificInfo());
501   // Enqueued TL0 frame.
502   tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
503                   IgnoredCodecSpecificInfo());
504 
505   // TL2 frame, all buffers are now in a known good state, OK to reference.
506   tl_config = tl.NextFrameConfig(0, ++timestamp + 1);
507   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
508   EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
509   EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
510 }
511 
TEST_F(TemporalLayersTest,DoesNotReferenceUnlessGuaranteedToExistLongDelay)512 TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
513   constexpr int kNumLayers = 3;
514   // Use a repeating pattern of tl 0, 2, 1, 2.
515   // Tl 0, 1 updates last, golden, altref respectively.
516   ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
517   DefaultTemporalLayers tl(kNumLayers);
518   DefaultTemporalLayersChecker checker(kNumLayers);
519   tl.OnRatesUpdated(0,
520                     GetTemporalLayerRates(kDefaultBytesPerFrame,
521                                           kDefaultFramerate, kNumLayers),
522                     kDefaultFramerate);
523   tl.UpdateConfiguration(0);
524 
525   // Start with a keyframe.
526   uint32_t timestamp = 0;
527   Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
528   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
529                   IgnoredCodecSpecificInfo());
530 
531   // Do a full cycle of the pattern.
532   for (int i = 0; i < 3; ++i) {
533     tl_config = tl.NextFrameConfig(0, ++timestamp);
534     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
535                     IgnoredCodecSpecificInfo());
536   }
537 
538   // TL0 base layer frame, starting the cycle over.
539   tl_config = tl.NextFrameConfig(0, ++timestamp);
540   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
541                   IgnoredCodecSpecificInfo());
542 
543   // TL2 frame.
544   tl_config = tl.NextFrameConfig(0, ++timestamp);
545   tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
546                   IgnoredCodecSpecificInfo());
547 
548   // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
549   // Encoded, but delayed frames in TL 1, 2.
550   tl_config = tl.NextFrameConfig(0, timestamp + 1);
551   tl_config = tl.NextFrameConfig(0, timestamp + 2);
552 
553   // Restart of the pattern!
554 
555   // Encoded, but delayed frames in TL 2, 1.
556   tl_config = tl.NextFrameConfig(0, timestamp + 3);
557   tl_config = tl.NextFrameConfig(0, timestamp + 4);
558 
559   // TL1 frame from last cycle is ready.
560   tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
561                   IgnoredCodecSpecificInfo());
562   // TL2 frame from last cycle is ready.
563   tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
564                   IgnoredCodecSpecificInfo());
565 
566   // TL2 frame, that should be referencing all buffers, but altref and golden
567   // haven not been updated this cycle. (Don't be fooled by the late frames from
568   // the last cycle!)
569   tl_config = tl.NextFrameConfig(0, timestamp + 5);
570   EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
571   EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
572   EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
573 }
574 
TEST_F(TemporalLayersTest,KeyFrame)575 TEST_F(TemporalLayersTest, KeyFrame) {
576   constexpr int kNumLayers = 3;
577   DefaultTemporalLayers tl(kNumLayers);
578   DefaultTemporalLayersChecker checker(kNumLayers);
579   tl.OnRatesUpdated(0,
580                     GetTemporalLayerRates(kDefaultBytesPerFrame,
581                                           kDefaultFramerate, kNumLayers),
582                     kDefaultFramerate);
583   tl.UpdateConfiguration(0);
584 
585   int expected_flags[8] = {
586       kTemporalUpdateLastRefAltRef,
587       kTemporalUpdateNoneNoRefGoldenAltRef,
588       kTemporalUpdateGoldenWithoutDependency,
589       kTemporalUpdateNoneNoRefAltRef,
590       kTemporalUpdateLast,
591       kTemporalUpdateNoneNoRefAltRef,
592       kTemporalUpdateGolden,
593       kTemporalUpdateNone,
594   };
595   int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
596   bool expected_layer_sync[8] = {true,  true,  true,  false,
597                                  false, false, false, false};
598 
599   uint32_t timestamp = 0;
600   for (int i = 0; i < 7; ++i) {
601     // Temporal pattern starts from 0 after key frame. Let the first `i` - 1
602     // frames be delta frames, and the `i`th one key frame.
603     for (int j = 1; j <= i; ++j) {
604       // Since last frame was always a keyframe and thus index 0 in the pattern,
605       // this loop starts at index 1.
606       Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
607       EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
608           << j;
609       tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
610                       IgnoredCodecSpecificInfo());
611       EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
612       EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
613       EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
614       EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync);
615       timestamp += 3000;
616     }
617 
618     CodecSpecificInfo info;
619     Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
620     tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
621                     &info);
622     EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
623         << "Key frame should be marked layer sync.";
624     EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
625         << "Key frame should always be packetized as layer 0";
626     EXPECT_EQ(0, info.generic_frame_info->temporal_id)
627         << "Key frame should always be packetized as layer 0";
628     EXPECT_THAT(info.generic_frame_info->decode_target_indications,
629                 Each(DecodeTargetIndication::kSwitch))
630         << "Key frame is universal switch";
631     EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
632   }
633 }
634 
TEST_F(TemporalLayersTest,SetsTlCountOnFirstConfigUpdate)635 TEST_F(TemporalLayersTest, SetsTlCountOnFirstConfigUpdate) {
636   // Create an instance and fetch config update without setting any rate.
637   constexpr int kNumLayers = 2;
638   DefaultTemporalLayers tl(kNumLayers);
639   Vp8EncoderConfig config = tl.UpdateConfiguration(0);
640 
641   // Config should indicate correct number of temporal layers, but zero bitrate.
642   ASSERT_TRUE(config.temporal_layer_config.has_value());
643   EXPECT_EQ(config.temporal_layer_config->ts_number_layers,
644             uint32_t{kNumLayers});
645   std::array<uint32_t, Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers>
646       kZeroRate = {};
647   EXPECT_EQ(config.temporal_layer_config->ts_target_bitrate, kZeroRate);
648 
649   // On second call, no new update.
650   config = tl.UpdateConfiguration(0);
651   EXPECT_FALSE(config.temporal_layer_config.has_value());
652 }
653 
654 class TemporalLayersReferenceTest : public TemporalLayersTest,
655                                     public ::testing::WithParamInterface<int> {
656  public:
TemporalLayersReferenceTest()657   TemporalLayersReferenceTest()
658       : timestamp_(1),
659         last_sync_timestamp_(timestamp_),
660         tl0_reference_(nullptr) {}
~TemporalLayersReferenceTest()661   virtual ~TemporalLayersReferenceTest() {}
662 
663  protected:
664   static const int kMaxPatternLength = 32;
665 
666   struct BufferState {
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState667     BufferState() : BufferState(-1, 0, false) {}
BufferStatewebrtc::test::TemporalLayersReferenceTest::BufferState668     BufferState(int temporal_idx, uint32_t timestamp, bool sync)
669         : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
670     int temporal_idx;
671     uint32_t timestamp;
672     bool sync;
673   };
674 
UpdateSyncRefState(const BufferFlags & flags,BufferState * buffer_state)675   bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) {
676     if (flags & BufferFlags::kReference) {
677       if (buffer_state->temporal_idx == -1)
678         return true;  // References key-frame.
679       if (buffer_state->temporal_idx == 0) {
680         // No more than one reference to TL0 frame.
681         EXPECT_EQ(nullptr, tl0_reference_);
682         tl0_reference_ = buffer_state;
683         return true;
684       }
685       return false;  // References higher layer.
686     }
687     return true;  // No reference, does not affect sync frame status.
688   }
689 
ValidateReference(const BufferFlags & flags,const BufferState & buffer_state,int temporal_layer)690   void ValidateReference(const BufferFlags& flags,
691                          const BufferState& buffer_state,
692                          int temporal_layer) {
693     if (flags & BufferFlags::kReference) {
694       if (temporal_layer > 0 && buffer_state.timestamp > 0) {
695         // Check that high layer reference does not go past last sync frame.
696         EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
697       }
698       // No reference to buffer in higher layer.
699       EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
700     }
701   }
702 
703   uint32_t timestamp_ = 1;
704   uint32_t last_sync_timestamp_ = timestamp_;
705   BufferState* tl0_reference_;
706 
707   BufferState last_state;
708   BufferState golden_state;
709   BufferState altref_state;
710 };
711 
712 INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest,
713                          TemporalLayersReferenceTest,
714                          ::testing::Range(1, kMaxTemporalStreams + 1));
715 
TEST_P(TemporalLayersReferenceTest,ValidFrameConfigs)716 TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
717   const int num_layers = GetParam();
718   DefaultTemporalLayers tl(num_layers);
719   tl.OnRatesUpdated(
720       0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1),
721       kDefaultFramerate);
722   tl.UpdateConfiguration(0);
723 
724   // Run through the pattern and store the frame dependencies, plus keep track
725   // of the buffer state; which buffers references which temporal layers (if
726   // (any). If a given buffer is never updated, it is legal to reference it
727   // even for sync frames. In order to be general, don't assume TL0 always
728   // updates `last`.
729   std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength);
730   for (int i = 0; i < kMaxPatternLength; ++i) {
731     Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_);
732     tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
733                     IgnoredCodecSpecificInfo());
734     ++timestamp_;
735     EXPECT_FALSE(tl_config.drop_frame);
736     tl_configs.push_back(tl_config);
737     int temporal_idx = tl_config.encoder_layer_id;
738     // For the default layers, always keep encoder and rtp layers in sync.
739     EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
740 
741     // Determine if this frame is in a higher layer but references only TL0
742     // or untouched buffers, if so verify it is marked as a layer sync.
743     bool is_sync_frame = true;
744     tl0_reference_ = nullptr;
745     if (temporal_idx <= 0) {
746       is_sync_frame = false;  // TL0 by definition not a sync frame.
747     } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
748       is_sync_frame = false;
749     } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
750                                    &golden_state)) {
751       is_sync_frame = false;
752     } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
753       is_sync_frame = false;
754     }
755     if (is_sync_frame) {
756       // Cache timestamp for last found sync frame, so that we can verify no
757       // references back past this frame.
758       ASSERT_TRUE(tl0_reference_);
759       last_sync_timestamp_ = tl0_reference_->timestamp;
760     }
761     EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
762 
763     // Validate no reference from lower to high temporal layer, or backwards
764     // past last reference frame.
765     ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
766     ValidateReference(tl_config.golden_buffer_flags, golden_state,
767                       temporal_idx);
768     ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
769 
770     // Update the current layer state.
771     BufferState state = {temporal_idx, timestamp_, is_sync_frame};
772     if (tl_config.last_buffer_flags & BufferFlags::kUpdate)
773       last_state = state;
774     if (tl_config.golden_buffer_flags & BufferFlags::kUpdate)
775       golden_state = state;
776     if (tl_config.arf_buffer_flags & BufferFlags::kUpdate)
777       altref_state = state;
778   }
779 }
780 }  // namespace test
781 }  // namespace webrtc
782