1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "video/frame_encode_metadata_writer.h"
12
13 #include <cstddef>
14 #include <vector>
15
16 #include "api/video/i420_buffer.h"
17 #include "api/video/video_frame.h"
18 #include "api/video/video_timing.h"
19 #include "common_video/h264/h264_common.h"
20 #include "common_video/test/utilities.h"
21 #include "modules/video_coding/include/video_coding_defines.h"
22 #include "rtc_base/time_utils.h"
23 #include "test/gmock.h"
24 #include "test/gtest.h"
25
26 namespace webrtc {
27 namespace test {
28 namespace {
29
30 const rtc::scoped_refptr<I420Buffer> kFrameBuffer = I420Buffer::Create(4, 4);
31
FrameSize(const size_t & min_frame_size,const size_t & max_frame_size,const int & s,const int & i)32 inline size_t FrameSize(const size_t& min_frame_size,
33 const size_t& max_frame_size,
34 const int& s,
35 const int& i) {
36 return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size);
37 }
38
39 class FakeEncodedImageCallback : public EncodedImageCallback {
40 public:
FakeEncodedImageCallback()41 FakeEncodedImageCallback() : num_frames_dropped_(0) {}
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info)42 Result OnEncodedImage(const EncodedImage& encoded_image,
43 const CodecSpecificInfo* codec_specific_info) override {
44 return Result(Result::OK);
45 }
OnDroppedFrame(DropReason reason)46 void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; }
GetNumFramesDropped()47 size_t GetNumFramesDropped() { return num_frames_dropped_; }
48
49 private:
50 size_t num_frames_dropped_;
51 };
52
53 enum class FrameType {
54 kNormal,
55 kTiming,
56 kDropped,
57 };
58
IsTimingFrame(const EncodedImage & image)59 bool IsTimingFrame(const EncodedImage& image) {
60 return image.timing_.flags != VideoSendTiming::kInvalid &&
61 image.timing_.flags != VideoSendTiming::kNotTriggered;
62 }
63
64 // Emulates `num_frames` on `num_streams` frames with capture timestamps
65 // increased by 1 from 0. Size of each frame is between
66 // `min_frame_size` and `max_frame_size`, outliers are counted relatevely to
67 // `average_frame_sizes[]` for each stream.
GetTimingFrames(const int64_t delay_ms,const size_t min_frame_size,const size_t max_frame_size,std::vector<size_t> average_frame_sizes,const int num_streams,const int num_frames)68 std::vector<std::vector<FrameType>> GetTimingFrames(
69 const int64_t delay_ms,
70 const size_t min_frame_size,
71 const size_t max_frame_size,
72 std::vector<size_t> average_frame_sizes,
73 const int num_streams,
74 const int num_frames) {
75 FakeEncodedImageCallback sink;
76 FrameEncodeMetadataWriter encode_timer(&sink);
77 VideoCodec codec_settings;
78 codec_settings.numberOfSimulcastStreams = num_streams;
79 codec_settings.timing_frame_thresholds = {delay_ms,
80 kDefaultOutlierFrameSizePercent};
81 encode_timer.OnEncoderInit(codec_settings);
82 const size_t kFramerate = 30;
83 VideoBitrateAllocation bitrate_allocation;
84 for (int si = 0; si < num_streams; ++si) {
85 bitrate_allocation.SetBitrate(si, 0,
86 average_frame_sizes[si] * 8 * kFramerate);
87 }
88 encode_timer.OnSetRates(bitrate_allocation, kFramerate);
89
90 std::vector<std::vector<FrameType>> result(num_streams);
91 int64_t current_timestamp = 0;
92 for (int i = 0; i < num_frames; ++i) {
93 current_timestamp += 1;
94 VideoFrame frame = VideoFrame::Builder()
95 .set_timestamp_rtp(current_timestamp * 90)
96 .set_timestamp_ms(current_timestamp)
97 .set_video_frame_buffer(kFrameBuffer)
98 .build();
99 encode_timer.OnEncodeStarted(frame);
100 for (int si = 0; si < num_streams; ++si) {
101 // every (5+s)-th frame is dropped on s-th stream by design.
102 bool dropped = i % (5 + si) == 0;
103
104 EncodedImage image;
105 image.SetEncodedData(EncodedImageBuffer::Create(max_frame_size));
106 image.set_size(FrameSize(min_frame_size, max_frame_size, si, i));
107 image.capture_time_ms_ = current_timestamp;
108 image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90));
109 image.SetSpatialIndex(si);
110
111 if (dropped) {
112 result[si].push_back(FrameType::kDropped);
113 continue;
114 }
115
116 encode_timer.FillTimingInfo(si, &image);
117
118 if (IsTimingFrame(image)) {
119 result[si].push_back(FrameType::kTiming);
120 } else {
121 result[si].push_back(FrameType::kNormal);
122 }
123 }
124 }
125 return result;
126 }
127 } // namespace
128
TEST(FrameEncodeMetadataWriterTest,MarksTimingFramesPeriodicallyTogether)129 TEST(FrameEncodeMetadataWriterTest, MarksTimingFramesPeriodicallyTogether) {
130 const int64_t kDelayMs = 29;
131 const size_t kMinFrameSize = 10;
132 const size_t kMaxFrameSize = 20;
133 const int kNumFrames = 1000;
134 const int kNumStreams = 3;
135 // No outliers as 1000 is larger than anything from range [10,20].
136 const std::vector<size_t> kAverageSize = {1000, 1000, 1000};
137 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
138 kAverageSize, kNumStreams, kNumFrames);
139 // Timing frames should be tirggered every delayMs.
140 // As no outliers are expected, frames on all streams have to be
141 // marked together.
142 int last_timing_frame = -1;
143 for (int i = 0; i < kNumFrames; ++i) {
144 int num_normal = 0;
145 int num_timing = 0;
146 int num_dropped = 0;
147 for (int s = 0; s < kNumStreams; ++s) {
148 if (frames[s][i] == FrameType::kTiming) {
149 ++num_timing;
150 } else if (frames[s][i] == FrameType::kNormal) {
151 ++num_normal;
152 } else {
153 ++num_dropped;
154 }
155 }
156 // Can't have both normal and timing frames at the same timstamp.
157 EXPECT_TRUE(num_timing == 0 || num_normal == 0);
158 if (num_dropped < kNumStreams) {
159 if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) {
160 // If didn't have timing frames for a period, current sent frame has to
161 // be one. No normal frames should be sent.
162 EXPECT_EQ(num_normal, 0);
163 } else {
164 // No unneeded timing frames should be sent.
165 EXPECT_EQ(num_timing, 0);
166 }
167 }
168 if (num_timing > 0)
169 last_timing_frame = i;
170 }
171 }
172
TEST(FrameEncodeMetadataWriterTest,MarksOutliers)173 TEST(FrameEncodeMetadataWriterTest, MarksOutliers) {
174 const int64_t kDelayMs = 29;
175 const size_t kMinFrameSize = 2495;
176 const size_t kMaxFrameSize = 2505;
177 const int kNumFrames = 1000;
178 const int kNumStreams = 3;
179 // Possible outliers as 1000 lies in range [995, 1005].
180 const std::vector<size_t> kAverageSize = {998, 1000, 1004};
181 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
182 kAverageSize, kNumStreams, kNumFrames);
183 // All outliers should be marked.
184 for (int i = 0; i < kNumFrames; ++i) {
185 for (int s = 0; s < kNumStreams; ++s) {
186 if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >=
187 kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) {
188 // Too big frame. May be dropped or timing, but not normal.
189 EXPECT_NE(frames[s][i], FrameType::kNormal);
190 }
191 }
192 }
193 }
194
TEST(FrameEncodeMetadataWriterTest,NoTimingFrameIfNoEncodeStartTime)195 TEST(FrameEncodeMetadataWriterTest, NoTimingFrameIfNoEncodeStartTime) {
196 int64_t timestamp = 1;
197 constexpr size_t kFrameSize = 500;
198 EncodedImage image;
199 image.SetEncodedData(EncodedImageBuffer::Create(kFrameSize));
200 image.capture_time_ms_ = timestamp;
201 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
202
203 FakeEncodedImageCallback sink;
204 FrameEncodeMetadataWriter encode_timer(&sink);
205 VideoCodec codec_settings;
206 // Make all frames timing frames.
207 codec_settings.timing_frame_thresholds.delay_ms = 1;
208 encode_timer.OnEncoderInit(codec_settings);
209 VideoBitrateAllocation bitrate_allocation;
210 bitrate_allocation.SetBitrate(0, 0, 500000);
211 encode_timer.OnSetRates(bitrate_allocation, 30);
212
213 // Verify a single frame works with encode start time set.
214 VideoFrame frame = VideoFrame::Builder()
215 .set_timestamp_ms(timestamp)
216 .set_timestamp_rtp(timestamp * 90)
217 .set_video_frame_buffer(kFrameBuffer)
218 .build();
219 encode_timer.OnEncodeStarted(frame);
220 encode_timer.FillTimingInfo(0, &image);
221 EXPECT_TRUE(IsTimingFrame(image));
222
223 // New frame, now skip OnEncodeStarted. Should not result in timing frame.
224 image.capture_time_ms_ = ++timestamp;
225 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
226 image.timing_ = EncodedImage::Timing();
227 encode_timer.FillTimingInfo(0, &image);
228 EXPECT_FALSE(IsTimingFrame(image));
229 }
230
TEST(FrameEncodeMetadataWriterTest,NotifiesAboutDroppedFrames)231 TEST(FrameEncodeMetadataWriterTest, NotifiesAboutDroppedFrames) {
232 const int64_t kTimestampMs1 = 47721840;
233 const int64_t kTimestampMs2 = 47721850;
234 const int64_t kTimestampMs3 = 47721860;
235 const int64_t kTimestampMs4 = 47721870;
236
237 FakeEncodedImageCallback sink;
238 FrameEncodeMetadataWriter encode_timer(&sink);
239 encode_timer.OnEncoderInit(VideoCodec());
240 // Any non-zero bitrate needed to be set before the first frame.
241 VideoBitrateAllocation bitrate_allocation;
242 bitrate_allocation.SetBitrate(0, 0, 500000);
243 encode_timer.OnSetRates(bitrate_allocation, 30);
244
245 EncodedImage image;
246 VideoFrame frame = VideoFrame::Builder()
247 .set_timestamp_rtp(kTimestampMs1 * 90)
248 .set_timestamp_ms(kTimestampMs1)
249 .set_video_frame_buffer(kFrameBuffer)
250 .build();
251
252 image.capture_time_ms_ = kTimestampMs1;
253 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
254 frame.set_timestamp(image.capture_time_ms_ * 90);
255 frame.set_timestamp_us(image.capture_time_ms_ * 1000);
256 encode_timer.OnEncodeStarted(frame);
257
258 EXPECT_EQ(0u, sink.GetNumFramesDropped());
259 encode_timer.FillTimingInfo(0, &image);
260
261 image.capture_time_ms_ = kTimestampMs2;
262 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
263 image.timing_ = EncodedImage::Timing();
264 frame.set_timestamp(image.capture_time_ms_ * 90);
265 frame.set_timestamp_us(image.capture_time_ms_ * 1000);
266 encode_timer.OnEncodeStarted(frame);
267 // No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known
268 // that frame with timestamp2 was dropped.
269 EXPECT_EQ(0u, sink.GetNumFramesDropped());
270
271 image.capture_time_ms_ = kTimestampMs3;
272 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
273 image.timing_ = EncodedImage::Timing();
274 frame.set_timestamp(image.capture_time_ms_ * 90);
275 frame.set_timestamp_us(image.capture_time_ms_ * 1000);
276 encode_timer.OnEncodeStarted(frame);
277 encode_timer.FillTimingInfo(0, &image);
278 EXPECT_EQ(1u, sink.GetNumFramesDropped());
279
280 image.capture_time_ms_ = kTimestampMs4;
281 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
282 image.timing_ = EncodedImage::Timing();
283 frame.set_timestamp(image.capture_time_ms_ * 90);
284 frame.set_timestamp_us(image.capture_time_ms_ * 1000);
285 encode_timer.OnEncodeStarted(frame);
286 encode_timer.FillTimingInfo(0, &image);
287 EXPECT_EQ(1u, sink.GetNumFramesDropped());
288 }
289
TEST(FrameEncodeMetadataWriterTest,RestoresCaptureTimestamps)290 TEST(FrameEncodeMetadataWriterTest, RestoresCaptureTimestamps) {
291 EncodedImage image;
292 const int64_t kTimestampMs = 123456;
293 FakeEncodedImageCallback sink;
294
295 FrameEncodeMetadataWriter encode_timer(&sink);
296 encode_timer.OnEncoderInit(VideoCodec());
297 // Any non-zero bitrate needed to be set before the first frame.
298 VideoBitrateAllocation bitrate_allocation;
299 bitrate_allocation.SetBitrate(0, 0, 500000);
300 encode_timer.OnSetRates(bitrate_allocation, 30);
301
302 image.capture_time_ms_ = kTimestampMs; // Correct timestamp.
303 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
304 VideoFrame frame = VideoFrame::Builder()
305 .set_timestamp_ms(image.capture_time_ms_)
306 .set_timestamp_rtp(image.capture_time_ms_ * 90)
307 .set_video_frame_buffer(kFrameBuffer)
308 .build();
309 encode_timer.OnEncodeStarted(frame);
310 image.capture_time_ms_ = 0; // Incorrect timestamp.
311 encode_timer.FillTimingInfo(0, &image);
312 EXPECT_EQ(kTimestampMs, image.capture_time_ms_);
313 }
314
TEST(FrameEncodeMetadataWriterTest,CopiesRotation)315 TEST(FrameEncodeMetadataWriterTest, CopiesRotation) {
316 EncodedImage image;
317 const int64_t kTimestampMs = 123456;
318 FakeEncodedImageCallback sink;
319
320 FrameEncodeMetadataWriter encode_timer(&sink);
321 encode_timer.OnEncoderInit(VideoCodec());
322 // Any non-zero bitrate needed to be set before the first frame.
323 VideoBitrateAllocation bitrate_allocation;
324 bitrate_allocation.SetBitrate(0, 0, 500000);
325 encode_timer.OnSetRates(bitrate_allocation, 30);
326
327 image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
328 VideoFrame frame = VideoFrame::Builder()
329 .set_timestamp_ms(kTimestampMs)
330 .set_timestamp_rtp(kTimestampMs * 90)
331 .set_rotation(kVideoRotation_180)
332 .set_video_frame_buffer(kFrameBuffer)
333 .build();
334 encode_timer.OnEncodeStarted(frame);
335 encode_timer.FillTimingInfo(0, &image);
336 EXPECT_EQ(kVideoRotation_180, image.rotation_);
337 }
338
TEST(FrameEncodeMetadataWriterTest,SetsContentType)339 TEST(FrameEncodeMetadataWriterTest, SetsContentType) {
340 EncodedImage image;
341 const int64_t kTimestampMs = 123456;
342 FakeEncodedImageCallback sink;
343
344 FrameEncodeMetadataWriter encode_timer(&sink);
345 VideoCodec codec;
346 codec.mode = VideoCodecMode::kScreensharing;
347 encode_timer.OnEncoderInit(codec);
348 // Any non-zero bitrate needed to be set before the first frame.
349 VideoBitrateAllocation bitrate_allocation;
350 bitrate_allocation.SetBitrate(0, 0, 500000);
351 encode_timer.OnSetRates(bitrate_allocation, 30);
352
353 image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
354 VideoFrame frame = VideoFrame::Builder()
355 .set_timestamp_ms(kTimestampMs)
356 .set_timestamp_rtp(kTimestampMs * 90)
357 .set_rotation(kVideoRotation_180)
358 .set_video_frame_buffer(kFrameBuffer)
359 .build();
360 encode_timer.OnEncodeStarted(frame);
361 encode_timer.FillTimingInfo(0, &image);
362 EXPECT_EQ(VideoContentType::SCREENSHARE, image.content_type_);
363 }
364
TEST(FrameEncodeMetadataWriterTest,CopiesColorSpace)365 TEST(FrameEncodeMetadataWriterTest, CopiesColorSpace) {
366 EncodedImage image;
367 const int64_t kTimestampMs = 123456;
368 FakeEncodedImageCallback sink;
369
370 FrameEncodeMetadataWriter encode_timer(&sink);
371 encode_timer.OnEncoderInit(VideoCodec());
372 // Any non-zero bitrate needed to be set before the first frame.
373 VideoBitrateAllocation bitrate_allocation;
374 bitrate_allocation.SetBitrate(0, 0, 500000);
375 encode_timer.OnSetRates(bitrate_allocation, 30);
376
377 webrtc::ColorSpace color_space =
378 CreateTestColorSpace(/*with_hdr_metadata=*/true);
379 image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
380 VideoFrame frame = VideoFrame::Builder()
381 .set_timestamp_ms(kTimestampMs)
382 .set_timestamp_rtp(kTimestampMs * 90)
383 .set_color_space(color_space)
384 .set_video_frame_buffer(kFrameBuffer)
385 .build();
386 encode_timer.OnEncodeStarted(frame);
387 encode_timer.FillTimingInfo(0, &image);
388 ASSERT_NE(image.ColorSpace(), nullptr);
389 EXPECT_EQ(color_space, *image.ColorSpace());
390 }
391
TEST(FrameEncodeMetadataWriterTest,CopiesPacketInfos)392 TEST(FrameEncodeMetadataWriterTest, CopiesPacketInfos) {
393 EncodedImage image;
394 const int64_t kTimestampMs = 123456;
395 FakeEncodedImageCallback sink;
396
397 FrameEncodeMetadataWriter encode_timer(&sink);
398 encode_timer.OnEncoderInit(VideoCodec());
399 // Any non-zero bitrate needed to be set before the first frame.
400 VideoBitrateAllocation bitrate_allocation;
401 bitrate_allocation.SetBitrate(0, 0, 500000);
402 encode_timer.OnSetRates(bitrate_allocation, 30);
403
404 RtpPacketInfos packet_infos = CreatePacketInfos(3);
405 image.SetTimestamp(static_cast<uint32_t>(kTimestampMs * 90));
406 VideoFrame frame = VideoFrame::Builder()
407 .set_timestamp_ms(kTimestampMs)
408 .set_timestamp_rtp(kTimestampMs * 90)
409 .set_packet_infos(packet_infos)
410 .set_video_frame_buffer(kFrameBuffer)
411 .build();
412 encode_timer.OnEncodeStarted(frame);
413 encode_timer.FillTimingInfo(0, &image);
414 EXPECT_EQ(image.PacketInfos().size(), 3U);
415 }
416
TEST(FrameEncodeMetadataWriterTest,DoesNotRewriteBitstreamWithoutCodecInfo)417 TEST(FrameEncodeMetadataWriterTest, DoesNotRewriteBitstreamWithoutCodecInfo) {
418 uint8_t buffer[] = {1, 2, 3};
419 auto image_buffer = EncodedImageBuffer::Create(buffer, sizeof(buffer));
420 EncodedImage image;
421 image.SetEncodedData(image_buffer);
422
423 FakeEncodedImageCallback sink;
424 FrameEncodeMetadataWriter encode_metadata_writer(&sink);
425 encode_metadata_writer.UpdateBitstream(nullptr, &image);
426 EXPECT_EQ(image.GetEncodedData(), image_buffer);
427 EXPECT_EQ(image.size(), sizeof(buffer));
428 }
429
TEST(FrameEncodeMetadataWriterTest,DoesNotRewriteVp8Bitstream)430 TEST(FrameEncodeMetadataWriterTest, DoesNotRewriteVp8Bitstream) {
431 uint8_t buffer[] = {1, 2, 3};
432 auto image_buffer = EncodedImageBuffer::Create(buffer, sizeof(buffer));
433 EncodedImage image;
434 image.SetEncodedData(image_buffer);
435 CodecSpecificInfo codec_specific_info;
436 codec_specific_info.codecType = kVideoCodecVP8;
437
438 FakeEncodedImageCallback sink;
439 FrameEncodeMetadataWriter encode_metadata_writer(&sink);
440 encode_metadata_writer.UpdateBitstream(&codec_specific_info, &image);
441 EXPECT_EQ(image.GetEncodedData(), image_buffer);
442 EXPECT_EQ(image.size(), sizeof(buffer));
443 }
444
TEST(FrameEncodeMetadataWriterTest,RewritesH264BitstreamWithNonOptimalSps)445 TEST(FrameEncodeMetadataWriterTest, RewritesH264BitstreamWithNonOptimalSps) {
446 const uint8_t kOriginalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
447 0x00, 0x00, 0x03, 0x03, 0xF4,
448 0x05, 0x03, 0xC7, 0xC0};
449 const uint8_t kRewrittenSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
450 0x00, 0x00, 0x03, 0x03, 0xF4,
451 0x05, 0x03, 0xC7, 0xE0, 0x1B,
452 0x41, 0x10, 0x8D, 0x00};
453
454 EncodedImage image;
455 image.SetEncodedData(
456 EncodedImageBuffer::Create(kOriginalSps, sizeof(kOriginalSps)));
457 image._frameType = VideoFrameType::kVideoFrameKey;
458
459 CodecSpecificInfo codec_specific_info;
460 codec_specific_info.codecType = kVideoCodecH264;
461
462 FakeEncodedImageCallback sink;
463 FrameEncodeMetadataWriter encode_metadata_writer(&sink);
464 encode_metadata_writer.UpdateBitstream(&codec_specific_info, &image);
465
466 EXPECT_THAT(std::vector<uint8_t>(image.data(), image.data() + image.size()),
467 testing::ElementsAreArray(kRewrittenSps));
468 }
469
470 } // namespace test
471 } // namespace webrtc
472