1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "common_video/h264/sps_vui_rewriter.h"
12
13 #include <cstdint>
14 #include <vector>
15
16 #include "api/video/color_space.h"
17 #include "common_video/h264/h264_common.h"
18 #include "rtc_base/bit_buffer.h"
19 #include "rtc_base/buffer.h"
20 #include "rtc_base/logging.h"
21 #include "test/gmock.h"
22 #include "test/gtest.h"
23
24 namespace webrtc {
25
26 namespace {
27 enum SpsMode {
28 kNoRewriteRequired_VuiOptimal,
29 kRewriteRequired_NoVui,
30 kRewriteRequired_NoBitstreamRestriction,
31 kRewriteRequired_VuiSuboptimal,
32 };
33
34 static const size_t kSpsBufferMaxSize = 256;
35 static const size_t kWidth = 640;
36 static const size_t kHeight = 480;
37
38 static const uint8_t kStartSequence[] = {0x00, 0x00, 0x00, 0x01};
39 static const uint8_t kAud[] = {H264::NaluType::kAud, 0x09, 0x10};
40 static const uint8_t kSpsNaluType[] = {H264::NaluType::kSps};
41 static const uint8_t kIdr1[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x00, 0x04};
42 static const uint8_t kIdr2[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x11};
43
44 struct VuiHeader {
45 uint32_t vui_parameters_present_flag;
46 uint32_t bitstream_restriction_flag;
47 uint32_t max_num_reorder_frames;
48 uint32_t max_dec_frame_buffering;
49 uint32_t video_signal_type_present_flag;
50 uint32_t video_full_range_flag;
51 uint32_t colour_description_present_flag;
52 uint8_t colour_primaries;
53 uint8_t transfer_characteristics;
54 uint8_t matrix_coefficients;
55 };
56
57 static const VuiHeader kVuiNotPresent = {
58 /* vui_parameters_present_flag= */ 0,
59 /* bitstream_restriction_flag= */ 0,
60 /* max_num_reorder_frames= */ 0,
61 /* max_dec_frame_buffering= */ 0,
62 /* video_signal_type_present_flag= */ 0,
63 /* video_full_range_flag= */ 0,
64 /* colour_description_present_flag= */ 0,
65 /* colour_primaries= */ 0,
66 /* transfer_characteristics= */ 0,
67 /* matrix_coefficients= */ 0};
68
69 static const VuiHeader kVuiNoBitstreamRestriction = {
70 /* vui_parameters_present_flag= */ 1,
71 /* bitstream_restriction_flag= */ 0,
72 /* max_num_reorder_frames= */ 0,
73 /* max_dec_frame_buffering= */ 0,
74 /* video_signal_type_present_flag= */ 0,
75 /* video_full_range_flag= */ 0,
76 /* colour_description_present_flag= */ 0,
77 /* colour_primaries= */ 0,
78 /* transfer_characteristics= */ 0,
79 /* matrix_coefficients= */ 0};
80
81 static const VuiHeader kVuiNoFrameBuffering = {
82 /* vui_parameters_present_flag= */ 1,
83 /* bitstream_restriction_flag= */ 1,
84 /* max_num_reorder_frames= */ 0,
85 /* max_dec_frame_buffering= */ 1,
86 /* video_signal_type_present_flag= */ 0,
87 /* video_full_range_flag= */ 0,
88 /* colour_description_present_flag= */ 0,
89 /* colour_primaries= */ 0,
90 /* transfer_characteristics= */ 0,
91 /* matrix_coefficients= */ 0};
92
93 static const VuiHeader kVuiFrameBuffering = {
94 /* vui_parameters_present_flag= */ 1,
95 /* bitstream_restriction_flag= */ 1,
96 /* max_num_reorder_frames= */ 3,
97 /* max_dec_frame_buffering= */ 3,
98 /* video_signal_type_present_flag= */ 0,
99 /* video_full_range_flag= */ 0,
100 /* colour_description_present_flag= */ 0,
101 /* colour_primaries= */ 0,
102 /* transfer_characteristics= */ 0,
103 /* matrix_coefficients= */ 0};
104
105 static const VuiHeader kVuiNoVideoSignalType = {
106 /* vui_parameters_present_flag= */ 1,
107 /* bitstream_restriction_flag= */ 1,
108 /* max_num_reorder_frames= */ 0,
109 /* max_dec_frame_buffering= */ 1,
110 /* video_signal_type_present_flag= */ 0,
111 /* video_full_range_flag= */ 0,
112 /* colour_description_present_flag= */ 0,
113 /* colour_primaries= */ 0,
114 /* transfer_characteristics= */ 0,
115 /* matrix_coefficients= */ 0};
116
117 static const VuiHeader kVuiLimitedRangeNoColourDescription = {
118 /* vui_parameters_present_flag= */ 1,
119 /* bitstream_restriction_flag= */ 1,
120 /* max_num_reorder_frames= */ 0,
121 /* max_dec_frame_buffering= */ 1,
122 /* video_signal_type_present_flag= */ 1,
123 /* video_full_range_flag= */ 0,
124 /* colour_description_present_flag= */ 0,
125 /* colour_primaries= */ 0,
126 /* transfer_characteristics= */ 0,
127 /* matrix_coefficients= */ 0};
128
129 static const VuiHeader kVuiFullRangeNoColourDescription = {
130 /* vui_parameters_present_flag= */ 1,
131 /* bitstream_restriction_flag= */ 1,
132 /* max_num_reorder_frames= */ 0,
133 /* max_dec_frame_buffering= */ 1,
134 /* video_signal_type_present_flag= */ 1,
135 /* video_full_range_flag= */ 1,
136 /* colour_description_present_flag= */ 0,
137 /* colour_primaries= */ 0,
138 /* transfer_characteristics= */ 0,
139 /* matrix_coefficients= */ 0};
140
141 static const VuiHeader kVuiLimitedRangeBt709Color = {
142 /* vui_parameters_present_flag= */ 1,
143 /* bitstream_restriction_flag= */ 1,
144 /* max_num_reorder_frames= */ 0,
145 /* max_dec_frame_buffering= */ 1,
146 /* video_signal_type_present_flag= */ 1,
147 /* video_full_range_flag= */ 0,
148 /* colour_description_present_flag= */ 1,
149 /* colour_primaries= */ 1,
150 /* transfer_characteristics= */ 1,
151 /* matrix_coefficients= */ 1};
152
153 static const webrtc::ColorSpace kColorSpaceH264Default(
154 ColorSpace::PrimaryID::kUnspecified,
155 ColorSpace::TransferID::kUnspecified,
156 ColorSpace::MatrixID::kUnspecified,
157 ColorSpace::RangeID::kLimited);
158
159 static const webrtc::ColorSpace kColorSpacePrimariesBt709(
160 ColorSpace::PrimaryID::kBT709,
161 ColorSpace::TransferID::kUnspecified,
162 ColorSpace::MatrixID::kUnspecified,
163 ColorSpace::RangeID::kLimited);
164
165 static const webrtc::ColorSpace kColorSpaceTransferBt709(
166 ColorSpace::PrimaryID::kUnspecified,
167 ColorSpace::TransferID::kBT709,
168 ColorSpace::MatrixID::kUnspecified,
169 ColorSpace::RangeID::kLimited);
170
171 static const webrtc::ColorSpace kColorSpaceMatrixBt709(
172 ColorSpace::PrimaryID::kUnspecified,
173 ColorSpace::TransferID::kUnspecified,
174 ColorSpace::MatrixID::kBT709,
175 ColorSpace::RangeID::kLimited);
176
177 static const webrtc::ColorSpace kColorSpaceFullRange(
178 ColorSpace::PrimaryID::kBT709,
179 ColorSpace::TransferID::kUnspecified,
180 ColorSpace::MatrixID::kUnspecified,
181 ColorSpace::RangeID::kFull);
182
183 static const webrtc::ColorSpace kColorSpaceBt709LimitedRange(
184 ColorSpace::PrimaryID::kBT709,
185 ColorSpace::TransferID::kBT709,
186 ColorSpace::MatrixID::kBT709,
187 ColorSpace::RangeID::kLimited);
188 } // namespace
189
190 // Generates a fake SPS with basically everything empty and with characteristics
191 // based off SpsMode.
192 // Pass in a buffer of at least kSpsBufferMaxSize.
193 // The fake SPS that this generates also always has at least one emulation byte
194 // at offset 2, since the first two bytes are always 0, and has a 0x3 as the
195 // level_idc, to make sure the parser doesn't eat all 0x3 bytes.
GenerateFakeSps(const VuiHeader & vui,rtc::Buffer * out_buffer)196 void GenerateFakeSps(const VuiHeader& vui, rtc::Buffer* out_buffer) {
197 uint8_t rbsp[kSpsBufferMaxSize] = {0};
198 rtc::BitBufferWriter writer(rbsp, kSpsBufferMaxSize);
199 // Profile byte.
200 writer.WriteUInt8(0);
201 // Constraint sets and reserved zero bits.
202 writer.WriteUInt8(0);
203 // level_idc.
204 writer.WriteUInt8(3);
205 // seq_paramter_set_id.
206 writer.WriteExponentialGolomb(0);
207 // Profile is not special, so we skip all the chroma format settings.
208
209 // Now some bit magic.
210 // log2_max_frame_num_minus4: ue(v). 0 is fine.
211 writer.WriteExponentialGolomb(0);
212 // pic_order_cnt_type: ue(v).
213 writer.WriteExponentialGolomb(0);
214 // log2_max_pic_order_cnt_lsb_minus4: ue(v). 0 is fine.
215 writer.WriteExponentialGolomb(0);
216
217 // max_num_ref_frames: ue(v). Use 1, to make optimal/suboptimal more obvious.
218 writer.WriteExponentialGolomb(1);
219 // gaps_in_frame_num_value_allowed_flag: u(1).
220 writer.WriteBits(0, 1);
221 // Next are width/height. First, calculate the mbs/map_units versions.
222 uint16_t width_in_mbs_minus1 = (kWidth + 15) / 16 - 1;
223
224 // For the height, we're going to define frame_mbs_only_flag, so we need to
225 // divide by 2. See the parser for the full calculation.
226 uint16_t height_in_map_units_minus1 = ((kHeight + 15) / 16 - 1) / 2;
227 // Write each as ue(v).
228 writer.WriteExponentialGolomb(width_in_mbs_minus1);
229 writer.WriteExponentialGolomb(height_in_map_units_minus1);
230 // frame_mbs_only_flag: u(1). Needs to be false.
231 writer.WriteBits(0, 1);
232 // mb_adaptive_frame_field_flag: u(1).
233 writer.WriteBits(0, 1);
234 // direct_8x8_inferene_flag: u(1).
235 writer.WriteBits(0, 1);
236 // frame_cropping_flag: u(1). 1, so we can supply crop.
237 writer.WriteBits(1, 1);
238 // Now we write the left/right/top/bottom crop. For simplicity, we'll put all
239 // the crop at the left/top.
240 // We picked a 4:2:0 format, so the crops are 1/2 the pixel crop values.
241 // Left/right.
242 writer.WriteExponentialGolomb(((16 - (kWidth % 16)) % 16) / 2);
243 writer.WriteExponentialGolomb(0);
244 // Top/bottom.
245 writer.WriteExponentialGolomb(((16 - (kHeight % 16)) % 16) / 2);
246 writer.WriteExponentialGolomb(0);
247
248 // Finally! The VUI.
249 // vui_parameters_present_flag: u(1)
250 writer.WriteBits(vui.vui_parameters_present_flag, 1);
251 if (vui.vui_parameters_present_flag) {
252 // aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1).
253 writer.WriteBits(0, 2);
254
255 writer.WriteBits(vui.video_signal_type_present_flag, 1);
256 if (vui.video_signal_type_present_flag) {
257 // video_format: u(3). 5 = Unspecified
258 writer.WriteBits(5, 3);
259 writer.WriteBits(vui.video_full_range_flag, 1);
260 writer.WriteBits(vui.colour_description_present_flag, 1);
261 if (vui.colour_description_present_flag) {
262 writer.WriteUInt8(vui.colour_primaries);
263 writer.WriteUInt8(vui.transfer_characteristics);
264 writer.WriteUInt8(vui.matrix_coefficients);
265 }
266 }
267
268 // chroma_loc_info_present_flag, timing_info_present_flag,
269 // nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag,
270 // pic_struct_present_flag, All u(1)
271 writer.WriteBits(0, 5);
272
273 writer.WriteBits(vui.bitstream_restriction_flag, 1);
274 if (vui.bitstream_restriction_flag) {
275 // Write some defaults. Shouldn't matter for parsing, though.
276 // motion_vectors_over_pic_boundaries_flag: u(1)
277 writer.WriteBits(1, 1);
278 // max_bytes_per_pic_denom: ue(v)
279 writer.WriteExponentialGolomb(2);
280 // max_bits_per_mb_denom: ue(v)
281 writer.WriteExponentialGolomb(1);
282 // log2_max_mv_length_horizontal: ue(v)
283 // log2_max_mv_length_vertical: ue(v)
284 writer.WriteExponentialGolomb(16);
285 writer.WriteExponentialGolomb(16);
286
287 // Next are the limits we care about.
288 writer.WriteExponentialGolomb(vui.max_num_reorder_frames);
289 writer.WriteExponentialGolomb(vui.max_dec_frame_buffering);
290 }
291 }
292
293 // Get the number of bytes written (including the last partial byte).
294 size_t byte_count, bit_offset;
295 writer.GetCurrentOffset(&byte_count, &bit_offset);
296 if (bit_offset > 0) {
297 byte_count++;
298 }
299
300 H264::WriteRbsp(rbsp, byte_count, out_buffer);
301 }
302
TestSps(const VuiHeader & vui,const ColorSpace * color_space,SpsVuiRewriter::ParseResult expected_parse_result)303 void TestSps(const VuiHeader& vui,
304 const ColorSpace* color_space,
305 SpsVuiRewriter::ParseResult expected_parse_result) {
306 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
307 rtc::Buffer original_sps;
308 GenerateFakeSps(vui, &original_sps);
309
310 absl::optional<SpsParser::SpsState> sps;
311 rtc::Buffer rewritten_sps;
312 SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps(
313 original_sps.data(), original_sps.size(), &sps, color_space,
314 &rewritten_sps, SpsVuiRewriter::Direction::kIncoming);
315 EXPECT_EQ(expected_parse_result, result);
316 ASSERT_TRUE(sps);
317 EXPECT_EQ(sps->width, kWidth);
318 EXPECT_EQ(sps->height, kHeight);
319 if (vui.vui_parameters_present_flag) {
320 EXPECT_EQ(sps->vui_params_present, 1u);
321 }
322
323 if (result == SpsVuiRewriter::ParseResult::kVuiRewritten) {
324 // Ensure that added/rewritten SPS is parsable.
325 rtc::Buffer tmp;
326 result = SpsVuiRewriter::ParseAndRewriteSps(
327 rewritten_sps.data(), rewritten_sps.size(), &sps, nullptr, &tmp,
328 SpsVuiRewriter::Direction::kIncoming);
329 EXPECT_EQ(SpsVuiRewriter::ParseResult::kVuiOk, result);
330 ASSERT_TRUE(sps);
331 EXPECT_EQ(sps->width, kWidth);
332 EXPECT_EQ(sps->height, kHeight);
333 EXPECT_EQ(sps->vui_params_present, 1u);
334 }
335 }
336
337 class SpsVuiRewriterTest : public ::testing::Test,
338 public ::testing::WithParamInterface<
339 ::testing::tuple<VuiHeader,
340 const ColorSpace*,
341 SpsVuiRewriter::ParseResult>> {
342 };
343
TEST_P(SpsVuiRewriterTest,RewriteVui)344 TEST_P(SpsVuiRewriterTest, RewriteVui) {
345 VuiHeader vui = ::testing::get<0>(GetParam());
346 const ColorSpace* color_space = ::testing::get<1>(GetParam());
347 SpsVuiRewriter::ParseResult expected_parse_result =
348 ::testing::get<2>(GetParam());
349 TestSps(vui, color_space, expected_parse_result);
350 }
351
352 INSTANTIATE_TEST_SUITE_P(
353 All,
354 SpsVuiRewriterTest,
355 ::testing::Values(
356 std::make_tuple(kVuiNoFrameBuffering,
357 nullptr,
358 SpsVuiRewriter::ParseResult::kVuiOk),
359 std::make_tuple(kVuiNoVideoSignalType,
360 &kColorSpaceH264Default,
361 SpsVuiRewriter::ParseResult::kVuiOk),
362 std::make_tuple(kVuiLimitedRangeBt709Color,
363 &kColorSpaceBt709LimitedRange,
364 SpsVuiRewriter::ParseResult::kVuiOk),
365 std::make_tuple(kVuiNotPresent,
366 nullptr,
367 SpsVuiRewriter::ParseResult::kVuiRewritten),
368 std::make_tuple(kVuiNoBitstreamRestriction,
369 nullptr,
370 SpsVuiRewriter::ParseResult::kVuiRewritten),
371 std::make_tuple(kVuiFrameBuffering,
372 nullptr,
373 SpsVuiRewriter::ParseResult::kVuiRewritten),
374 std::make_tuple(kVuiLimitedRangeNoColourDescription,
375 &kColorSpaceFullRange,
376 SpsVuiRewriter::ParseResult::kVuiRewritten),
377 std::make_tuple(kVuiNoVideoSignalType,
378 &kColorSpacePrimariesBt709,
379 SpsVuiRewriter::ParseResult::kVuiRewritten),
380 std::make_tuple(kVuiNoVideoSignalType,
381 &kColorSpaceTransferBt709,
382 SpsVuiRewriter::ParseResult::kVuiRewritten),
383 std::make_tuple(kVuiNoVideoSignalType,
384 &kColorSpaceMatrixBt709,
385 SpsVuiRewriter::ParseResult::kVuiRewritten),
386 std::make_tuple(kVuiFullRangeNoColourDescription,
387 &kColorSpaceH264Default,
388 SpsVuiRewriter::ParseResult::kVuiRewritten),
389 std::make_tuple(kVuiLimitedRangeBt709Color,
390 &kColorSpaceH264Default,
391 SpsVuiRewriter::ParseResult::kVuiRewritten)));
392
TEST(SpsVuiRewriterOutgoingVuiTest,ParseOutgoingBitstreamOptimalVui)393 TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamOptimalVui) {
394 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
395
396 rtc::Buffer optimal_sps;
397 GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
398
399 rtc::Buffer buffer;
400 buffer.AppendData(kStartSequence);
401 buffer.AppendData(optimal_sps);
402 buffer.AppendData(kStartSequence);
403 buffer.AppendData(kIdr1);
404
405 EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr),
406 ::testing::ElementsAreArray(buffer));
407 }
408
TEST(SpsVuiRewriterOutgoingVuiTest,ParseOutgoingBitstreamNoVui)409 TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) {
410 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
411
412 rtc::Buffer sps;
413 GenerateFakeSps(kVuiNotPresent, &sps);
414
415 rtc::Buffer buffer;
416 buffer.AppendData(kStartSequence);
417 buffer.AppendData(kIdr1);
418 buffer.AppendData(kStartSequence);
419 buffer.AppendData(kSpsNaluType);
420 buffer.AppendData(sps);
421 buffer.AppendData(kStartSequence);
422 buffer.AppendData(kIdr2);
423
424 rtc::Buffer optimal_sps;
425 GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
426
427 rtc::Buffer expected_buffer;
428 expected_buffer.AppendData(kStartSequence);
429 expected_buffer.AppendData(kIdr1);
430 expected_buffer.AppendData(kStartSequence);
431 expected_buffer.AppendData(kSpsNaluType);
432 expected_buffer.AppendData(optimal_sps);
433 expected_buffer.AppendData(kStartSequence);
434 expected_buffer.AppendData(kIdr2);
435
436 EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr),
437 ::testing::ElementsAreArray(expected_buffer));
438 }
439
TEST(SpsVuiRewriterOutgoingAudTest,ParseOutgoingBitstreamWithAud)440 TEST(SpsVuiRewriterOutgoingAudTest, ParseOutgoingBitstreamWithAud) {
441 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
442
443 rtc::Buffer optimal_sps;
444 GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
445
446 rtc::Buffer buffer;
447 buffer.AppendData(kStartSequence);
448 buffer.AppendData(kAud);
449 buffer.AppendData(kStartSequence);
450 buffer.AppendData(optimal_sps);
451 buffer.AppendData(kStartSequence);
452 buffer.AppendData(kIdr1);
453
454 rtc::Buffer expected_buffer;
455 expected_buffer.AppendData(kStartSequence);
456 expected_buffer.AppendData(optimal_sps);
457 expected_buffer.AppendData(kStartSequence);
458 expected_buffer.AppendData(kIdr1);
459
460 EXPECT_THAT(SpsVuiRewriter::ParseOutgoingBitstreamAndRewrite(buffer, nullptr),
461 ::testing::ElementsAreArray(expected_buffer));
462 }
463 } // namespace webrtc
464