1 /*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <memory>
12 #include <string>
13 #include <tuple>
14 #include <vector>
15
16 #include "api/test/create_videocodec_test_fixture.h"
17 #include "media/base/media_constants.h"
18 #include "modules/video_coding/codecs/test/android_codec_factory_helper.h"
19 #include "modules/video_coding/codecs/test/videocodec_test_fixture_impl.h"
20 #include "rtc_base/strings/string_builder.h"
21 #include "test/gtest.h"
22 #include "test/testsupport/file_utils.h"
23
24 namespace webrtc {
25 namespace test {
26
27 namespace {
28 const int kForemanNumFrames = 300;
29 const int kForemanFramerateFps = 30;
30
31 struct RateProfileData {
32 std::string name;
33 std::vector<webrtc::test::RateProfile> rate_profile;
34 };
35
36 const size_t kConstRateIntervalSec = 10;
37
38 const RateProfileData kBitRateHighLowHigh = {
39 /*name=*/"BitRateHighLowHigh",
40 /*rate_profile=*/{
41 {/*target_kbps=*/3000, /*input_fps=*/30, /*frame_num=*/0},
42 {/*target_kbps=*/1500, /*input_fps=*/30, /*frame_num=*/300},
43 {/*target_kbps=*/750, /*input_fps=*/30, /*frame_num=*/600},
44 {/*target_kbps=*/1500, /*input_fps=*/30, /*frame_num=*/900},
45 {/*target_kbps=*/3000, /*input_fps=*/30, /*frame_num=*/1200}}};
46
47 const RateProfileData kBitRateLowHighLow = {
48 /*name=*/"BitRateLowHighLow",
49 /*rate_profile=*/{
50 {/*target_kbps=*/750, /*input_fps=*/30, /*frame_num=*/0},
51 {/*target_kbps=*/1500, /*input_fps=*/30, /*frame_num=*/300},
52 {/*target_kbps=*/3000, /*input_fps=*/30, /*frame_num=*/600},
53 {/*target_kbps=*/1500, /*input_fps=*/30, /*frame_num=*/900},
54 {/*target_kbps=*/750, /*input_fps=*/30, /*frame_num=*/1200}}};
55
56 const RateProfileData kFrameRateHighLowHigh = {
57 /*name=*/"FrameRateHighLowHigh",
58 /*rate_profile=*/{
59 {/*target_kbps=*/2000, /*input_fps=*/30, /*frame_num=*/0},
60 {/*target_kbps=*/2000, /*input_fps=*/15, /*frame_num=*/300},
61 {/*target_kbps=*/2000, /*input_fps=*/7.5, /*frame_num=*/450},
62 {/*target_kbps=*/2000, /*input_fps=*/15, /*frame_num=*/525},
63 {/*target_kbps=*/2000, /*input_fps=*/30, /*frame_num=*/675}}};
64
65 const RateProfileData kFrameRateLowHighLow = {
66 /*name=*/"FrameRateLowHighLow",
67 /*rate_profile=*/{
68 {/*target_kbps=*/2000, /*input_fps=*/7.5, /*frame_num=*/0},
69 {/*target_kbps=*/2000, /*input_fps=*/15, /*frame_num=*/75},
70 {/*target_kbps=*/2000, /*input_fps=*/30, /*frame_num=*/225},
71 {/*target_kbps=*/2000, /*input_fps=*/15, /*frame_num=*/525},
72 {/*target_kbps=*/2000, /*input_fps=*/7.5, /*frame_num=*/775}}};
73
CreateConfig()74 VideoCodecTestFixture::Config CreateConfig() {
75 VideoCodecTestFixture::Config config;
76 config.filename = "foreman_cif";
77 config.filepath = ResourcePath(config.filename, "yuv");
78 config.num_frames = kForemanNumFrames;
79 // In order to not overwhelm the OpenMAX buffers in the Android MediaCodec.
80 config.encode_in_real_time = true;
81 return config;
82 }
83
CreateTestFixtureWithConfig(VideoCodecTestFixture::Config config)84 std::unique_ptr<VideoCodecTestFixture> CreateTestFixtureWithConfig(
85 VideoCodecTestFixture::Config config) {
86 InitializeAndroidObjects(); // Idempotent.
87 auto encoder_factory = CreateAndroidEncoderFactory();
88 auto decoder_factory = CreateAndroidDecoderFactory();
89 return CreateVideoCodecTestFixture(config, std::move(decoder_factory),
90 std::move(encoder_factory));
91 }
92 } // namespace
93
TEST(VideoCodecTestMediaCodec,ForemanCif500kbpsVp8)94 TEST(VideoCodecTestMediaCodec, ForemanCif500kbpsVp8) {
95 auto config = CreateConfig();
96 config.SetCodecSettings(cricket::kVp8CodecName, 1, 1, 1, false, false, false,
97 352, 288);
98 auto fixture = CreateTestFixtureWithConfig(config);
99
100 std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
101
102 // The thresholds below may have to be tweaked to let even poor MediaCodec
103 // implementations pass. If this test fails on the bots, disable it and
104 // ping brandtr@.
105 std::vector<RateControlThresholds> rc_thresholds = {
106 {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
107
108 std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
109
110 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
111 }
112
TEST(VideoCodecTestMediaCodec,ForemanCif500kbpsH264CBP)113 TEST(VideoCodecTestMediaCodec, ForemanCif500kbpsH264CBP) {
114 auto config = CreateConfig();
115 const auto frame_checker =
116 std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
117 config.encoded_frame_checker = frame_checker.get();
118 config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
119 352, 288);
120 auto fixture = CreateTestFixtureWithConfig(config);
121
122 std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
123
124 // The thresholds below may have to be tweaked to let even poor MediaCodec
125 // implementations pass. If this test fails on the bots, disable it and
126 // ping brandtr@.
127 std::vector<RateControlThresholds> rc_thresholds = {
128 {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
129
130 std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
131
132 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
133 }
134
135 // TODO(brandtr): Enable this test when we have trybots/buildbots with
136 // HW encoders that support CHP.
TEST(VideoCodecTestMediaCodec,DISABLED_ForemanCif500kbpsH264CHP)137 TEST(VideoCodecTestMediaCodec, DISABLED_ForemanCif500kbpsH264CHP) {
138 auto config = CreateConfig();
139 const auto frame_checker =
140 std::make_unique<VideoCodecTestFixtureImpl::H264KeyframeChecker>();
141
142 config.h264_codec_settings.profile = H264Profile::kProfileConstrainedHigh;
143 config.encoded_frame_checker = frame_checker.get();
144 config.SetCodecSettings(cricket::kH264CodecName, 1, 1, 1, false, false, false,
145 352, 288);
146 auto fixture = CreateTestFixtureWithConfig(config);
147
148 std::vector<RateProfile> rate_profiles = {{500, kForemanFramerateFps, 0}};
149
150 // The thresholds below may have to be tweaked to let even poor MediaCodec
151 // implementations pass. If this test fails on the bots, disable it and
152 // ping brandtr@.
153 std::vector<RateControlThresholds> rc_thresholds = {
154 {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
155
156 std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
157
158 fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
159 }
160
TEST(VideoCodecTestMediaCodec,ForemanMixedRes100kbpsVp8H264)161 TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) {
162 auto config = CreateConfig();
163 const int kNumFrames = 30;
164 const std::vector<std::string> codecs = {cricket::kVp8CodecName,
165 cricket::kH264CodecName};
166 const std::vector<std::tuple<int, int>> resolutions = {
167 {128, 96}, {176, 144}, {320, 240}, {480, 272}};
168 const std::vector<RateProfile> rate_profiles = {
169 {100, kForemanFramerateFps, 0}};
170 const std::vector<QualityThresholds> quality_thresholds = {
171 {29, 26, 0.8, 0.75}};
172
173 for (const auto& codec : codecs) {
174 for (const auto& resolution : resolutions) {
175 const int width = std::get<0>(resolution);
176 const int height = std::get<1>(resolution);
177 config.filename = std::string("foreman_") + std::to_string(width) + "x" +
178 std::to_string(height);
179 config.filepath = ResourcePath(config.filename, "yuv");
180 config.num_frames = kNumFrames;
181 config.SetCodecSettings(codec, 1, 1, 1, false, false, false, width,
182 height);
183
184 auto fixture = CreateTestFixtureWithConfig(config);
185 fixture->RunTest(rate_profiles, nullptr /* rc_thresholds */,
186 &quality_thresholds, nullptr /* bs_thresholds */);
187 }
188 }
189 }
190
191 class VideoCodecTestMediaCodecRateAdaptation
192 : public ::testing::TestWithParam<
193 std::tuple<RateProfileData, std::string>> {
194 public:
ParamInfoToStr(const::testing::TestParamInfo<VideoCodecTestMediaCodecRateAdaptation::ParamType> & info)195 static std::string ParamInfoToStr(
196 const ::testing::TestParamInfo<
197 VideoCodecTestMediaCodecRateAdaptation::ParamType>& info) {
198 char buf[512];
199 rtc::SimpleStringBuilder ss(buf);
200 ss << std::get<0>(info.param).name << "_" << std::get<1>(info.param);
201 return ss.str();
202 }
203 };
204
TEST_P(VideoCodecTestMediaCodecRateAdaptation,DISABLED_RateAdaptation)205 TEST_P(VideoCodecTestMediaCodecRateAdaptation, DISABLED_RateAdaptation) {
206 const std::vector<webrtc::test::RateProfile> rate_profile =
207 std::get<0>(GetParam()).rate_profile;
208 const std::string codec_name = std::get<1>(GetParam());
209
210 VideoCodecTestFixture::Config config;
211 config.filename = "FourPeople_1280x720_30";
212 config.filepath = ResourcePath(config.filename, "yuv");
213 config.num_frames = rate_profile.back().frame_num +
214 static_cast<size_t>(kConstRateIntervalSec *
215 rate_profile.back().input_fps);
216 config.encode_in_real_time = true;
217 config.SetCodecSettings(codec_name, 1, 1, 1, false, false, false, 1280, 720);
218
219 auto fixture = CreateTestFixtureWithConfig(config);
220 fixture->RunTest(rate_profile, nullptr, nullptr, nullptr);
221
222 for (size_t i = 0; i < rate_profile.size(); ++i) {
223 const size_t num_frames =
224 static_cast<size_t>(rate_profile[i].input_fps * kConstRateIntervalSec);
225
226 auto stats = fixture->GetStats().SliceAndCalcLayerVideoStatistic(
227 rate_profile[i].frame_num, rate_profile[i].frame_num + num_frames - 1);
228 ASSERT_EQ(stats.size(), 1u);
229
230 // Bitrate mismatch is <= 10%.
231 EXPECT_LE(stats[0].avg_bitrate_mismatch_pct, 10);
232 EXPECT_GE(stats[0].avg_bitrate_mismatch_pct, -10);
233
234 // Avg frame transmission delay and processing latency is <=100..250ms
235 // depending on frame rate.
236 const double expected_delay_sec =
237 std::min(std::max(1 / rate_profile[i].input_fps, 0.1), 0.25);
238 EXPECT_LE(stats[0].avg_delay_sec, expected_delay_sec);
239 EXPECT_LE(stats[0].avg_encode_latency_sec, expected_delay_sec);
240 EXPECT_LE(stats[0].avg_decode_latency_sec, expected_delay_sec);
241
242 // Frame drops are not expected.
243 EXPECT_EQ(stats[0].num_encoded_frames, num_frames);
244 EXPECT_EQ(stats[0].num_decoded_frames, num_frames);
245
246 // Periodic keyframes are not expected.
247 EXPECT_EQ(stats[0].num_key_frames, i == 0 ? 1u : 0);
248
249 // Ensure codec delivers a reasonable spatial quality.
250 EXPECT_GE(stats[0].avg_psnr_y, 35);
251 }
252 }
253
254 INSTANTIATE_TEST_SUITE_P(
255 RateAdaptation,
256 VideoCodecTestMediaCodecRateAdaptation,
257 ::testing::Combine(::testing::Values(kBitRateLowHighLow,
258 kBitRateHighLowHigh,
259 kFrameRateLowHighLow,
260 kFrameRateHighLowHigh),
261 ::testing::Values(cricket::kVp8CodecName,
262 cricket::kVp9CodecName,
263 cricket::kH264CodecName)),
264 VideoCodecTestMediaCodecRateAdaptation::ParamInfoToStr);
265
266 } // namespace test
267 } // namespace webrtc
268