1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include <memory>
11 #include <string>
12 #include <utility>
13 #include <vector>
14
15 #include "api/media_stream_interface.h"
16 #include "api/test/create_network_emulation_manager.h"
17 #include "api/test/create_peer_connection_quality_test_frame_generator.h"
18 #include "api/test/create_peerconnection_quality_test_fixture.h"
19 #include "api/test/frame_generator_interface.h"
20 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
21 #include "api/test/network_emulation_manager.h"
22 #include "api/test/pclf/media_configuration.h"
23 #include "api/test/pclf/media_quality_test_params.h"
24 #include "api/test/pclf/peer_configurer.h"
25 #include "api/test/peerconnection_quality_test_fixture.h"
26 #include "api/test/simulated_network.h"
27 #include "api/test/time_controller.h"
28 #include "api/video_codecs/vp9_profile.h"
29 #include "call/simulated_network.h"
30 #include "modules/video_coding/codecs/vp9/include/vp9.h"
31 #include "modules/video_coding/svc/scalability_mode_util.h"
32 #include "rtc_base/containers/flat_map.h"
33 #include "system_wrappers/include/field_trial.h"
34 #include "test/field_trial.h"
35 #include "test/gmock.h"
36 #include "test/gtest.h"
37 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
38 #include "test/pc/e2e/network_quality_metrics_reporter.h"
39 #include "test/testsupport/file_utils.h"
40
41 namespace webrtc {
42 namespace {
43
44 using ::cricket::kAv1CodecName;
45 using ::cricket::kH264CodecName;
46 using ::cricket::kVp8CodecName;
47 using ::cricket::kVp9CodecName;
48 using ::testing::Combine;
49 using ::testing::UnitTest;
50 using ::testing::Values;
51 using ::testing::ValuesIn;
52 using ::webrtc::webrtc_pc_e2e::EmulatedSFUConfig;
53 using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
54 using ::webrtc::webrtc_pc_e2e::RunParams;
55 using ::webrtc::webrtc_pc_e2e::ScreenShareConfig;
56 using ::webrtc::webrtc_pc_e2e::VideoCodecConfig;
57 using ::webrtc::webrtc_pc_e2e::VideoConfig;
58
59 std::unique_ptr<webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture>
CreateTestFixture(absl::string_view test_case_name,TimeController & time_controller,std::pair<EmulatedNetworkManagerInterface *,EmulatedNetworkManagerInterface * > network_links,rtc::FunctionView<void (PeerConfigurer *)> alice_configurer,rtc::FunctionView<void (PeerConfigurer *)> bob_configurer,std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer=nullptr)60 CreateTestFixture(absl::string_view test_case_name,
61 TimeController& time_controller,
62 std::pair<EmulatedNetworkManagerInterface*,
63 EmulatedNetworkManagerInterface*> network_links,
64 rtc::FunctionView<void(PeerConfigurer*)> alice_configurer,
65 rtc::FunctionView<void(PeerConfigurer*)> bob_configurer,
66 std::unique_ptr<VideoQualityAnalyzerInterface>
67 video_quality_analyzer = nullptr) {
68 auto fixture = webrtc_pc_e2e::CreatePeerConnectionE2EQualityTestFixture(
69 std::string(test_case_name), time_controller, nullptr,
70 std::move(video_quality_analyzer));
71 auto alice = std::make_unique<PeerConfigurer>(
72 network_links.first->network_dependencies());
73 auto bob = std::make_unique<PeerConfigurer>(
74 network_links.second->network_dependencies());
75 alice_configurer(alice.get());
76 bob_configurer(bob.get());
77 fixture->AddPeer(std::move(alice));
78 fixture->AddPeer(std::move(bob));
79 return fixture;
80 }
81
82 // Takes the current active field trials set, and appends some new trials.
AppendFieldTrials(std::string new_trial_string)83 std::string AppendFieldTrials(std::string new_trial_string) {
84 return std::string(field_trial::GetFieldTrialString()) + new_trial_string;
85 }
86
87 enum class UseDependencyDescriptor {
88 Enabled,
89 Disabled,
90 };
91
92 struct SvcTestParameters {
Createwebrtc::__anonb904f79b0111::SvcTestParameters93 static SvcTestParameters Create(const std::string& codec_name,
94 const std::string& scalability_mode_str) {
95 absl::optional<ScalabilityMode> scalability_mode =
96 ScalabilityModeFromString(scalability_mode_str);
97 RTC_CHECK(scalability_mode.has_value())
98 << "Unsupported scalability mode: " << scalability_mode_str;
99
100 int num_spatial_layers =
101 ScalabilityModeToNumSpatialLayers(*scalability_mode);
102 int num_temporal_layers =
103 ScalabilityModeToNumTemporalLayers(*scalability_mode);
104
105 return SvcTestParameters{codec_name, scalability_mode_str,
106 num_spatial_layers, num_temporal_layers};
107 }
108
109 std::string codec_name;
110 std::string scalability_mode;
111 int expected_spatial_layers;
112 int expected_temporal_layers;
113 };
114
115 class SvcTest : public testing::TestWithParam<
116 std::tuple<SvcTestParameters, UseDependencyDescriptor>> {
117 public:
SvcTest()118 SvcTest()
119 : video_codec_config(ToVideoCodecConfig(SvcTestParameters().codec_name)) {
120 }
121
ToVideoCodecConfig(absl::string_view codec)122 static VideoCodecConfig ToVideoCodecConfig(absl::string_view codec) {
123 if (codec == cricket::kVp9CodecName) {
124 return VideoCodecConfig(
125 cricket::kVp9CodecName,
126 {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}});
127 }
128
129 return VideoCodecConfig(std::string(codec));
130 }
131
SvcTestParameters() const132 const SvcTestParameters& SvcTestParameters() const {
133 return std::get<0>(GetParam());
134 }
135
UseDependencyDescriptor() const136 bool UseDependencyDescriptor() const {
137 return std::get<1>(GetParam()) == UseDependencyDescriptor::Enabled;
138 }
139
IsSMode() const140 bool IsSMode() const {
141 return SvcTestParameters().scalability_mode[0] == 'S';
142 }
143
144 protected:
145 VideoCodecConfig video_codec_config;
146 };
147
SvcTestNameGenerator(const testing::TestParamInfo<SvcTest::ParamType> & info)148 std::string SvcTestNameGenerator(
149 const testing::TestParamInfo<SvcTest::ParamType>& info) {
150 return std::get<0>(info.param).scalability_mode +
151 (std::get<1>(info.param) == UseDependencyDescriptor::Enabled ? "_DD"
152 : "");
153 }
154
155 } // namespace
156
157 // Records how many frames are seen for each spatial and temporal index at the
158 // encoder and decoder level.
159 class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer {
160 public:
161 using SpatialTemporalLayerCounts =
162 webrtc::flat_map<int, webrtc::flat_map<int, int>>;
163
SvcVideoQualityAnalyzer(webrtc::Clock * clock)164 explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock)
165 : DefaultVideoQualityAnalyzer(clock,
166 test::GetGlobalMetricsLogger(),
167 DefaultVideoQualityAnalyzerOptions{
168 .compute_psnr = false,
169 .compute_ssim = false,
170 }) {}
171 ~SvcVideoQualityAnalyzer() override = default;
172
OnFrameEncoded(absl::string_view peer_name,uint16_t frame_id,const EncodedImage & encoded_image,const EncoderStats & stats,bool discarded)173 void OnFrameEncoded(absl::string_view peer_name,
174 uint16_t frame_id,
175 const EncodedImage& encoded_image,
176 const EncoderStats& stats,
177 bool discarded) override {
178 absl::optional<int> spatial_id = encoded_image.SpatialIndex();
179 absl::optional<int> temporal_id = encoded_image.TemporalIndex();
180 encoder_layers_seen_[spatial_id.value_or(0)][temporal_id.value_or(0)]++;
181 DefaultVideoQualityAnalyzer::OnFrameEncoded(
182 peer_name, frame_id, encoded_image, stats, discarded);
183 }
184
OnFramePreDecode(absl::string_view peer_name,uint16_t frame_id,const EncodedImage & input_image)185 void OnFramePreDecode(absl::string_view peer_name,
186 uint16_t frame_id,
187 const EncodedImage& input_image) override {
188 absl::optional<int> spatial_id = input_image.SpatialIndex();
189 absl::optional<int> temporal_id = input_image.TemporalIndex();
190 if (!spatial_id) {
191 decoder_layers_seen_[0][temporal_id.value_or(0)]++;
192 } else {
193 for (int i = 0; i <= *spatial_id; ++i) {
194 // If there are no spatial layers (for example VP8), we still want to
195 // record the temporal index for pseudo-layer "0" frames.
196 if (*spatial_id == 0 ||
197 input_image.SpatialLayerFrameSize(i).value_or(0) > 0) {
198 decoder_layers_seen_[i][temporal_id.value_or(0)]++;
199 }
200 }
201 }
202 DefaultVideoQualityAnalyzer::OnFramePreDecode(peer_name, frame_id,
203 input_image);
204 }
205
encoder_layers_seen() const206 const SpatialTemporalLayerCounts& encoder_layers_seen() const {
207 return encoder_layers_seen_;
208 }
decoder_layers_seen() const209 const SpatialTemporalLayerCounts& decoder_layers_seen() const {
210 return decoder_layers_seen_;
211 }
212
213 private:
214 SpatialTemporalLayerCounts encoder_layers_seen_;
215 SpatialTemporalLayerCounts decoder_layers_seen_;
216 };
217
218 MATCHER_P2(HasSpatialAndTemporalLayers,
219 expected_spatial_layers,
220 expected_temporal_layers,
221 "") {
222 if (arg.size() != static_cast<size_t>(expected_spatial_layers)) {
223 *result_listener << "spatial layer count mismatch expected "
224 << expected_spatial_layers << " but got " << arg.size();
225 return false;
226 }
227 for (const auto& [spatial_layer_index, temporal_layers] : arg) {
228 if (spatial_layer_index < 0 ||
229 spatial_layer_index >= expected_spatial_layers) {
230 *result_listener << "spatial layer index is not in range [0,"
231 << expected_spatial_layers << "[.";
232 return false;
233 }
234
235 if (temporal_layers.size() !=
236 static_cast<size_t>(expected_temporal_layers)) {
237 *result_listener << "temporal layer count mismatch on spatial layer "
238 << spatial_layer_index << ", expected "
239 << expected_temporal_layers << " but got "
240 << temporal_layers.size();
241 return false;
242 }
243 for (const auto& [temporal_layer_index, temporal_layer_frame_count] :
244 temporal_layers) {
245 if (temporal_layer_index < 0 ||
246 temporal_layer_index >= expected_temporal_layers) {
247 *result_listener << "temporal layer index on spatial layer "
248 << spatial_layer_index << " is not in range [0,"
249 << expected_temporal_layers << "[.";
250 return false;
251 }
252 }
253 }
254 return true;
255 }
256
257 MATCHER_P2(HasSpatialAndTemporalLayersSMode,
258 expected_spatial_layers,
259 expected_temporal_layers,
260 "") {
261 if (arg.size() != 1) {
262 *result_listener << "spatial layer count mismatch expected 1 but got "
263 << arg.size();
264 return false;
265 }
266 for (const auto& [spatial_layer_index, temporal_layers] : arg) {
267 if (spatial_layer_index != expected_spatial_layers - 1) {
268 *result_listener << "spatial layer index is not equal to "
269 << expected_spatial_layers - 1 << ".";
270 return false;
271 }
272
273 if (temporal_layers.size() !=
274 static_cast<size_t>(expected_temporal_layers)) {
275 *result_listener << "temporal layer count mismatch on spatial layer "
276 << spatial_layer_index << ", expected "
277 << expected_temporal_layers << " but got "
278 << temporal_layers.size();
279 return false;
280 }
281 for (const auto& [temporal_layer_index, temporal_layer_frame_count] :
282 temporal_layers) {
283 if (temporal_layer_index < 0 ||
284 temporal_layer_index >= expected_temporal_layers) {
285 *result_listener << "temporal layer index on spatial layer "
286 << spatial_layer_index << " is not in range [0,"
287 << expected_temporal_layers << "[.";
288 return false;
289 }
290 }
291 }
292 return true;
293 }
294
TEST_P(SvcTest,ScalabilityModeSupported)295 TEST_P(SvcTest, ScalabilityModeSupported) {
296 std::string trials;
297 if (UseDependencyDescriptor()) {
298 trials += "WebRTC-DependencyDescriptorAdvertised/Enabled/";
299 }
300 webrtc::test::ScopedFieldTrials override_trials(AppendFieldTrials(trials));
301 std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
302 CreateNetworkEmulationManager(webrtc::TimeMode::kSimulated);
303 auto analyzer = std::make_unique<SvcVideoQualityAnalyzer>(
304 network_emulation_manager->time_controller()->GetClock());
305 SvcVideoQualityAnalyzer* analyzer_ptr = analyzer.get();
306 auto fixture = CreateTestFixture(
307 UnitTest::GetInstance()->current_test_info()->name(),
308 *network_emulation_manager->time_controller(),
309 network_emulation_manager->CreateEndpointPairWithTwoWayRoutes(
310 BuiltInNetworkBehaviorConfig()),
311 [this](PeerConfigurer* alice) {
312 VideoConfig video(/*stream_label=*/"alice-video", /*width=*/1850,
313 /*height=*/1110, /*fps=*/30);
314 if (IsSMode()) {
315 video.emulated_sfu_config = EmulatedSFUConfig(
316 SvcTestParameters().expected_spatial_layers - 1,
317 SvcTestParameters().expected_temporal_layers - 1);
318 }
319 RtpEncodingParameters parameters;
320 parameters.scalability_mode = SvcTestParameters().scalability_mode;
321 video.encoding_params.push_back(parameters);
322 alice->AddVideoConfig(
323 std::move(video),
324 CreateScreenShareFrameGenerator(
325 video, ScreenShareConfig(TimeDelta::Seconds(5))));
326 alice->SetVideoCodecs({video_codec_config});
327 },
328 [](PeerConfigurer* bob) {}, std::move(analyzer));
329 fixture->Run(RunParams(TimeDelta::Seconds(5)));
330 EXPECT_THAT(analyzer_ptr->encoder_layers_seen(),
331 HasSpatialAndTemporalLayers(
332 SvcTestParameters().expected_spatial_layers,
333 SvcTestParameters().expected_temporal_layers));
334 if (IsSMode()) {
335 EXPECT_THAT(analyzer_ptr->decoder_layers_seen(),
336 HasSpatialAndTemporalLayersSMode(
337 SvcTestParameters().expected_spatial_layers,
338 SvcTestParameters().expected_temporal_layers));
339 } else {
340 EXPECT_THAT(analyzer_ptr->decoder_layers_seen(),
341 HasSpatialAndTemporalLayers(
342 SvcTestParameters().expected_spatial_layers,
343 SvcTestParameters().expected_temporal_layers));
344 }
345
346 RTC_LOG(LS_INFO) << "Encoder layers seen: "
347 << analyzer_ptr->encoder_layers_seen().size();
348 for (auto& [spatial_index, temporal_layers] :
349 analyzer_ptr->encoder_layers_seen()) {
350 for (auto& [temporal_index, frame_count] : temporal_layers) {
351 RTC_LOG(LS_INFO) << " Layer: " << spatial_index << "," << temporal_index
352 << " frames: " << frame_count;
353 }
354 }
355 RTC_LOG(LS_INFO) << "Decoder layers seen: "
356 << analyzer_ptr->decoder_layers_seen().size();
357 for (auto& [spatial_index, temporal_layers] :
358 analyzer_ptr->decoder_layers_seen()) {
359 for (auto& [temporal_index, frame_count] : temporal_layers) {
360 RTC_LOG(LS_INFO) << " Layer: " << spatial_index << "," << temporal_index
361 << " frames: " << frame_count;
362 }
363 }
364 }
365
366 INSTANTIATE_TEST_SUITE_P(
367 SvcTestVP8,
368 SvcTest,
369 Combine(Values(SvcTestParameters::Create(kVp8CodecName, "L1T1"),
370 SvcTestParameters::Create(kVp8CodecName, "L1T2"),
371 SvcTestParameters::Create(kVp8CodecName, "L1T3")),
372 Values(UseDependencyDescriptor::Disabled,
373 UseDependencyDescriptor::Enabled)),
374 SvcTestNameGenerator);
375
376 #if defined(WEBRTC_USE_H264)
377 INSTANTIATE_TEST_SUITE_P(
378 SvcTestH264,
379 SvcTest,
380 Combine(ValuesIn({
381 SvcTestParameters::Create(kH264CodecName, "L1T1"),
382 SvcTestParameters::Create(kH264CodecName, "L1T2"),
383 SvcTestParameters::Create(kH264CodecName, "L1T3"),
384 }),
385 // Like AV1, H.264 RTP format does not include SVC related
386 // information, so always use Dependency Descriptor.
387 Values(UseDependencyDescriptor::Enabled)),
388 SvcTestNameGenerator);
389 #endif
390
391 #if defined(RTC_ENABLE_VP9)
392 INSTANTIATE_TEST_SUITE_P(
393 SvcTestVP9,
394 SvcTest,
395 Combine(
396 // TODO(bugs.webrtc.org/13960): Fix and enable remaining VP9 modes
397 ValuesIn({
398 SvcTestParameters::Create(kVp9CodecName, "L1T1"),
399 SvcTestParameters::Create(kVp9CodecName, "L1T2"),
400 SvcTestParameters::Create(kVp9CodecName, "L1T3"),
401 SvcTestParameters::Create(kVp9CodecName, "L2T1"),
402 SvcTestParameters::Create(kVp9CodecName, "L2T1h"),
403 SvcTestParameters::Create(kVp9CodecName, "L2T1_KEY"),
404 SvcTestParameters::Create(kVp9CodecName, "L2T2"),
405 SvcTestParameters::Create(kVp9CodecName, "L2T2h"),
406 SvcTestParameters::Create(kVp9CodecName, "L2T2_KEY"),
407 SvcTestParameters::Create(kVp9CodecName, "L2T2_KEY_SHIFT"),
408 SvcTestParameters::Create(kVp9CodecName, "L2T3"),
409 SvcTestParameters::Create(kVp9CodecName, "L2T3h"),
410 SvcTestParameters::Create(kVp9CodecName, "L2T3_KEY"),
411 // SvcTestParameters::Create(kVp9CodecName, "L2T3_KEY_SHIFT"),
412 SvcTestParameters::Create(kVp9CodecName, "L3T1"),
413 SvcTestParameters::Create(kVp9CodecName, "L3T1h"),
414 SvcTestParameters::Create(kVp9CodecName, "L3T1_KEY"),
415 SvcTestParameters::Create(kVp9CodecName, "L3T2"),
416 SvcTestParameters::Create(kVp9CodecName, "L3T2h"),
417 SvcTestParameters::Create(kVp9CodecName, "L3T2_KEY"),
418 // SvcTestParameters::Create(kVp9CodecName, "L3T2_KEY_SHIFT"),
419 SvcTestParameters::Create(kVp9CodecName, "L3T3"),
420 SvcTestParameters::Create(kVp9CodecName, "L3T3h"),
421 SvcTestParameters::Create(kVp9CodecName, "L3T3_KEY"),
422 // SvcTestParameters::Create(kVp9CodecName, "L3T3_KEY_SHIFT"),
423 SvcTestParameters::Create(kVp9CodecName, "S2T1"),
424 SvcTestParameters::Create(kVp9CodecName, "S2T1h"),
425 SvcTestParameters::Create(kVp9CodecName, "S2T2"),
426 SvcTestParameters::Create(kVp9CodecName, "S2T2h"),
427 SvcTestParameters::Create(kVp9CodecName, "S2T3"),
428 SvcTestParameters::Create(kVp9CodecName, "S2T3h"),
429 SvcTestParameters::Create(kVp9CodecName, "S3T1"),
430 SvcTestParameters::Create(kVp9CodecName, "S3T1h"),
431 SvcTestParameters::Create(kVp9CodecName, "S3T2"),
432 SvcTestParameters::Create(kVp9CodecName, "S3T2h"),
433 SvcTestParameters::Create(kVp9CodecName, "S3T3"),
434 SvcTestParameters::Create(kVp9CodecName, "S3T3h"),
435 }),
436 Values(UseDependencyDescriptor::Disabled,
437 UseDependencyDescriptor::Enabled)),
438 SvcTestNameGenerator);
439
440 INSTANTIATE_TEST_SUITE_P(
441 SvcTestAV1,
442 SvcTest,
443 Combine(ValuesIn({
444 SvcTestParameters::Create(kAv1CodecName, "L1T1"),
445 SvcTestParameters::Create(kAv1CodecName, "L1T2"),
446 SvcTestParameters::Create(kAv1CodecName, "L1T3"),
447 SvcTestParameters::Create(kAv1CodecName, "L2T1"),
448 SvcTestParameters::Create(kAv1CodecName, "L2T1h"),
449 SvcTestParameters::Create(kAv1CodecName, "L2T1_KEY"),
450 SvcTestParameters::Create(kAv1CodecName, "L2T2"),
451 SvcTestParameters::Create(kAv1CodecName, "L2T2h"),
452 SvcTestParameters::Create(kAv1CodecName, "L2T2_KEY"),
453 SvcTestParameters::Create(kAv1CodecName, "L2T2_KEY_SHIFT"),
454 SvcTestParameters::Create(kAv1CodecName, "L2T3"),
455 SvcTestParameters::Create(kAv1CodecName, "L2T3h"),
456 SvcTestParameters::Create(kAv1CodecName, "L2T3_KEY"),
457 // SvcTestParameters::Create(kAv1CodecName, "L2T3_KEY_SHIFT"),
458 SvcTestParameters::Create(kAv1CodecName, "L3T1"),
459 SvcTestParameters::Create(kAv1CodecName, "L3T1h"),
460 SvcTestParameters::Create(kAv1CodecName, "L3T1_KEY"),
461 SvcTestParameters::Create(kAv1CodecName, "L3T2"),
462 SvcTestParameters::Create(kAv1CodecName, "L3T2h"),
463 SvcTestParameters::Create(kAv1CodecName, "L3T2_KEY"),
464 // SvcTestParameters::Create(kAv1CodecName, "L3T2_KEY_SHIFT"),
465 SvcTestParameters::Create(kAv1CodecName, "L3T3"),
466 SvcTestParameters::Create(kAv1CodecName, "L3T3h"),
467 SvcTestParameters::Create(kAv1CodecName, "L3T3_KEY"),
468 // SvcTestParameters::Create(kAv1CodecName, "L3T3_KEY_SHIFT"),
469 SvcTestParameters::Create(kAv1CodecName, "S2T1"),
470 SvcTestParameters::Create(kAv1CodecName, "S2T1h"),
471 SvcTestParameters::Create(kAv1CodecName, "S2T2"),
472 SvcTestParameters::Create(kAv1CodecName, "S2T2h"),
473 SvcTestParameters::Create(kAv1CodecName, "S2T3"),
474 SvcTestParameters::Create(kAv1CodecName, "S2T3h"),
475 SvcTestParameters::Create(kAv1CodecName, "S3T1"),
476 SvcTestParameters::Create(kAv1CodecName, "S3T1h"),
477 SvcTestParameters::Create(kAv1CodecName, "S3T2"),
478 SvcTestParameters::Create(kAv1CodecName, "S3T2h"),
479 SvcTestParameters::Create(kAv1CodecName, "S3T3"),
480 SvcTestParameters::Create(kAv1CodecName, "S3T3h"),
481 }),
482 Values(UseDependencyDescriptor::Enabled)),
483 SvcTestNameGenerator);
484
485 #endif
486
487 } // namespace webrtc
488