1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <map>
12 #include <memory>
13 #include <string>
14
15 #include "api/test/create_network_emulation_manager.h"
16 #include "api/test/create_peer_connection_quality_test_frame_generator.h"
17 #include "api/test/metrics/metrics_logger.h"
18 #include "api/test/metrics/stdout_metrics_exporter.h"
19 #include "api/test/network_emulation_manager.h"
20 #include "api/test/pclf/media_configuration.h"
21 #include "api/test/pclf/media_quality_test_params.h"
22 #include "api/test/pclf/peer_configurer.h"
23 #include "api/test/peerconnection_quality_test_fixture.h"
24 #include "api/units/time_delta.h"
25 #include "test/gmock.h"
26 #include "test/gtest.h"
27 #include "test/pc/e2e/metric_metadata_keys.h"
28 #include "test/pc/e2e/peer_connection_quality_test.h"
29 #include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
30
31 namespace webrtc {
32 namespace webrtc_pc_e2e {
33 namespace {
34
35 using ::testing::UnorderedElementsAre;
36
37 using ::webrtc::test::DefaultMetricsLogger;
38 using ::webrtc::test::ImprovementDirection;
39 using ::webrtc::test::Metric;
40 using ::webrtc::test::MetricsExporter;
41 using ::webrtc::test::StdoutMetricsExporter;
42 using ::webrtc::test::Unit;
43 using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
44
45 // Adds a peer with some audio and video (the client should not care about
46 // details about audio and video configs).
AddDefaultAudioVideoPeer(absl::string_view peer_name,absl::string_view audio_stream_label,absl::string_view video_stream_label,const PeerNetworkDependencies & network_dependencies,PeerConnectionE2EQualityTestFixture & fixture)47 void AddDefaultAudioVideoPeer(
48 absl::string_view peer_name,
49 absl::string_view audio_stream_label,
50 absl::string_view video_stream_label,
51 const PeerNetworkDependencies& network_dependencies,
52 PeerConnectionE2EQualityTestFixture& fixture) {
53 AudioConfig audio{std::string(audio_stream_label)};
54 audio.sync_group = std::string(peer_name);
55 VideoConfig video(std::string(video_stream_label), 320, 180, 15);
56 video.sync_group = std::string(peer_name);
57 auto peer = std::make_unique<PeerConfigurer>(network_dependencies);
58 peer->SetName(peer_name);
59 peer->SetAudioConfig(std::move(audio));
60 peer->AddVideoConfig(std::move(video));
61 peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
62 fixture.AddPeer(std::move(peer));
63 }
64
65 // Metric fields to assert on
66 struct MetricValidationInfo {
67 std::string test_case;
68 std::string name;
69 Unit unit;
70 ImprovementDirection improvement_direction;
71 std::map<std::string, std::string> metadata;
72 };
73
operator ==(const MetricValidationInfo & a,const MetricValidationInfo & b)74 bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
75 return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
76 a.improvement_direction == b.improvement_direction &&
77 a.metadata == b.metadata;
78 }
79
operator <<(std::ostream & os,const MetricValidationInfo & m)80 std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
81 os << "{ test_case=" << m.test_case << "; name=" << m.name
82 << "; unit=" << test::ToString(m.unit)
83 << "; improvement_direction=" << test::ToString(m.improvement_direction)
84 << "; metadata={ ";
85 for (const auto& [key, value] : m.metadata) {
86 os << "{ key=" << key << "; value=" << value << " }";
87 }
88 os << " }}";
89 return os;
90 }
91
ToValidationInfo(const std::vector<Metric> & metrics)92 std::vector<MetricValidationInfo> ToValidationInfo(
93 const std::vector<Metric>& metrics) {
94 std::vector<MetricValidationInfo> out;
95 for (const Metric& m : metrics) {
96 out.push_back(
97 MetricValidationInfo{.test_case = m.test_case,
98 .name = m.name,
99 .unit = m.unit,
100 .improvement_direction = m.improvement_direction,
101 .metadata = m.metric_metadata});
102 }
103 return out;
104 }
105
TEST(PeerConnectionE2EQualityTestMetricNamesTest,ExportedMetricsHasCorrectNamesAndAnnotation)106 TEST(PeerConnectionE2EQualityTestMetricNamesTest,
107 ExportedMetricsHasCorrectNamesAndAnnotation) {
108 std::unique_ptr<NetworkEmulationManager> network_emulation =
109 CreateNetworkEmulationManager(TimeMode::kSimulated);
110 DefaultMetricsLogger metrics_logger(
111 network_emulation->time_controller()->GetClock());
112 PeerConnectionE2EQualityTest fixture(
113 "test_case", *network_emulation->time_controller(),
114 /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
115 &metrics_logger);
116
117 EmulatedEndpoint* alice_endpoint =
118 network_emulation->CreateEndpoint(EmulatedEndpointConfig());
119 EmulatedEndpoint* bob_endpoint =
120 network_emulation->CreateEndpoint(EmulatedEndpointConfig());
121
122 network_emulation->CreateRoute(
123 alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
124 bob_endpoint);
125 network_emulation->CreateRoute(
126 bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
127 alice_endpoint);
128
129 EmulatedNetworkManagerInterface* alice_network =
130 network_emulation->CreateEmulatedNetworkManagerInterface(
131 {alice_endpoint});
132 EmulatedNetworkManagerInterface* bob_network =
133 network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
134
135 AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
136 alice_network->network_dependencies(), fixture);
137 AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
138 bob_network->network_dependencies(), fixture);
139 fixture.AddQualityMetricsReporter(
140 std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
141 std::map<std::string, std::vector<EmulatedEndpoint*>>(
142 {{"alice", alice_network->endpoints()},
143 {"bob", bob_network->endpoints()}}),
144 network_emulation.get(), &metrics_logger));
145
146 // Run for at least 7 seconds, so AV-sync metrics will be collected.
147 fixture.Run(RunParams(TimeDelta::Seconds(7)));
148
149 std::vector<MetricValidationInfo> metrics =
150 ToValidationInfo(metrics_logger.GetCollectedMetrics());
151 EXPECT_THAT(
152 metrics,
153 UnorderedElementsAre(
154 // Metrics from PeerConnectionE2EQualityTest
155 MetricValidationInfo{
156 .test_case = "test_case",
157 .name = "alice_connected",
158 .unit = Unit::kUnitless,
159 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
160 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
161 MetricValidationInfo{
162 .test_case = "test_case",
163 .name = "bob_connected",
164 .unit = Unit::kUnitless,
165 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
166 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
167
168 // Metrics from DefaultAudioQualityAnalyzer
169 MetricValidationInfo{
170 .test_case = "test_case/alice_audio",
171 .name = "expand_rate",
172 .unit = Unit::kUnitless,
173 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
174 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
175 "alice_audio"},
176 {MetricMetadataKey::kPeerMetadataKey, "bob"},
177 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
178 MetricValidationInfo{
179 .test_case = "test_case/alice_audio",
180 .name = "accelerate_rate",
181 .unit = Unit::kUnitless,
182 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
183 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
184 "alice_audio"},
185 {MetricMetadataKey::kPeerMetadataKey, "bob"},
186 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
187 MetricValidationInfo{
188 .test_case = "test_case/alice_audio",
189 .name = "preemptive_rate",
190 .unit = Unit::kUnitless,
191 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
192 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
193 "alice_audio"},
194 {MetricMetadataKey::kPeerMetadataKey, "bob"},
195 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
196 MetricValidationInfo{
197 .test_case = "test_case/alice_audio",
198 .name = "speech_expand_rate",
199 .unit = Unit::kUnitless,
200 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
201 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
202 "alice_audio"},
203 {MetricMetadataKey::kPeerMetadataKey, "bob"},
204 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
205 MetricValidationInfo{
206 .test_case = "test_case/alice_audio",
207 .name = "average_jitter_buffer_delay_ms",
208 .unit = Unit::kMilliseconds,
209 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
210 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
211 "alice_audio"},
212 {MetricMetadataKey::kPeerMetadataKey, "bob"},
213 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
214 MetricValidationInfo{
215 .test_case = "test_case/alice_audio",
216 .name = "preferred_buffer_size_ms",
217 .unit = Unit::kMilliseconds,
218 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
219 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
220 "alice_audio"},
221 {MetricMetadataKey::kPeerMetadataKey, "bob"},
222 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
223 MetricValidationInfo{
224 .test_case = "test_case/bob_audio",
225 .name = "expand_rate",
226 .unit = Unit::kUnitless,
227 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
228 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
229 "bob_audio"},
230 {MetricMetadataKey::kPeerMetadataKey, "alice"},
231 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
232 MetricValidationInfo{
233 .test_case = "test_case/bob_audio",
234 .name = "accelerate_rate",
235 .unit = Unit::kUnitless,
236 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
237 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
238 "bob_audio"},
239 {MetricMetadataKey::kPeerMetadataKey, "alice"},
240 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
241 MetricValidationInfo{
242 .test_case = "test_case/bob_audio",
243 .name = "preemptive_rate",
244 .unit = Unit::kUnitless,
245 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
246 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
247 "bob_audio"},
248 {MetricMetadataKey::kPeerMetadataKey, "alice"},
249 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
250 MetricValidationInfo{
251 .test_case = "test_case/bob_audio",
252 .name = "speech_expand_rate",
253 .unit = Unit::kUnitless,
254 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
255 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
256 "bob_audio"},
257 {MetricMetadataKey::kPeerMetadataKey, "alice"},
258 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
259 MetricValidationInfo{
260 .test_case = "test_case/bob_audio",
261 .name = "average_jitter_buffer_delay_ms",
262 .unit = Unit::kMilliseconds,
263 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
264 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
265 "bob_audio"},
266 {MetricMetadataKey::kPeerMetadataKey, "alice"},
267 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
268 MetricValidationInfo{
269 .test_case = "test_case/bob_audio",
270 .name = "preferred_buffer_size_ms",
271 .unit = Unit::kMilliseconds,
272 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
273 .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
274 "bob_audio"},
275 {MetricMetadataKey::kPeerMetadataKey, "alice"},
276 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
277
278 // Metrics from DefaultVideoQualityAnalyzer
279 MetricValidationInfo{
280 .test_case = "test_case/alice_video",
281 .name = "psnr_dB",
282 .unit = Unit::kUnitless,
283 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
284 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
285 {MetricMetadataKey::kVideoStreamMetadataKey,
286 "alice_video"},
287 {MetricMetadataKey::kSenderMetadataKey, "alice"},
288 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
289 MetricValidationInfo{
290 .test_case = "test_case/alice_video",
291 .name = "ssim",
292 .unit = Unit::kUnitless,
293 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
294 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
295 {MetricMetadataKey::kVideoStreamMetadataKey,
296 "alice_video"},
297 {MetricMetadataKey::kSenderMetadataKey, "alice"},
298 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
299 MetricValidationInfo{
300 .test_case = "test_case/alice_video",
301 .name = "transport_time",
302 .unit = Unit::kMilliseconds,
303 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
304 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
305 {MetricMetadataKey::kVideoStreamMetadataKey,
306 "alice_video"},
307 {MetricMetadataKey::kSenderMetadataKey, "alice"},
308 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
309 MetricValidationInfo{
310 .test_case = "test_case/alice_video",
311 .name = "total_delay_incl_transport",
312 .unit = Unit::kMilliseconds,
313 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
314 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
315 {MetricMetadataKey::kVideoStreamMetadataKey,
316 "alice_video"},
317 {MetricMetadataKey::kSenderMetadataKey, "alice"},
318 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
319 MetricValidationInfo{
320 .test_case = "test_case/alice_video",
321 .name = "time_between_rendered_frames",
322 .unit = Unit::kMilliseconds,
323 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
324 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
325 {MetricMetadataKey::kVideoStreamMetadataKey,
326 "alice_video"},
327 {MetricMetadataKey::kSenderMetadataKey, "alice"},
328 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
329 MetricValidationInfo{
330 .test_case = "test_case/alice_video",
331 .name = "harmonic_framerate",
332 .unit = Unit::kHertz,
333 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
334 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
335 {MetricMetadataKey::kVideoStreamMetadataKey,
336 "alice_video"},
337 {MetricMetadataKey::kSenderMetadataKey, "alice"},
338 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
339 MetricValidationInfo{
340 .test_case = "test_case/alice_video",
341 .name = "encode_frame_rate",
342 .unit = Unit::kHertz,
343 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
344 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
345 {MetricMetadataKey::kVideoStreamMetadataKey,
346 "alice_video"},
347 {MetricMetadataKey::kSenderMetadataKey, "alice"},
348 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
349 MetricValidationInfo{
350 .test_case = "test_case/alice_video",
351 .name = "encode_time",
352 .unit = Unit::kMilliseconds,
353 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
354 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
355 {MetricMetadataKey::kVideoStreamMetadataKey,
356 "alice_video"},
357 {MetricMetadataKey::kSenderMetadataKey, "alice"},
358 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
359 MetricValidationInfo{
360 .test_case = "test_case/alice_video",
361 .name = "time_between_freezes",
362 .unit = Unit::kMilliseconds,
363 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
364 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
365 {MetricMetadataKey::kVideoStreamMetadataKey,
366 "alice_video"},
367 {MetricMetadataKey::kSenderMetadataKey, "alice"},
368 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
369 MetricValidationInfo{
370 .test_case = "test_case/alice_video",
371 .name = "freeze_time_ms",
372 .unit = Unit::kMilliseconds,
373 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
374 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
375 {MetricMetadataKey::kVideoStreamMetadataKey,
376 "alice_video"},
377 {MetricMetadataKey::kSenderMetadataKey, "alice"},
378 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
379 MetricValidationInfo{
380 .test_case = "test_case/alice_video",
381 .name = "pixels_per_frame",
382 .unit = Unit::kCount,
383 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
384 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
385 {MetricMetadataKey::kVideoStreamMetadataKey,
386 "alice_video"},
387 {MetricMetadataKey::kSenderMetadataKey, "alice"},
388 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
389 MetricValidationInfo{
390 .test_case = "test_case/alice_video",
391 .name = "min_psnr_dB",
392 .unit = Unit::kUnitless,
393 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
394 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
395 {MetricMetadataKey::kVideoStreamMetadataKey,
396 "alice_video"},
397 {MetricMetadataKey::kSenderMetadataKey, "alice"},
398 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
399 MetricValidationInfo{
400 .test_case = "test_case/alice_video",
401 .name = "decode_time",
402 .unit = Unit::kMilliseconds,
403 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
404 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
405 {MetricMetadataKey::kVideoStreamMetadataKey,
406 "alice_video"},
407 {MetricMetadataKey::kSenderMetadataKey, "alice"},
408 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
409 MetricValidationInfo{
410 .test_case = "test_case/alice_video",
411 .name = "receive_to_render_time",
412 .unit = Unit::kMilliseconds,
413 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
414 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
415 {MetricMetadataKey::kVideoStreamMetadataKey,
416 "alice_video"},
417 {MetricMetadataKey::kSenderMetadataKey, "alice"},
418 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
419 MetricValidationInfo{
420 .test_case = "test_case/alice_video",
421 .name = "dropped_frames",
422 .unit = Unit::kCount,
423 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
424 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
425 {MetricMetadataKey::kVideoStreamMetadataKey,
426 "alice_video"},
427 {MetricMetadataKey::kSenderMetadataKey, "alice"},
428 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
429 MetricValidationInfo{
430 .test_case = "test_case/alice_video",
431 .name = "frames_in_flight",
432 .unit = Unit::kCount,
433 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
434 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
435 {MetricMetadataKey::kVideoStreamMetadataKey,
436 "alice_video"},
437 {MetricMetadataKey::kSenderMetadataKey, "alice"},
438 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
439 MetricValidationInfo{
440 .test_case = "test_case/alice_video",
441 .name = "rendered_frames",
442 .unit = Unit::kCount,
443 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
444 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
445 {MetricMetadataKey::kVideoStreamMetadataKey,
446 "alice_video"},
447 {MetricMetadataKey::kSenderMetadataKey, "alice"},
448 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
449 MetricValidationInfo{
450 .test_case = "test_case/alice_video",
451 .name = "max_skipped",
452 .unit = Unit::kCount,
453 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
454 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
455 {MetricMetadataKey::kVideoStreamMetadataKey,
456 "alice_video"},
457 {MetricMetadataKey::kSenderMetadataKey, "alice"},
458 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
459 MetricValidationInfo{
460 .test_case = "test_case/alice_video",
461 .name = "target_encode_bitrate",
462 .unit = Unit::kKilobitsPerSecond,
463 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
464 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
465 {MetricMetadataKey::kVideoStreamMetadataKey,
466 "alice_video"},
467 {MetricMetadataKey::kSenderMetadataKey, "alice"},
468 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
469 MetricValidationInfo{
470 .test_case = "test_case/alice_video",
471 .name = "qp",
472 .unit = Unit::kUnitless,
473 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
474 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
475 {MetricMetadataKey::kVideoStreamMetadataKey,
476 "alice_video"},
477 {MetricMetadataKey::kSenderMetadataKey, "alice"},
478 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
479 MetricValidationInfo{
480 .test_case = "test_case/alice_video",
481 .name = "actual_encode_bitrate",
482 .unit = Unit::kKilobitsPerSecond,
483 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
484 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
485 {MetricMetadataKey::kVideoStreamMetadataKey,
486 "alice_video"},
487 {MetricMetadataKey::kSenderMetadataKey, "alice"},
488 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
489 MetricValidationInfo{
490 .test_case = "test_case/bob_video",
491 .name = "psnr_dB",
492 .unit = Unit::kUnitless,
493 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
494 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
495 {MetricMetadataKey::kVideoStreamMetadataKey,
496 "bob_video"},
497 {MetricMetadataKey::kSenderMetadataKey, "bob"},
498 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
499 MetricValidationInfo{
500 .test_case = "test_case/bob_video",
501 .name = "ssim",
502 .unit = Unit::kUnitless,
503 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
504 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
505 {MetricMetadataKey::kVideoStreamMetadataKey,
506 "bob_video"},
507 {MetricMetadataKey::kSenderMetadataKey, "bob"},
508 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
509 MetricValidationInfo{
510 .test_case = "test_case/bob_video",
511 .name = "transport_time",
512 .unit = Unit::kMilliseconds,
513 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
514 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
515 {MetricMetadataKey::kVideoStreamMetadataKey,
516 "bob_video"},
517 {MetricMetadataKey::kSenderMetadataKey, "bob"},
518 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
519 MetricValidationInfo{
520 .test_case = "test_case/bob_video",
521 .name = "total_delay_incl_transport",
522 .unit = Unit::kMilliseconds,
523 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
524 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
525 {MetricMetadataKey::kVideoStreamMetadataKey,
526 "bob_video"},
527 {MetricMetadataKey::kSenderMetadataKey, "bob"},
528 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
529 MetricValidationInfo{
530 .test_case = "test_case/bob_video",
531 .name = "time_between_rendered_frames",
532 .unit = Unit::kMilliseconds,
533 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
534 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
535 {MetricMetadataKey::kVideoStreamMetadataKey,
536 "bob_video"},
537 {MetricMetadataKey::kSenderMetadataKey, "bob"},
538 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
539 MetricValidationInfo{
540 .test_case = "test_case/bob_video",
541 .name = "harmonic_framerate",
542 .unit = Unit::kHertz,
543 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
544 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
545 {MetricMetadataKey::kVideoStreamMetadataKey,
546 "bob_video"},
547 {MetricMetadataKey::kSenderMetadataKey, "bob"},
548 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
549 MetricValidationInfo{
550 .test_case = "test_case/bob_video",
551 .name = "encode_frame_rate",
552 .unit = Unit::kHertz,
553 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
554 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
555 {MetricMetadataKey::kVideoStreamMetadataKey,
556 "bob_video"},
557 {MetricMetadataKey::kSenderMetadataKey, "bob"},
558 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
559 MetricValidationInfo{
560 .test_case = "test_case/bob_video",
561 .name = "encode_time",
562 .unit = Unit::kMilliseconds,
563 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
564 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
565 {MetricMetadataKey::kVideoStreamMetadataKey,
566 "bob_video"},
567 {MetricMetadataKey::kSenderMetadataKey, "bob"},
568 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
569 MetricValidationInfo{
570 .test_case = "test_case/bob_video",
571 .name = "time_between_freezes",
572 .unit = Unit::kMilliseconds,
573 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
574 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
575 {MetricMetadataKey::kVideoStreamMetadataKey,
576 "bob_video"},
577 {MetricMetadataKey::kSenderMetadataKey, "bob"},
578 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
579 MetricValidationInfo{
580 .test_case = "test_case/bob_video",
581 .name = "freeze_time_ms",
582 .unit = Unit::kMilliseconds,
583 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
584 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
585 {MetricMetadataKey::kVideoStreamMetadataKey,
586 "bob_video"},
587 {MetricMetadataKey::kSenderMetadataKey, "bob"},
588 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
589 MetricValidationInfo{
590 .test_case = "test_case/bob_video",
591 .name = "pixels_per_frame",
592 .unit = Unit::kCount,
593 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
594 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
595 {MetricMetadataKey::kVideoStreamMetadataKey,
596 "bob_video"},
597 {MetricMetadataKey::kSenderMetadataKey, "bob"},
598 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
599 MetricValidationInfo{
600 .test_case = "test_case/bob_video",
601 .name = "min_psnr_dB",
602 .unit = Unit::kUnitless,
603 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
604 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
605 {MetricMetadataKey::kVideoStreamMetadataKey,
606 "bob_video"},
607 {MetricMetadataKey::kSenderMetadataKey, "bob"},
608 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
609 MetricValidationInfo{
610 .test_case = "test_case/bob_video",
611 .name = "decode_time",
612 .unit = Unit::kMilliseconds,
613 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
614 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
615 {MetricMetadataKey::kVideoStreamMetadataKey,
616 "bob_video"},
617 {MetricMetadataKey::kSenderMetadataKey, "bob"},
618 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
619 MetricValidationInfo{
620 .test_case = "test_case/bob_video",
621 .name = "receive_to_render_time",
622 .unit = Unit::kMilliseconds,
623 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
624 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
625 {MetricMetadataKey::kVideoStreamMetadataKey,
626 "bob_video"},
627 {MetricMetadataKey::kSenderMetadataKey, "bob"},
628 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
629 MetricValidationInfo{
630 .test_case = "test_case/bob_video",
631 .name = "dropped_frames",
632 .unit = Unit::kCount,
633 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
634 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
635 {MetricMetadataKey::kVideoStreamMetadataKey,
636 "bob_video"},
637 {MetricMetadataKey::kSenderMetadataKey, "bob"},
638 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
639 MetricValidationInfo{
640 .test_case = "test_case/bob_video",
641 .name = "frames_in_flight",
642 .unit = Unit::kCount,
643 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
644 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
645 {MetricMetadataKey::kVideoStreamMetadataKey,
646 "bob_video"},
647 {MetricMetadataKey::kSenderMetadataKey, "bob"},
648 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
649 MetricValidationInfo{
650 .test_case = "test_case/bob_video",
651 .name = "rendered_frames",
652 .unit = Unit::kCount,
653 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
654 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
655 {MetricMetadataKey::kVideoStreamMetadataKey,
656 "bob_video"},
657 {MetricMetadataKey::kSenderMetadataKey, "bob"},
658 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
659 MetricValidationInfo{
660 .test_case = "test_case/bob_video",
661 .name = "max_skipped",
662 .unit = Unit::kCount,
663 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
664 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
665 {MetricMetadataKey::kVideoStreamMetadataKey,
666 "bob_video"},
667 {MetricMetadataKey::kSenderMetadataKey, "bob"},
668 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
669 MetricValidationInfo{
670 .test_case = "test_case/bob_video",
671 .name = "target_encode_bitrate",
672 .unit = Unit::kKilobitsPerSecond,
673 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
674 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
675 {MetricMetadataKey::kVideoStreamMetadataKey,
676 "bob_video"},
677 {MetricMetadataKey::kSenderMetadataKey, "bob"},
678 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
679 MetricValidationInfo{
680 .test_case = "test_case/bob_video",
681 .name = "actual_encode_bitrate",
682 .unit = Unit::kKilobitsPerSecond,
683 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
684 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
685 {MetricMetadataKey::kVideoStreamMetadataKey,
686 "bob_video"},
687 {MetricMetadataKey::kSenderMetadataKey, "bob"},
688 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
689 MetricValidationInfo{
690 .test_case = "test_case/bob_video",
691 .name = "qp",
692 .unit = Unit::kUnitless,
693 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
694 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
695 {MetricMetadataKey::kVideoStreamMetadataKey,
696 "bob_video"},
697 {MetricMetadataKey::kSenderMetadataKey, "bob"},
698 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
699 MetricValidationInfo{
700 .test_case = "test_case",
701 .name = "cpu_usage_%",
702 .unit = Unit::kUnitless,
703 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
704 .metadata = {}},
705
706 // Metrics from StatsBasedNetworkQualityMetricsReporter
707 MetricValidationInfo{
708 .test_case = "test_case/alice",
709 .name = "bytes_discarded_no_receiver",
710 .unit = Unit::kBytes,
711 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
712 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
713 MetricValidationInfo{
714 .test_case = "test_case/alice",
715 .name = "packets_discarded_no_receiver",
716 .unit = Unit::kUnitless,
717 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
718 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
719 MetricValidationInfo{
720 .test_case = "test_case/alice",
721 .name = "payload_bytes_received",
722 .unit = Unit::kBytes,
723 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
724 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
725 MetricValidationInfo{
726 .test_case = "test_case/alice",
727 .name = "payload_bytes_sent",
728 .unit = Unit::kBytes,
729 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
730 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
731 MetricValidationInfo{
732 .test_case = "test_case/alice",
733 .name = "bytes_sent",
734 .unit = Unit::kBytes,
735 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
736 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
737 MetricValidationInfo{
738 .test_case = "test_case/alice",
739 .name = "packets_sent",
740 .unit = Unit::kUnitless,
741 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
742 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
743 MetricValidationInfo{
744 .test_case = "test_case/alice",
745 .name = "average_send_rate",
746 .unit = Unit::kKilobitsPerSecond,
747 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
748 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
749 MetricValidationInfo{
750 .test_case = "test_case/alice",
751 .name = "bytes_received",
752 .unit = Unit::kBytes,
753 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
754 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
755 MetricValidationInfo{
756 .test_case = "test_case/alice",
757 .name = "packets_received",
758 .unit = Unit::kUnitless,
759 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
760 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
761 MetricValidationInfo{
762 .test_case = "test_case/alice",
763 .name = "average_receive_rate",
764 .unit = Unit::kKilobitsPerSecond,
765 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
766 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
767 MetricValidationInfo{
768 .test_case = "test_case/alice",
769 .name = "sent_packets_loss",
770 .unit = Unit::kUnitless,
771 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
772 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
773 MetricValidationInfo{
774 .test_case = "test_case/bob",
775 .name = "bytes_discarded_no_receiver",
776 .unit = Unit::kBytes,
777 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
778 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
779 MetricValidationInfo{
780 .test_case = "test_case/bob",
781 .name = "packets_discarded_no_receiver",
782 .unit = Unit::kUnitless,
783 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
784 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
785 MetricValidationInfo{
786 .test_case = "test_case/bob",
787 .name = "payload_bytes_received",
788 .unit = Unit::kBytes,
789 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
790 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
791 MetricValidationInfo{
792 .test_case = "test_case/bob",
793 .name = "payload_bytes_sent",
794 .unit = Unit::kBytes,
795 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
796 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
797 MetricValidationInfo{
798 .test_case = "test_case/bob",
799 .name = "bytes_sent",
800 .unit = Unit::kBytes,
801 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
802 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
803 MetricValidationInfo{
804 .test_case = "test_case/bob",
805 .name = "packets_sent",
806 .unit = Unit::kUnitless,
807 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
808 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
809 MetricValidationInfo{
810 .test_case = "test_case/bob",
811 .name = "average_send_rate",
812 .unit = Unit::kKilobitsPerSecond,
813 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
814 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
815 MetricValidationInfo{
816 .test_case = "test_case/bob",
817 .name = "bytes_received",
818 .unit = Unit::kBytes,
819 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
820 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
821 MetricValidationInfo{
822 .test_case = "test_case/bob",
823 .name = "packets_received",
824 .unit = Unit::kUnitless,
825 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
826 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
827 MetricValidationInfo{
828 .test_case = "test_case/bob",
829 .name = "average_receive_rate",
830 .unit = Unit::kKilobitsPerSecond,
831 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
832 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
833 MetricValidationInfo{
834 .test_case = "test_case/bob",
835 .name = "sent_packets_loss",
836 .unit = Unit::kUnitless,
837 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
838 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
839
840 // Metrics from VideoQualityMetricsReporter
841 MetricValidationInfo{
842 .test_case = "test_case/alice",
843 .name = "available_send_bandwidth",
844 .unit = Unit::kKilobitsPerSecond,
845 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
846 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
847 MetricValidationInfo{
848 .test_case = "test_case/alice",
849 .name = "transmission_bitrate",
850 .unit = Unit::kKilobitsPerSecond,
851 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
852 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
853 MetricValidationInfo{
854 .test_case = "test_case/alice",
855 .name = "retransmission_bitrate",
856 .unit = Unit::kKilobitsPerSecond,
857 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
858 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
859 MetricValidationInfo{
860 .test_case = "test_case/bob",
861 .name = "available_send_bandwidth",
862 .unit = Unit::kKilobitsPerSecond,
863 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
864 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
865 MetricValidationInfo{
866 .test_case = "test_case/bob",
867 .name = "transmission_bitrate",
868 .unit = Unit::kKilobitsPerSecond,
869 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
870 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
871 MetricValidationInfo{
872 .test_case = "test_case/bob",
873 .name = "retransmission_bitrate",
874 .unit = Unit::kKilobitsPerSecond,
875 .improvement_direction = ImprovementDirection::kNeitherIsBetter,
876 .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
877
878 // Metrics from CrossMediaMetricsReporter
879 MetricValidationInfo{
880 .test_case = "test_case/alice_alice_audio",
881 .name = "audio_ahead_ms",
882 .unit = Unit::kMilliseconds,
883 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
884 .metadata =
885 {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"},
886 {MetricMetadataKey::kPeerMetadataKey, "bob"},
887 {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
888 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
889 MetricValidationInfo{
890 .test_case = "test_case/alice_alice_video",
891 .name = "video_ahead_ms",
892 .unit = Unit::kMilliseconds,
893 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
894 .metadata =
895 {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"},
896 {MetricMetadataKey::kPeerMetadataKey, "bob"},
897 {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
898 {MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
899 MetricValidationInfo{
900 .test_case = "test_case/bob_bob_audio",
901 .name = "audio_ahead_ms",
902 .unit = Unit::kMilliseconds,
903 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
904 .metadata =
905 {{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"},
906 {MetricMetadataKey::kPeerMetadataKey, "alice"},
907 {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
908 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
909 MetricValidationInfo{
910 .test_case = "test_case/bob_bob_video",
911 .name = "video_ahead_ms",
912 .unit = Unit::kMilliseconds,
913 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
914 .metadata = {
915 {MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"},
916 {MetricMetadataKey::kPeerMetadataKey, "alice"},
917 {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
918 {MetricMetadataKey::kReceiverMetadataKey, "alice"}}}));
919 }
920
921 } // namespace
922 } // namespace webrtc_pc_e2e
923 } // namespace webrtc
924