1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/pc/e2e/network_quality_metrics_reporter.h"
11
12 #include <utility>
13
14 #include "api/stats/rtc_stats.h"
15 #include "api/stats/rtcstats_objects.h"
16 #include "api/test/metrics/metric.h"
17 #include "rtc_base/checks.h"
18 #include "rtc_base/event.h"
19 #include "system_wrappers/include/field_trial.h"
20
21 namespace webrtc {
22 namespace webrtc_pc_e2e {
23 namespace {
24
25 using ::webrtc::test::ImprovementDirection;
26 using ::webrtc::test::Unit;
27
28 constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
29
30 // Field trial which controls whether to report standard-compliant bytes
31 // sent/received per stream. If enabled, padding and headers are not included
32 // in bytes sent or received.
33 constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
34
35 } // namespace
36
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface * alice_network,EmulatedNetworkManagerInterface * bob_network,test::MetricsLogger * metrics_logger)37 NetworkQualityMetricsReporter::NetworkQualityMetricsReporter(
38 EmulatedNetworkManagerInterface* alice_network,
39 EmulatedNetworkManagerInterface* bob_network,
40 test::MetricsLogger* metrics_logger)
41 : alice_network_(alice_network),
42 bob_network_(bob_network),
43 metrics_logger_(metrics_logger) {
44 RTC_CHECK(metrics_logger_);
45 }
46
Start(absl::string_view test_case_name,const TrackIdStreamInfoMap *)47 void NetworkQualityMetricsReporter::Start(
48 absl::string_view test_case_name,
49 const TrackIdStreamInfoMap* /*reporter_helper*/) {
50 test_case_name_ = std::string(test_case_name);
51 // Check that network stats are clean before test execution.
52 EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
53 RTC_CHECK_EQ(alice_stats.overall_outgoing_stats.packets_sent, 0);
54 RTC_CHECK_EQ(alice_stats.overall_incoming_stats.packets_received, 0);
55 EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
56 RTC_CHECK_EQ(bob_stats.overall_outgoing_stats.packets_sent, 0);
57 RTC_CHECK_EQ(bob_stats.overall_incoming_stats.packets_received, 0);
58 }
59
OnStatsReports(absl::string_view pc_label,const rtc::scoped_refptr<const RTCStatsReport> & report)60 void NetworkQualityMetricsReporter::OnStatsReports(
61 absl::string_view pc_label,
62 const rtc::scoped_refptr<const RTCStatsReport>& report) {
63 DataSize payload_received = DataSize::Zero();
64 DataSize payload_sent = DataSize::Zero();
65
66 auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
67 for (const auto& stat : inbound_stats) {
68 payload_received +=
69 DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) +
70 stat->header_bytes_received.ValueOrDefault(0ul));
71 }
72
73 auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>();
74 for (const auto& stat : outbound_stats) {
75 payload_sent +=
76 DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) +
77 stat->header_bytes_sent.ValueOrDefault(0ul));
78 }
79
80 MutexLock lock(&lock_);
81 PCStats& stats = pc_stats_[std::string(pc_label)];
82 stats.payload_received = payload_received;
83 stats.payload_sent = payload_sent;
84 }
85
StopAndReportResults()86 void NetworkQualityMetricsReporter::StopAndReportResults() {
87 EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
88 EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
89 int64_t alice_packets_loss =
90 alice_stats.overall_outgoing_stats.packets_sent -
91 bob_stats.overall_incoming_stats.packets_received;
92 int64_t bob_packets_loss =
93 bob_stats.overall_outgoing_stats.packets_sent -
94 alice_stats.overall_incoming_stats.packets_received;
95 ReportStats("alice", alice_stats, alice_packets_loss);
96 ReportStats("bob", bob_stats, bob_packets_loss);
97
98 if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) {
99 RTC_LOG(LS_ERROR)
100 << "Non-standard GetStats; \"payload\" counts include RTP headers";
101 }
102
103 MutexLock lock(&lock_);
104 for (const auto& pair : pc_stats_) {
105 ReportPCStats(pair.first, pair.second);
106 }
107 }
108
PopulateStats(EmulatedNetworkManagerInterface * network)109 EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats(
110 EmulatedNetworkManagerInterface* network) {
111 rtc::Event wait;
112 EmulatedNetworkStats stats;
113 network->GetStats([&](EmulatedNetworkStats s) {
114 stats = std::move(s);
115 wait.Set();
116 });
117 bool stats_received = wait.Wait(kStatsWaitTimeout);
118 RTC_CHECK(stats_received);
119 return stats;
120 }
121
ReportStats(const std::string & network_label,const EmulatedNetworkStats & stats,int64_t packet_loss)122 void NetworkQualityMetricsReporter::ReportStats(
123 const std::string& network_label,
124 const EmulatedNetworkStats& stats,
125 int64_t packet_loss) {
126 metrics_logger_->LogSingleValueMetric(
127 "bytes_sent", GetTestCaseName(network_label),
128 stats.overall_outgoing_stats.bytes_sent.bytes(), Unit::kBytes,
129 ImprovementDirection::kNeitherIsBetter);
130 metrics_logger_->LogSingleValueMetric(
131 "packets_sent", GetTestCaseName(network_label),
132 stats.overall_outgoing_stats.packets_sent, Unit::kUnitless,
133 ImprovementDirection::kNeitherIsBetter);
134 metrics_logger_->LogSingleValueMetric(
135 "average_send_rate", GetTestCaseName(network_label),
136 stats.overall_outgoing_stats.packets_sent >= 2
137 ? stats.overall_outgoing_stats.AverageSendRate().kbps<double>()
138 : 0,
139 Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
140 metrics_logger_->LogSingleValueMetric(
141 "bytes_discarded_no_receiver", GetTestCaseName(network_label),
142 stats.overall_incoming_stats.bytes_discarded_no_receiver.bytes(),
143 Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
144 metrics_logger_->LogSingleValueMetric(
145 "packets_discarded_no_receiver", GetTestCaseName(network_label),
146 stats.overall_incoming_stats.packets_discarded_no_receiver,
147 Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
148 metrics_logger_->LogSingleValueMetric(
149 "bytes_received", GetTestCaseName(network_label),
150 stats.overall_incoming_stats.bytes_received.bytes(), Unit::kBytes,
151 ImprovementDirection::kNeitherIsBetter);
152 metrics_logger_->LogSingleValueMetric(
153 "packets_received", GetTestCaseName(network_label),
154 stats.overall_incoming_stats.packets_received, Unit::kUnitless,
155 ImprovementDirection::kNeitherIsBetter);
156 metrics_logger_->LogSingleValueMetric(
157 "average_receive_rate", GetTestCaseName(network_label),
158 stats.overall_incoming_stats.packets_received >= 2
159 ? stats.overall_incoming_stats.AverageReceiveRate().kbps<double>()
160 : 0,
161 Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
162 metrics_logger_->LogSingleValueMetric(
163 "sent_packets_loss", GetTestCaseName(network_label), packet_loss,
164 Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
165 }
166
ReportPCStats(const std::string & pc_label,const PCStats & stats)167 void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label,
168 const PCStats& stats) {
169 metrics_logger_->LogSingleValueMetric(
170 "payload_bytes_received", pc_label, stats.payload_received.bytes(),
171 Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
172 metrics_logger_->LogSingleValueMetric(
173 "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes,
174 ImprovementDirection::kNeitherIsBetter);
175 }
176
GetTestCaseName(const std::string & network_label) const177 std::string NetworkQualityMetricsReporter::GetTestCaseName(
178 const std::string& network_label) const {
179 return test_case_name_ + "/" + network_label;
180 }
181
182 } // namespace webrtc_pc_e2e
183 } // namespace webrtc
184