1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h"
11
12 #include <fstream>
13 #include <map>
14 #include <vector>
15
16 #include "api/test/metrics/metric.h"
17 #include "api/units/timestamp.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20 #include "test/testsupport/file_utils.h"
21 #include "third_party/catapult/tracing/tracing/value/histogram.h"
22
23 namespace webrtc {
24 namespace test {
25 namespace {
26
27 using ::testing::DoubleNear;
28 using ::testing::Eq;
29 using ::testing::Test;
30
31 namespace proto = ::catapult::tracing::tracing::proto;
32
DefaultMetadata()33 std::map<std::string, std::string> DefaultMetadata() {
34 return std::map<std::string, std::string>{{"key", "value"}};
35 }
36
Sample(double value)37 Metric::TimeSeries::Sample Sample(double value) {
38 return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
39 .value = value,
40 .sample_metadata = DefaultMetadata()};
41 }
42
ReadFileAsString(const std::string & filename)43 std::string ReadFileAsString(const std::string& filename) {
44 std::ifstream infile(filename, std::ios_base::binary);
45 auto buffer = std::vector<char>(std::istreambuf_iterator<char>(infile),
46 std::istreambuf_iterator<char>());
47 return std::string(buffer.begin(), buffer.end());
48 }
49
50 class ChromePerfDashboardMetricsExporterTest : public Test {
51 protected:
52 ~ChromePerfDashboardMetricsExporterTest() override = default;
53
SetUp()54 void SetUp() override {
55 temp_filename_ = webrtc::test::TempFilename(
56 webrtc::test::OutputPath(),
57 "chrome_perf_dashboard_metrics_exporter_test");
58 }
59
TearDown()60 void TearDown() override {
61 ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_));
62 }
63
64 std::string temp_filename_;
65 };
66
TEST_F(ChromePerfDashboardMetricsExporterTest,ExportMetricFormatCorrect)67 TEST_F(ChromePerfDashboardMetricsExporterTest, ExportMetricFormatCorrect) {
68 Metric metric1{
69 .name = "test_metric1",
70 .unit = Unit::kMilliseconds,
71 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
72 .test_case = "test_case_name1",
73 .metric_metadata = DefaultMetadata(),
74 .time_series =
75 Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
76 .stats =
77 Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
78 Metric metric2{
79 .name = "test_metric2",
80 .unit = Unit::kKilobitsPerSecond,
81 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
82 .test_case = "test_case_name2",
83 .metric_metadata = DefaultMetadata(),
84 .time_series =
85 Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
86 .stats = Metric::Stats{
87 .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
88
89 ChromePerfDashboardMetricsExporter exporter(temp_filename_);
90
91 ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
92 proto::HistogramSet actual_histogram_set;
93 actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_));
94 EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(2));
95
96 // Validate output for `metric1`
97 EXPECT_THAT(actual_histogram_set.histograms(0).name(), Eq("test_metric1"));
98 EXPECT_THAT(actual_histogram_set.histograms(0).unit().unit(),
99 Eq(proto::Unit::MS_BEST_FIT_FORMAT));
100 EXPECT_THAT(actual_histogram_set.histograms(0).unit().improvement_direction(),
101 Eq(proto::ImprovementDirection::BIGGER_IS_BETTER));
102 EXPECT_THAT(
103 actual_histogram_set.histograms(0).diagnostics().diagnostic_map().size(),
104 Eq(1lu));
105 EXPECT_THAT(actual_histogram_set.histograms(0)
106 .diagnostics()
107 .diagnostic_map()
108 .at("stories")
109 .generic_set()
110 .values(0),
111 Eq("\"test_case_name1\""));
112 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(2));
113 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(10.0));
114 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(1), Eq(20.0));
115 EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(2));
116 EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(20));
117 EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(),
118 DoubleNear(2.64916, 0.1));
119 EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(15));
120 EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(10));
121 EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(30));
122 EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(50));
123
124 // Validate output for `metric2`
125 EXPECT_THAT(actual_histogram_set.histograms(1).name(), Eq("test_metric2"));
126 EXPECT_THAT(actual_histogram_set.histograms(1).unit().unit(),
127 Eq(proto::Unit::BYTES_PER_SECOND));
128 EXPECT_THAT(actual_histogram_set.histograms(1).unit().improvement_direction(),
129 Eq(proto::ImprovementDirection::SMALLER_IS_BETTER));
130 EXPECT_THAT(
131 actual_histogram_set.histograms(1).diagnostics().diagnostic_map().size(),
132 Eq(1lu));
133 EXPECT_THAT(actual_histogram_set.histograms(1)
134 .diagnostics()
135 .diagnostic_map()
136 .at("stories")
137 .generic_set()
138 .values(0),
139 Eq("\"test_case_name2\""));
140 EXPECT_THAT(actual_histogram_set.histograms(1).sample_values().size(), Eq(2));
141 EXPECT_THAT(actual_histogram_set.histograms(1).sample_values(0), Eq(2500.0));
142 EXPECT_THAT(actual_histogram_set.histograms(1).sample_values(1), Eq(5000.0));
143 EXPECT_THAT(actual_histogram_set.histograms(1).running().count(), Eq(2));
144 EXPECT_THAT(actual_histogram_set.histograms(1).running().max(), Eq(5000));
145 EXPECT_THAT(actual_histogram_set.histograms(1).running().meanlogs(),
146 DoubleNear(8.17062, 0.1));
147 EXPECT_THAT(actual_histogram_set.histograms(1).running().mean(), Eq(3750));
148 EXPECT_THAT(actual_histogram_set.histograms(1).running().min(), Eq(2500));
149 EXPECT_THAT(actual_histogram_set.histograms(1).running().sum(), Eq(7500));
150 EXPECT_THAT(actual_histogram_set.histograms(1).running().variance(),
151 Eq(3125000));
152 }
153
TEST_F(ChromePerfDashboardMetricsExporterTest,ExportEmptyMetricExportsZeroValue)154 TEST_F(ChromePerfDashboardMetricsExporterTest,
155 ExportEmptyMetricExportsZeroValue) {
156 Metric metric{.name = "test_metric",
157 .unit = Unit::kMilliseconds,
158 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
159 .test_case = "test_case_name",
160 .metric_metadata = DefaultMetadata(),
161 .time_series = Metric::TimeSeries{.samples = {}},
162 .stats = Metric::Stats{}};
163
164 ChromePerfDashboardMetricsExporter exporter(temp_filename_);
165
166 ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric}));
167 proto::HistogramSet actual_histogram_set;
168 actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_));
169 EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1));
170
171 // Validate values for `metric`
172 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1));
173 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(0.0));
174 EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1));
175 EXPECT_THAT(actual_histogram_set.histograms(0).running().max(),
176 DoubleNear(0, 1e-6));
177 EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(), Eq(0));
178 EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(0));
179 EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(0));
180 EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(0));
181 EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0));
182 }
183
TEST_F(ChromePerfDashboardMetricsExporterTest,ExportMetricWithOnlyStatsExportsMeanValues)184 TEST_F(ChromePerfDashboardMetricsExporterTest,
185 ExportMetricWithOnlyStatsExportsMeanValues) {
186 Metric metric{.name = "test_metric",
187 .unit = Unit::kMilliseconds,
188 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
189 .test_case = "test_case_name",
190 .metric_metadata = DefaultMetadata(),
191 .time_series = Metric::TimeSeries{.samples = {}},
192 .stats = Metric::Stats{
193 .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
194
195 ChromePerfDashboardMetricsExporter exporter(temp_filename_);
196
197 ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric}));
198 proto::HistogramSet actual_histogram_set;
199 actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_));
200 EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1));
201
202 // Validate values for `metric`
203 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1));
204 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(15.0));
205 EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1));
206 EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(15));
207 EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(),
208 DoubleNear(2.70805, 0.1));
209 EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(15));
210 EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(15));
211 EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(15));
212 EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0));
213 }
214
TEST_F(ChromePerfDashboardMetricsExporterTest,ExportMetricWithOnlyStatsConvertsMeanValuesWhenRequired)215 TEST_F(ChromePerfDashboardMetricsExporterTest,
216 ExportMetricWithOnlyStatsConvertsMeanValuesWhenRequired) {
217 Metric metric{.name = "test_metric",
218 .unit = Unit::kKilobitsPerSecond,
219 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
220 .test_case = "test_case_name",
221 .metric_metadata = DefaultMetadata(),
222 .time_series = Metric::TimeSeries{.samples = {}},
223 .stats = Metric::Stats{
224 .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
225
226 ChromePerfDashboardMetricsExporter exporter(temp_filename_);
227
228 ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric}));
229 proto::HistogramSet actual_histogram_set;
230 actual_histogram_set.ParseFromString(ReadFileAsString(temp_filename_));
231 EXPECT_THAT(actual_histogram_set.histograms().size(), Eq(1));
232
233 // Validate values for `metric`
234 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values().size(), Eq(1));
235 EXPECT_THAT(actual_histogram_set.histograms(0).sample_values(0), Eq(1875.0));
236 EXPECT_THAT(actual_histogram_set.histograms(0).running().count(), Eq(1));
237 EXPECT_THAT(actual_histogram_set.histograms(0).running().max(), Eq(1875));
238 EXPECT_THAT(actual_histogram_set.histograms(0).running().meanlogs(),
239 DoubleNear(7.53636, 0.1));
240 EXPECT_THAT(actual_histogram_set.histograms(0).running().mean(), Eq(1875));
241 EXPECT_THAT(actual_histogram_set.histograms(0).running().min(), Eq(1875));
242 EXPECT_THAT(actual_histogram_set.histograms(0).running().sum(), Eq(1875));
243 EXPECT_THAT(actual_histogram_set.histograms(0).running().variance(), Eq(0));
244 }
245
246 } // namespace
247 } // namespace test
248 } // namespace webrtc
249