1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "api/test/metrics/stdout_metrics_exporter.h"
11
12 #include <map>
13 #include <string>
14 #include <vector>
15
16 #include "api/test/metrics/metric.h"
17 #include "api/units/timestamp.h"
18 #include "test/gmock.h"
19 #include "test/gtest.h"
20
21 namespace webrtc {
22 namespace test {
23 namespace {
24
25 using ::testing::TestWithParam;
26
DefaultMetadata()27 std::map<std::string, std::string> DefaultMetadata() {
28 return std::map<std::string, std::string>{{"key", "value"}};
29 }
30
Sample(double value)31 Metric::TimeSeries::Sample Sample(double value) {
32 return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
33 .value = value,
34 .sample_metadata = DefaultMetadata()};
35 }
36
PsnrForTestFoo(double mean,double stddev)37 Metric PsnrForTestFoo(double mean, double stddev) {
38 return Metric{.name = "psnr",
39 .unit = Unit::kUnitless,
40 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
41 .test_case = "foo",
42 .time_series = Metric::TimeSeries{},
43 .stats = Metric::Stats{.mean = mean, .stddev = stddev}};
44 }
45
TEST(StdoutMetricsExporterTest,ExportMetricFormatCorrect)46 TEST(StdoutMetricsExporterTest, ExportMetricFormatCorrect) {
47 Metric metric1{
48 .name = "test_metric1",
49 .unit = Unit::kMilliseconds,
50 .improvement_direction = ImprovementDirection::kBiggerIsBetter,
51 .test_case = "test_case_name1",
52 .metric_metadata = DefaultMetadata(),
53 .time_series =
54 Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
55 .stats =
56 Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
57 Metric metric2{
58 .name = "test_metric2",
59 .unit = Unit::kKilobitsPerSecond,
60 .improvement_direction = ImprovementDirection::kSmallerIsBetter,
61 .test_case = "test_case_name2",
62 .metric_metadata = DefaultMetadata(),
63 .time_series =
64 Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
65 .stats = Metric::Stats{
66 .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
67
68 testing::internal::CaptureStdout();
69 StdoutMetricsExporter exporter;
70
71 std::string expected =
72 "RESULT: test_case_name1 / test_metric1= "
73 "{mean=15, stddev=5} Milliseconds (BiggerIsBetter)\n"
74 "RESULT: test_case_name2 / test_metric2= "
75 "{mean=30, stddev=10} KilobitsPerSecond (SmallerIsBetter)\n";
76
77 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
78 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
79 }
80
TEST(StdoutMetricsExporterNumberFormatTest,PositiveNumberMaxPrecision)81 TEST(StdoutMetricsExporterNumberFormatTest, PositiveNumberMaxPrecision) {
82 testing::internal::CaptureStdout();
83 StdoutMetricsExporter exporter;
84
85 Metric metric = PsnrForTestFoo(15.00000001, 0.00000001);
86 std::string expected =
87 "RESULT: foo / psnr= "
88 "{mean=15.00000001, stddev=0.00000001} Unitless (BiggerIsBetter)\n";
89 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
90 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
91 }
92
TEST(StdoutMetricsExporterNumberFormatTest,PositiveNumberTrailingZeroNotAdded)93 TEST(StdoutMetricsExporterNumberFormatTest,
94 PositiveNumberTrailingZeroNotAdded) {
95 testing::internal::CaptureStdout();
96 StdoutMetricsExporter exporter;
97
98 Metric metric = PsnrForTestFoo(15.12345, 0.12);
99 std::string expected =
100 "RESULT: foo / psnr= "
101 "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n";
102 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
103 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
104 }
105
TEST(StdoutMetricsExporterNumberFormatTest,PositiveNumberTrailingZeroAreRemoved)106 TEST(StdoutMetricsExporterNumberFormatTest,
107 PositiveNumberTrailingZeroAreRemoved) {
108 testing::internal::CaptureStdout();
109 StdoutMetricsExporter exporter;
110
111 Metric metric = PsnrForTestFoo(15.123450000, 0.120000000);
112 std::string expected =
113 "RESULT: foo / psnr= "
114 "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n";
115 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
116 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
117 }
118
TEST(StdoutMetricsExporterNumberFormatTest,PositiveNumberRoundsUpOnPrecisionCorrectly)119 TEST(StdoutMetricsExporterNumberFormatTest,
120 PositiveNumberRoundsUpOnPrecisionCorrectly) {
121 testing::internal::CaptureStdout();
122 StdoutMetricsExporter exporter;
123
124 Metric metric = PsnrForTestFoo(15.000000009, 0.999999999);
125 std::string expected =
126 "RESULT: foo / psnr= "
127 "{mean=15.00000001, stddev=1} Unitless (BiggerIsBetter)\n";
128 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
129 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
130 }
131
TEST(StdoutMetricsExporterNumberFormatTest,PositiveNumberRoundsDownOnPrecisionCorrectly)132 TEST(StdoutMetricsExporterNumberFormatTest,
133 PositiveNumberRoundsDownOnPrecisionCorrectly) {
134 testing::internal::CaptureStdout();
135 StdoutMetricsExporter exporter;
136
137 Metric metric = PsnrForTestFoo(15.0000000049, 0.9999999949);
138 std::string expected =
139 "RESULT: foo / psnr= "
140 "{mean=15, stddev=0.99999999} Unitless (BiggerIsBetter)\n";
141 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
142 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
143 }
144
TEST(StdoutMetricsExporterNumberFormatTest,NegativeNumberMaxPrecision)145 TEST(StdoutMetricsExporterNumberFormatTest, NegativeNumberMaxPrecision) {
146 testing::internal::CaptureStdout();
147 StdoutMetricsExporter exporter;
148
149 Metric metric = PsnrForTestFoo(-15.00000001, -0.00000001);
150 std::string expected =
151 "RESULT: foo / psnr= "
152 "{mean=-15.00000001, stddev=-0.00000001} Unitless (BiggerIsBetter)\n";
153 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
154 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
155 }
156
TEST(StdoutMetricsExporterNumberFormatTest,NegativeNumberTrailingZeroNotAdded)157 TEST(StdoutMetricsExporterNumberFormatTest,
158 NegativeNumberTrailingZeroNotAdded) {
159 testing::internal::CaptureStdout();
160 StdoutMetricsExporter exporter;
161
162 Metric metric = PsnrForTestFoo(-15.12345, -0.12);
163 std::string expected =
164 "RESULT: foo / psnr= "
165 "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n";
166 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
167 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
168 }
169
TEST(StdoutMetricsExporterNumberFormatTest,NegativeNumberTrailingZeroAreRemoved)170 TEST(StdoutMetricsExporterNumberFormatTest,
171 NegativeNumberTrailingZeroAreRemoved) {
172 testing::internal::CaptureStdout();
173 StdoutMetricsExporter exporter;
174
175 Metric metric = PsnrForTestFoo(-15.123450000, -0.120000000);
176 std::string expected =
177 "RESULT: foo / psnr= "
178 "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n";
179 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
180 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
181 }
182
TEST(StdoutMetricsExporterNumberFormatTest,NegativeNumberRoundsUpOnPrecisionCorrectly)183 TEST(StdoutMetricsExporterNumberFormatTest,
184 NegativeNumberRoundsUpOnPrecisionCorrectly) {
185 testing::internal::CaptureStdout();
186 StdoutMetricsExporter exporter;
187
188 Metric metric = PsnrForTestFoo(-15.000000009, -0.999999999);
189 std::string expected =
190 "RESULT: foo / psnr= "
191 "{mean=-15.00000001, stddev=-1} Unitless (BiggerIsBetter)\n";
192 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
193 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
194 }
195
TEST(StdoutMetricsExporterNumberFormatTest,NegativeNumberRoundsDownOnPrecisionCorrectly)196 TEST(StdoutMetricsExporterNumberFormatTest,
197 NegativeNumberRoundsDownOnPrecisionCorrectly) {
198 testing::internal::CaptureStdout();
199 StdoutMetricsExporter exporter;
200
201 Metric metric = PsnrForTestFoo(-15.0000000049, -0.9999999949);
202 std::string expected =
203 "RESULT: foo / psnr= "
204 "{mean=-15, stddev=-0.99999999} Unitless (BiggerIsBetter)\n";
205 EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
206 EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
207 }
208
209 } // namespace
210 } // namespace test
211 } // namespace webrtc
212