1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_TESTSUPPORT_PERF_TEST_H_ 12 #define TEST_TESTSUPPORT_PERF_TEST_H_ 13 14 #include <sstream> 15 #include <string> 16 #include <vector> 17 18 #include "absl/strings/string_view.h" 19 #include "api/array_view.h" 20 #include "api/numerics/samples_stats_counter.h" 21 22 namespace webrtc { 23 namespace test { 24 25 enum class ImproveDirection { 26 // Direction is undefined. 27 kNone, 28 // Smaller value is better. 29 kSmallerIsBetter, 30 // Bigger value is better. 31 kBiggerIsBetter, 32 }; 33 34 // Prints a performance test result. 35 // 36 // For example, 37 // PrintResult("ramp_up_time_", "turn_over_tcp", 38 // "bwe_15s", 1234.2, "ms", false); 39 // 40 // will show up in the http://chromeperf.appspot.com under 41 // 42 // (test binary name) > (bot) > ramp_up_time_turn_over_tcp > bwe_15s. 43 // 44 // The `measurement` + `modifier` is what we're measuring. `user_story` is the 45 // scenario we're testing under. 46 // 47 // The binary this runs in must be hooked up as a perf test in the WebRTC 48 // recipes for this to actually be uploaded to chromeperf.appspot.com. 49 void PrintResult(absl::string_view measurement, 50 absl::string_view modifier, 51 absl::string_view user_story, 52 double value, 53 absl::string_view units, 54 bool important, 55 ImproveDirection improve_direction = ImproveDirection::kNone); 56 57 // Like PrintResult(), but prints a (mean, standard deviation) result pair. 58 // The |<values>| should be two comma-separated numbers, the mean and 59 // standard deviation (or other error metric) of the measurement. 60 // DEPRECATED: soon unsupported. 61 void PrintResultMeanAndError( 62 absl::string_view measurement, 63 absl::string_view modifier, 64 absl::string_view user_story, 65 double mean, 66 double error, 67 absl::string_view units, 68 bool important, 69 ImproveDirection improve_direction = ImproveDirection::kNone); 70 71 // Like PrintResult(), but prints an entire list of results. The `values` 72 // will generally be a list of comma-separated numbers. A typical 73 // post-processing step might produce plots of their mean and standard 74 // deviation. 75 void PrintResultList( 76 absl::string_view measurement, 77 absl::string_view modifier, 78 absl::string_view user_story, 79 rtc::ArrayView<const double> values, 80 absl::string_view units, 81 bool important, 82 ImproveDirection improve_direction = ImproveDirection::kNone); 83 84 // Like PrintResult(), but prints a (mean, standard deviation) from stats 85 // counter. Also add specified metric to the plotable metrics output. 86 void PrintResult(absl::string_view measurement, 87 absl::string_view modifier, 88 absl::string_view user_story, 89 const SamplesStatsCounter& counter, 90 absl::string_view units, 91 bool important, 92 ImproveDirection improve_direction = ImproveDirection::kNone); 93 94 // Returns a string-encoded proto as described in 95 // tracing/tracing/proto/histogram.proto in 96 // https://github.com/catapult-project/catapult/blob/master/. 97 // If you want to print the proto in human readable format, use 98 // tracing/bin/proto2json from third_party/catapult in your WebRTC checkout. 99 std::string GetPerfResults(); 100 101 // Print into stdout plottable metrics for further post processing. 102 // `desired_graphs` - list of metrics, that should be plotted. If empty - all 103 // available metrics will be plotted. If some of `desired_graphs` are missing 104 // they will be skipped. 105 void PrintPlottableResults(const std::vector<std::string>& desired_graphs); 106 107 // Call GetPerfResults() and write its output to a file. Returns false if we 108 // failed to write to the file. If you want to print the proto in human readable 109 // format, use tracing/bin/proto2json from third_party/catapult in your WebRTC 110 // checkout. 111 bool WritePerfResults(const std::string& output_path); 112 113 // By default, human-readable perf results are printed to stdout. Set the FILE* 114 // to where they should be printing instead. These results are not used to 115 // upload to the dashboard, however - this is only through WritePerfResults. 116 void SetPerfResultsOutput(FILE* output); 117 118 // Only for use by tests. 119 void ClearPerfResults(); 120 121 } // namespace test 122 } // namespace webrtc 123 124 #endif // TEST_TESTSUPPORT_PERF_TEST_H_ 125