1
2 #undef NDEBUG
3 #include <numeric>
4 #include <utility>
5
6 #include "benchmark/benchmark.h"
7 #include "output_test.h"
8
9 // ========================================================================= //
10 // ---------------------- Testing Prologue Output -------------------------- //
11 // ========================================================================= //
12
13 ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
14 {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
15 {"^[-]+$", MR_Next}});
AddContextCases()16 static int AddContextCases() {
17 AddCases(TC_ConsoleErr,
18 {
19 {"^%int-%int-%intT%int:%int:%int[-+]%int:%int$", MR_Default},
20 {"Running .*(/|\\\\)reporter_output_test(\\.exe)?$", MR_Next},
21 {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
22 });
23 AddCases(TC_JSONOut,
24 {{"^\\{", MR_Default},
25 {"\"context\":", MR_Next},
26 {"\"date\": \"", MR_Next},
27 {"\"host_name\":", MR_Next},
28 {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
29 MR_Next},
30 {"\"num_cpus\": %int,$", MR_Next},
31 {"\"mhz_per_cpu\": %float,$", MR_Next},
32 {"\"caches\": \\[$", MR_Default}});
33 auto const& Info = benchmark::CPUInfo::Get();
34 auto const& Caches = Info.caches;
35 if (!Caches.empty()) {
36 AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
37 }
38 for (size_t I = 0; I < Caches.size(); ++I) {
39 std::string num_caches_str =
40 Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
41 AddCases(TC_ConsoleErr,
42 {{"L%int (Data|Instruction|Unified) %int KiB" + num_caches_str,
43 MR_Next}});
44 AddCases(TC_JSONOut, {{"\\{$", MR_Next},
45 {"\"type\": \"", MR_Next},
46 {"\"level\": %int,$", MR_Next},
47 {"\"size\": %int,$", MR_Next},
48 {"\"num_sharing\": %int$", MR_Next},
49 {"}[,]{0,1}$", MR_Next}});
50 }
51 AddCases(TC_JSONOut, {{"],$"}});
52 auto const& LoadAvg = Info.load_avg;
53 if (!LoadAvg.empty()) {
54 AddCases(TC_ConsoleErr,
55 {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
56 }
57 AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
58 AddCases(TC_JSONOut, {{"\"library_version\": \".*\",$", MR_Next}});
59 AddCases(TC_JSONOut, {{"\"library_build_type\": \".*\",$", MR_Next}});
60 AddCases(TC_JSONOut, {{"\"json_schema_version\": 1$", MR_Next}});
61 return 0;
62 }
63 int dummy_register = AddContextCases();
64 ADD_CASES(TC_CSVOut, {{"%csv_header"}});
65
66 // ========================================================================= //
67 // ------------------------ Testing Basic Output --------------------------- //
68 // ========================================================================= //
69
BM_basic(benchmark::State & state)70 void BM_basic(benchmark::State& state) {
71 for (auto _ : state) {
72 }
73 }
74 BENCHMARK(BM_basic);
75
76 ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
77 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
78 {"\"family_index\": 0,$", MR_Next},
79 {"\"per_family_instance_index\": 0,$", MR_Next},
80 {"\"run_name\": \"BM_basic\",$", MR_Next},
81 {"\"run_type\": \"iteration\",$", MR_Next},
82 {"\"repetitions\": 1,$", MR_Next},
83 {"\"repetition_index\": 0,$", MR_Next},
84 {"\"threads\": 1,$", MR_Next},
85 {"\"iterations\": %int,$", MR_Next},
86 {"\"real_time\": %float,$", MR_Next},
87 {"\"cpu_time\": %float,$", MR_Next},
88 {"\"time_unit\": \"ns\"$", MR_Next},
89 {"}", MR_Next}});
90 ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
91
92 // ========================================================================= //
93 // ------------------------ Testing Bytes per Second Output ---------------- //
94 // ========================================================================= //
95
BM_bytes_per_second(benchmark::State & state)96 void BM_bytes_per_second(benchmark::State& state) {
97 for (auto _ : state) {
98 // This test requires a non-zero CPU time to avoid divide-by-zero
99 auto iterations = double(state.iterations()) * double(state.iterations());
100 benchmark::DoNotOptimize(iterations);
101 }
102 state.SetBytesProcessed(1);
103 }
104 BENCHMARK(BM_bytes_per_second);
105
106 ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
107 "bytes_per_second=%float[kM]{0,1}/s$"}});
108 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
109 {"\"family_index\": 1,$", MR_Next},
110 {"\"per_family_instance_index\": 0,$", MR_Next},
111 {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
112 {"\"run_type\": \"iteration\",$", MR_Next},
113 {"\"repetitions\": 1,$", MR_Next},
114 {"\"repetition_index\": 0,$", MR_Next},
115 {"\"threads\": 1,$", MR_Next},
116 {"\"iterations\": %int,$", MR_Next},
117 {"\"real_time\": %float,$", MR_Next},
118 {"\"cpu_time\": %float,$", MR_Next},
119 {"\"time_unit\": \"ns\",$", MR_Next},
120 {"\"bytes_per_second\": %float$", MR_Next},
121 {"}", MR_Next}});
122 ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
123
124 // ========================================================================= //
125 // ------------------------ Testing Items per Second Output ---------------- //
126 // ========================================================================= //
127
BM_items_per_second(benchmark::State & state)128 void BM_items_per_second(benchmark::State& state) {
129 for (auto _ : state) {
130 // This test requires a non-zero CPU time to avoid divide-by-zero
131 auto iterations = double(state.iterations()) * double(state.iterations());
132 benchmark::DoNotOptimize(iterations);
133 }
134 state.SetItemsProcessed(1);
135 }
136 BENCHMARK(BM_items_per_second);
137
138 ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
139 "items_per_second=%float[kM]{0,1}/s$"}});
140 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
141 {"\"family_index\": 2,$", MR_Next},
142 {"\"per_family_instance_index\": 0,$", MR_Next},
143 {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
144 {"\"run_type\": \"iteration\",$", MR_Next},
145 {"\"repetitions\": 1,$", MR_Next},
146 {"\"repetition_index\": 0,$", MR_Next},
147 {"\"threads\": 1,$", MR_Next},
148 {"\"iterations\": %int,$", MR_Next},
149 {"\"real_time\": %float,$", MR_Next},
150 {"\"cpu_time\": %float,$", MR_Next},
151 {"\"time_unit\": \"ns\",$", MR_Next},
152 {"\"items_per_second\": %float$", MR_Next},
153 {"}", MR_Next}});
154 ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
155
156 // ========================================================================= //
157 // ------------------------ Testing Label Output --------------------------- //
158 // ========================================================================= //
159
BM_label(benchmark::State & state)160 void BM_label(benchmark::State& state) {
161 for (auto _ : state) {
162 }
163 state.SetLabel("some label");
164 }
165 BENCHMARK(BM_label);
166
167 ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
168 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
169 {"\"family_index\": 3,$", MR_Next},
170 {"\"per_family_instance_index\": 0,$", MR_Next},
171 {"\"run_name\": \"BM_label\",$", MR_Next},
172 {"\"run_type\": \"iteration\",$", MR_Next},
173 {"\"repetitions\": 1,$", MR_Next},
174 {"\"repetition_index\": 0,$", MR_Next},
175 {"\"threads\": 1,$", MR_Next},
176 {"\"iterations\": %int,$", MR_Next},
177 {"\"real_time\": %float,$", MR_Next},
178 {"\"cpu_time\": %float,$", MR_Next},
179 {"\"time_unit\": \"ns\",$", MR_Next},
180 {"\"label\": \"some label\"$", MR_Next},
181 {"}", MR_Next}});
182 ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
183 "label\"%csv_label_report_end$"}});
184
185 // ========================================================================= //
186 // ------------------------ Testing Time Label Output ---------------------- //
187 // ========================================================================= //
188
BM_time_label_nanosecond(benchmark::State & state)189 void BM_time_label_nanosecond(benchmark::State& state) {
190 for (auto _ : state) {
191 }
192 }
193 BENCHMARK(BM_time_label_nanosecond)->Unit(benchmark::kNanosecond);
194
195 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_nanosecond %console_report$"}});
196 ADD_CASES(TC_JSONOut,
197 {{"\"name\": \"BM_time_label_nanosecond\",$"},
198 {"\"family_index\": 4,$", MR_Next},
199 {"\"per_family_instance_index\": 0,$", MR_Next},
200 {"\"run_name\": \"BM_time_label_nanosecond\",$", MR_Next},
201 {"\"run_type\": \"iteration\",$", MR_Next},
202 {"\"repetitions\": 1,$", MR_Next},
203 {"\"repetition_index\": 0,$", MR_Next},
204 {"\"threads\": 1,$", MR_Next},
205 {"\"iterations\": %int,$", MR_Next},
206 {"\"real_time\": %float,$", MR_Next},
207 {"\"cpu_time\": %float,$", MR_Next},
208 {"\"time_unit\": \"ns\"$", MR_Next},
209 {"}", MR_Next}});
210 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_nanosecond\",%csv_report$"}});
211
BM_time_label_microsecond(benchmark::State & state)212 void BM_time_label_microsecond(benchmark::State& state) {
213 for (auto _ : state) {
214 }
215 }
216 BENCHMARK(BM_time_label_microsecond)->Unit(benchmark::kMicrosecond);
217
218 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_microsecond %console_us_report$"}});
219 ADD_CASES(TC_JSONOut,
220 {{"\"name\": \"BM_time_label_microsecond\",$"},
221 {"\"family_index\": 5,$", MR_Next},
222 {"\"per_family_instance_index\": 0,$", MR_Next},
223 {"\"run_name\": \"BM_time_label_microsecond\",$", MR_Next},
224 {"\"run_type\": \"iteration\",$", MR_Next},
225 {"\"repetitions\": 1,$", MR_Next},
226 {"\"repetition_index\": 0,$", MR_Next},
227 {"\"threads\": 1,$", MR_Next},
228 {"\"iterations\": %int,$", MR_Next},
229 {"\"real_time\": %float,$", MR_Next},
230 {"\"cpu_time\": %float,$", MR_Next},
231 {"\"time_unit\": \"us\"$", MR_Next},
232 {"}", MR_Next}});
233 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_microsecond\",%csv_us_report$"}});
234
BM_time_label_millisecond(benchmark::State & state)235 void BM_time_label_millisecond(benchmark::State& state) {
236 for (auto _ : state) {
237 }
238 }
239 BENCHMARK(BM_time_label_millisecond)->Unit(benchmark::kMillisecond);
240
241 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_millisecond %console_ms_report$"}});
242 ADD_CASES(TC_JSONOut,
243 {{"\"name\": \"BM_time_label_millisecond\",$"},
244 {"\"family_index\": 6,$", MR_Next},
245 {"\"per_family_instance_index\": 0,$", MR_Next},
246 {"\"run_name\": \"BM_time_label_millisecond\",$", MR_Next},
247 {"\"run_type\": \"iteration\",$", MR_Next},
248 {"\"repetitions\": 1,$", MR_Next},
249 {"\"repetition_index\": 0,$", MR_Next},
250 {"\"threads\": 1,$", MR_Next},
251 {"\"iterations\": %int,$", MR_Next},
252 {"\"real_time\": %float,$", MR_Next},
253 {"\"cpu_time\": %float,$", MR_Next},
254 {"\"time_unit\": \"ms\"$", MR_Next},
255 {"}", MR_Next}});
256 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_millisecond\",%csv_ms_report$"}});
257
BM_time_label_second(benchmark::State & state)258 void BM_time_label_second(benchmark::State& state) {
259 for (auto _ : state) {
260 }
261 }
262 BENCHMARK(BM_time_label_second)->Unit(benchmark::kSecond);
263
264 ADD_CASES(TC_ConsoleOut, {{"^BM_time_label_second %console_s_report$"}});
265 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_time_label_second\",$"},
266 {"\"family_index\": 7,$", MR_Next},
267 {"\"per_family_instance_index\": 0,$", MR_Next},
268 {"\"run_name\": \"BM_time_label_second\",$", MR_Next},
269 {"\"run_type\": \"iteration\",$", MR_Next},
270 {"\"repetitions\": 1,$", MR_Next},
271 {"\"repetition_index\": 0,$", MR_Next},
272 {"\"threads\": 1,$", MR_Next},
273 {"\"iterations\": %int,$", MR_Next},
274 {"\"real_time\": %float,$", MR_Next},
275 {"\"cpu_time\": %float,$", MR_Next},
276 {"\"time_unit\": \"s\"$", MR_Next},
277 {"}", MR_Next}});
278 ADD_CASES(TC_CSVOut, {{"^\"BM_time_label_second\",%csv_s_report$"}});
279
280 // ========================================================================= //
281 // ------------------------ Testing Error Output --------------------------- //
282 // ========================================================================= //
283
BM_error(benchmark::State & state)284 void BM_error(benchmark::State& state) {
285 state.SkipWithError("message");
286 for (auto _ : state) {
287 }
288 }
289 BENCHMARK(BM_error);
290 ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
291 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
292 {"\"family_index\": 8,$", MR_Next},
293 {"\"per_family_instance_index\": 0,$", MR_Next},
294 {"\"run_name\": \"BM_error\",$", MR_Next},
295 {"\"run_type\": \"iteration\",$", MR_Next},
296 {"\"repetitions\": 1,$", MR_Next},
297 {"\"repetition_index\": 0,$", MR_Next},
298 {"\"threads\": 1,$", MR_Next},
299 {"\"error_occurred\": true,$", MR_Next},
300 {"\"error_message\": \"message\",$", MR_Next}});
301
302 ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
303
304 // ========================================================================= //
305 // ------------------------ Testing No Arg Name Output -----------------------
306 // //
307 // ========================================================================= //
308
BM_no_arg_name(benchmark::State & state)309 void BM_no_arg_name(benchmark::State& state) {
310 for (auto _ : state) {
311 }
312 }
313 BENCHMARK(BM_no_arg_name)->Arg(3);
314 ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
315 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
316 {"\"family_index\": 9,$", MR_Next},
317 {"\"per_family_instance_index\": 0,$", MR_Next},
318 {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
319 {"\"run_type\": \"iteration\",$", MR_Next},
320 {"\"repetitions\": 1,$", MR_Next},
321 {"\"repetition_index\": 0,$", MR_Next},
322 {"\"threads\": 1,$", MR_Next}});
323 ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
324
325 // ========================================================================= //
326 // ------------------------ Testing Arg Name Output ------------------------ //
327 // ========================================================================= //
328
BM_arg_name(benchmark::State & state)329 void BM_arg_name(benchmark::State& state) {
330 for (auto _ : state) {
331 }
332 }
333 BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
334 ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
335 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
336 {"\"family_index\": 10,$", MR_Next},
337 {"\"per_family_instance_index\": 0,$", MR_Next},
338 {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
339 {"\"run_type\": \"iteration\",$", MR_Next},
340 {"\"repetitions\": 1,$", MR_Next},
341 {"\"repetition_index\": 0,$", MR_Next},
342 {"\"threads\": 1,$", MR_Next}});
343 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
344
345 // ========================================================================= //
346 // ------------------------ Testing Arg Names Output ----------------------- //
347 // ========================================================================= //
348
BM_arg_names(benchmark::State & state)349 void BM_arg_names(benchmark::State& state) {
350 for (auto _ : state) {
351 }
352 }
353 BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
354 ADD_CASES(TC_ConsoleOut,
355 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
356 ADD_CASES(TC_JSONOut,
357 {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
358 {"\"family_index\": 11,$", MR_Next},
359 {"\"per_family_instance_index\": 0,$", MR_Next},
360 {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
361 {"\"run_type\": \"iteration\",$", MR_Next},
362 {"\"repetitions\": 1,$", MR_Next},
363 {"\"repetition_index\": 0,$", MR_Next},
364 {"\"threads\": 1,$", MR_Next}});
365 ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
366
367 // ========================================================================= //
368 // ------------------------ Testing Name Output ---------------------------- //
369 // ========================================================================= //
370
BM_name(benchmark::State & state)371 void BM_name(benchmark::State& state) {
372 for (auto _ : state) {
373 }
374 }
375 BENCHMARK(BM_name)->Name("BM_custom_name");
376
377 ADD_CASES(TC_ConsoleOut, {{"^BM_custom_name %console_report$"}});
378 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_custom_name\",$"},
379 {"\"family_index\": 12,$", MR_Next},
380 {"\"per_family_instance_index\": 0,$", MR_Next},
381 {"\"run_name\": \"BM_custom_name\",$", MR_Next},
382 {"\"run_type\": \"iteration\",$", MR_Next},
383 {"\"repetitions\": 1,$", MR_Next},
384 {"\"repetition_index\": 0,$", MR_Next},
385 {"\"threads\": 1,$", MR_Next},
386 {"\"iterations\": %int,$", MR_Next},
387 {"\"real_time\": %float,$", MR_Next},
388 {"\"cpu_time\": %float,$", MR_Next},
389 {"\"time_unit\": \"ns\"$", MR_Next},
390 {"}", MR_Next}});
391 ADD_CASES(TC_CSVOut, {{"^\"BM_custom_name\",%csv_report$"}});
392
393 // ========================================================================= //
394 // ------------------------ Testing Big Args Output ------------------------ //
395 // ========================================================================= //
396
BM_BigArgs(benchmark::State & state)397 void BM_BigArgs(benchmark::State& state) {
398 for (auto _ : state) {
399 }
400 }
401 BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
402 ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
403 {"^BM_BigArgs/2147483648 %console_report$"}});
404
405 // ========================================================================= //
406 // ----------------------- Testing Complexity Output ----------------------- //
407 // ========================================================================= //
408
BM_Complexity_O1(benchmark::State & state)409 void BM_Complexity_O1(benchmark::State& state) {
410 for (auto _ : state) {
411 // This test requires a non-zero CPU time to avoid divide-by-zero
412 auto iterations = double(state.iterations()) * double(state.iterations());
413 benchmark::DoNotOptimize(iterations);
414 }
415 state.SetComplexityN(state.range(0));
416 }
417 BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
418 SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
419 {"%RMS", "[ ]*[0-9]+ %"}});
420 ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
421 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
422
423 // ========================================================================= //
424 // ----------------------- Testing Aggregate Output ------------------------ //
425 // ========================================================================= //
426
427 // Test that non-aggregate data is printed by default
BM_Repeat(benchmark::State & state)428 void BM_Repeat(benchmark::State& state) {
429 for (auto _ : state) {
430 }
431 }
432 // need two repetitions min to be able to output any aggregate output
433 BENCHMARK(BM_Repeat)->Repetitions(2);
434 ADD_CASES(TC_ConsoleOut,
435 {{"^BM_Repeat/repeats:2 %console_report$"},
436 {"^BM_Repeat/repeats:2 %console_report$"},
437 {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
438 {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
439 {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
440 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
441 {"\"family_index\": 15,$", MR_Next},
442 {"\"per_family_instance_index\": 0,$", MR_Next},
443 {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
444 {"\"run_type\": \"iteration\",$", MR_Next},
445 {"\"repetitions\": 2,$", MR_Next},
446 {"\"repetition_index\": 0,$", MR_Next},
447 {"\"threads\": 1,$", MR_Next},
448 {"\"name\": \"BM_Repeat/repeats:2\",$"},
449 {"\"family_index\": 15,$", MR_Next},
450 {"\"per_family_instance_index\": 0,$", MR_Next},
451 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
452 {"\"run_type\": \"iteration\",$", MR_Next},
453 {"\"repetitions\": 2,$", MR_Next},
454 {"\"repetition_index\": 1,$", MR_Next},
455 {"\"threads\": 1,$", MR_Next},
456 {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
457 {"\"family_index\": 15,$", MR_Next},
458 {"\"per_family_instance_index\": 0,$", MR_Next},
459 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
460 {"\"run_type\": \"aggregate\",$", MR_Next},
461 {"\"repetitions\": 2,$", MR_Next},
462 {"\"threads\": 1,$", MR_Next},
463 {"\"aggregate_name\": \"mean\",$", MR_Next},
464 {"\"aggregate_unit\": \"time\",$", MR_Next},
465 {"\"iterations\": 2,$", MR_Next},
466 {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
467 {"\"family_index\": 15,$", MR_Next},
468 {"\"per_family_instance_index\": 0,$", MR_Next},
469 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
470 {"\"run_type\": \"aggregate\",$", MR_Next},
471 {"\"repetitions\": 2,$", MR_Next},
472 {"\"threads\": 1,$", MR_Next},
473 {"\"aggregate_name\": \"median\",$", MR_Next},
474 {"\"aggregate_unit\": \"time\",$", MR_Next},
475 {"\"iterations\": 2,$", MR_Next},
476 {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
477 {"\"family_index\": 15,$", MR_Next},
478 {"\"per_family_instance_index\": 0,$", MR_Next},
479 {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
480 {"\"run_type\": \"aggregate\",$", MR_Next},
481 {"\"repetitions\": 2,$", MR_Next},
482 {"\"threads\": 1,$", MR_Next},
483 {"\"aggregate_name\": \"stddev\",$", MR_Next},
484 {"\"aggregate_unit\": \"time\",$", MR_Next},
485 {"\"iterations\": 2,$", MR_Next}});
486 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
487 {"^\"BM_Repeat/repeats:2\",%csv_report$"},
488 {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
489 {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
490 {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
491 // but for two repetitions, mean and median is the same, so let's repeat..
492 BENCHMARK(BM_Repeat)->Repetitions(3);
493 ADD_CASES(TC_ConsoleOut,
494 {{"^BM_Repeat/repeats:3 %console_report$"},
495 {"^BM_Repeat/repeats:3 %console_report$"},
496 {"^BM_Repeat/repeats:3 %console_report$"},
497 {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
498 {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
499 {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
500 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
501 {"\"family_index\": 16,$", MR_Next},
502 {"\"per_family_instance_index\": 0,$", MR_Next},
503 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
504 {"\"run_type\": \"iteration\",$", MR_Next},
505 {"\"repetitions\": 3,$", MR_Next},
506 {"\"repetition_index\": 0,$", MR_Next},
507 {"\"threads\": 1,$", MR_Next},
508 {"\"name\": \"BM_Repeat/repeats:3\",$"},
509 {"\"family_index\": 16,$", MR_Next},
510 {"\"per_family_instance_index\": 0,$", MR_Next},
511 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
512 {"\"run_type\": \"iteration\",$", MR_Next},
513 {"\"repetitions\": 3,$", MR_Next},
514 {"\"repetition_index\": 1,$", MR_Next},
515 {"\"threads\": 1,$", MR_Next},
516 {"\"name\": \"BM_Repeat/repeats:3\",$"},
517 {"\"family_index\": 16,$", MR_Next},
518 {"\"per_family_instance_index\": 0,$", MR_Next},
519 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
520 {"\"run_type\": \"iteration\",$", MR_Next},
521 {"\"repetitions\": 3,$", MR_Next},
522 {"\"repetition_index\": 2,$", MR_Next},
523 {"\"threads\": 1,$", MR_Next},
524 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
525 {"\"family_index\": 16,$", MR_Next},
526 {"\"per_family_instance_index\": 0,$", MR_Next},
527 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
528 {"\"run_type\": \"aggregate\",$", MR_Next},
529 {"\"repetitions\": 3,$", MR_Next},
530 {"\"threads\": 1,$", MR_Next},
531 {"\"aggregate_name\": \"mean\",$", MR_Next},
532 {"\"aggregate_unit\": \"time\",$", MR_Next},
533 {"\"iterations\": 3,$", MR_Next},
534 {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
535 {"\"family_index\": 16,$", MR_Next},
536 {"\"per_family_instance_index\": 0,$", MR_Next},
537 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
538 {"\"run_type\": \"aggregate\",$", MR_Next},
539 {"\"repetitions\": 3,$", MR_Next},
540 {"\"threads\": 1,$", MR_Next},
541 {"\"aggregate_name\": \"median\",$", MR_Next},
542 {"\"aggregate_unit\": \"time\",$", MR_Next},
543 {"\"iterations\": 3,$", MR_Next},
544 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
545 {"\"family_index\": 16,$", MR_Next},
546 {"\"per_family_instance_index\": 0,$", MR_Next},
547 {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
548 {"\"run_type\": \"aggregate\",$", MR_Next},
549 {"\"repetitions\": 3,$", MR_Next},
550 {"\"threads\": 1,$", MR_Next},
551 {"\"aggregate_name\": \"stddev\",$", MR_Next},
552 {"\"aggregate_unit\": \"time\",$", MR_Next},
553 {"\"iterations\": 3,$", MR_Next}});
554 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
555 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
556 {"^\"BM_Repeat/repeats:3\",%csv_report$"},
557 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
558 {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
559 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
560 // median differs between even/odd number of repetitions, so just to be sure
561 BENCHMARK(BM_Repeat)->Repetitions(4);
562 ADD_CASES(TC_ConsoleOut,
563 {{"^BM_Repeat/repeats:4 %console_report$"},
564 {"^BM_Repeat/repeats:4 %console_report$"},
565 {"^BM_Repeat/repeats:4 %console_report$"},
566 {"^BM_Repeat/repeats:4 %console_report$"},
567 {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
568 {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
569 {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
570 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
571 {"\"family_index\": 17,$", MR_Next},
572 {"\"per_family_instance_index\": 0,$", MR_Next},
573 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
574 {"\"run_type\": \"iteration\",$", MR_Next},
575 {"\"repetitions\": 4,$", MR_Next},
576 {"\"repetition_index\": 0,$", MR_Next},
577 {"\"threads\": 1,$", MR_Next},
578 {"\"name\": \"BM_Repeat/repeats:4\",$"},
579 {"\"family_index\": 17,$", MR_Next},
580 {"\"per_family_instance_index\": 0,$", MR_Next},
581 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
582 {"\"run_type\": \"iteration\",$", MR_Next},
583 {"\"repetitions\": 4,$", MR_Next},
584 {"\"repetition_index\": 1,$", MR_Next},
585 {"\"threads\": 1,$", MR_Next},
586 {"\"name\": \"BM_Repeat/repeats:4\",$"},
587 {"\"family_index\": 17,$", MR_Next},
588 {"\"per_family_instance_index\": 0,$", MR_Next},
589 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
590 {"\"run_type\": \"iteration\",$", MR_Next},
591 {"\"repetitions\": 4,$", MR_Next},
592 {"\"repetition_index\": 2,$", MR_Next},
593 {"\"threads\": 1,$", MR_Next},
594 {"\"name\": \"BM_Repeat/repeats:4\",$"},
595 {"\"family_index\": 17,$", MR_Next},
596 {"\"per_family_instance_index\": 0,$", MR_Next},
597 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
598 {"\"run_type\": \"iteration\",$", MR_Next},
599 {"\"repetitions\": 4,$", MR_Next},
600 {"\"repetition_index\": 3,$", MR_Next},
601 {"\"threads\": 1,$", MR_Next},
602 {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
603 {"\"family_index\": 17,$", MR_Next},
604 {"\"per_family_instance_index\": 0,$", MR_Next},
605 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
606 {"\"run_type\": \"aggregate\",$", MR_Next},
607 {"\"repetitions\": 4,$", MR_Next},
608 {"\"threads\": 1,$", MR_Next},
609 {"\"aggregate_name\": \"mean\",$", MR_Next},
610 {"\"aggregate_unit\": \"time\",$", MR_Next},
611 {"\"iterations\": 4,$", MR_Next},
612 {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
613 {"\"family_index\": 17,$", MR_Next},
614 {"\"per_family_instance_index\": 0,$", MR_Next},
615 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
616 {"\"run_type\": \"aggregate\",$", MR_Next},
617 {"\"repetitions\": 4,$", MR_Next},
618 {"\"threads\": 1,$", MR_Next},
619 {"\"aggregate_name\": \"median\",$", MR_Next},
620 {"\"aggregate_unit\": \"time\",$", MR_Next},
621 {"\"iterations\": 4,$", MR_Next},
622 {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
623 {"\"family_index\": 17,$", MR_Next},
624 {"\"per_family_instance_index\": 0,$", MR_Next},
625 {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
626 {"\"run_type\": \"aggregate\",$", MR_Next},
627 {"\"repetitions\": 4,$", MR_Next},
628 {"\"threads\": 1,$", MR_Next},
629 {"\"aggregate_name\": \"stddev\",$", MR_Next},
630 {"\"aggregate_unit\": \"time\",$", MR_Next},
631 {"\"iterations\": 4,$", MR_Next}});
632 ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
633 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
634 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
635 {"^\"BM_Repeat/repeats:4\",%csv_report$"},
636 {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
637 {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
638 {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
639
640 // Test that a non-repeated test still prints non-aggregate results even when
641 // only-aggregate reports have been requested
BM_RepeatOnce(benchmark::State & state)642 void BM_RepeatOnce(benchmark::State& state) {
643 for (auto _ : state) {
644 }
645 }
646 BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
647 ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
648 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
649 {"\"family_index\": 18,$", MR_Next},
650 {"\"per_family_instance_index\": 0,$", MR_Next},
651 {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
652 {"\"run_type\": \"iteration\",$", MR_Next},
653 {"\"repetitions\": 1,$", MR_Next},
654 {"\"repetition_index\": 0,$", MR_Next},
655 {"\"threads\": 1,$", MR_Next}});
656 ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
657
658 // Test that non-aggregate data is not reported
BM_SummaryRepeat(benchmark::State & state)659 void BM_SummaryRepeat(benchmark::State& state) {
660 for (auto _ : state) {
661 }
662 }
663 BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
664 ADD_CASES(
665 TC_ConsoleOut,
666 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
667 {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
668 {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
669 {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
670 ADD_CASES(TC_JSONOut,
671 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
672 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
673 {"\"family_index\": 19,$", MR_Next},
674 {"\"per_family_instance_index\": 0,$", MR_Next},
675 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
676 {"\"run_type\": \"aggregate\",$", MR_Next},
677 {"\"repetitions\": 3,$", MR_Next},
678 {"\"threads\": 1,$", MR_Next},
679 {"\"aggregate_name\": \"mean\",$", MR_Next},
680 {"\"aggregate_unit\": \"time\",$", MR_Next},
681 {"\"iterations\": 3,$", MR_Next},
682 {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
683 {"\"family_index\": 19,$", MR_Next},
684 {"\"per_family_instance_index\": 0,$", MR_Next},
685 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
686 {"\"run_type\": \"aggregate\",$", MR_Next},
687 {"\"repetitions\": 3,$", MR_Next},
688 {"\"threads\": 1,$", MR_Next},
689 {"\"aggregate_name\": \"median\",$", MR_Next},
690 {"\"aggregate_unit\": \"time\",$", MR_Next},
691 {"\"iterations\": 3,$", MR_Next},
692 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
693 {"\"family_index\": 19,$", MR_Next},
694 {"\"per_family_instance_index\": 0,$", MR_Next},
695 {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
696 {"\"run_type\": \"aggregate\",$", MR_Next},
697 {"\"repetitions\": 3,$", MR_Next},
698 {"\"threads\": 1,$", MR_Next},
699 {"\"aggregate_name\": \"stddev\",$", MR_Next},
700 {"\"aggregate_unit\": \"time\",$", MR_Next},
701 {"\"iterations\": 3,$", MR_Next}});
702 ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
703 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
704 {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
705 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
706
707 // Test that non-aggregate data is not displayed.
708 // NOTE: this test is kinda bad. we are only testing the display output.
709 // But we don't check that the file output still contains everything...
BM_SummaryDisplay(benchmark::State & state)710 void BM_SummaryDisplay(benchmark::State& state) {
711 for (auto _ : state) {
712 }
713 }
714 BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
715 ADD_CASES(
716 TC_ConsoleOut,
717 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
718 {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
719 {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
720 {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
721 ADD_CASES(TC_JSONOut,
722 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
723 {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
724 {"\"family_index\": 20,$", MR_Next},
725 {"\"per_family_instance_index\": 0,$", MR_Next},
726 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
727 {"\"run_type\": \"aggregate\",$", MR_Next},
728 {"\"repetitions\": 2,$", MR_Next},
729 {"\"threads\": 1,$", MR_Next},
730 {"\"aggregate_name\": \"mean\",$", MR_Next},
731 {"\"aggregate_unit\": \"time\",$", MR_Next},
732 {"\"iterations\": 2,$", MR_Next},
733 {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
734 {"\"family_index\": 20,$", MR_Next},
735 {"\"per_family_instance_index\": 0,$", MR_Next},
736 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
737 {"\"run_type\": \"aggregate\",$", MR_Next},
738 {"\"repetitions\": 2,$", MR_Next},
739 {"\"threads\": 1,$", MR_Next},
740 {"\"aggregate_name\": \"median\",$", MR_Next},
741 {"\"aggregate_unit\": \"time\",$", MR_Next},
742 {"\"iterations\": 2,$", MR_Next},
743 {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
744 {"\"family_index\": 20,$", MR_Next},
745 {"\"per_family_instance_index\": 0,$", MR_Next},
746 {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
747 {"\"run_type\": \"aggregate\",$", MR_Next},
748 {"\"repetitions\": 2,$", MR_Next},
749 {"\"threads\": 1,$", MR_Next},
750 {"\"aggregate_name\": \"stddev\",$", MR_Next},
751 {"\"aggregate_unit\": \"time\",$", MR_Next},
752 {"\"iterations\": 2,$", MR_Next}});
753 ADD_CASES(TC_CSVOut,
754 {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
755 {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
756 {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
757 {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
758
759 // Test repeats with custom time unit.
BM_RepeatTimeUnit(benchmark::State & state)760 void BM_RepeatTimeUnit(benchmark::State& state) {
761 for (auto _ : state) {
762 }
763 }
764 BENCHMARK(BM_RepeatTimeUnit)
765 ->Repetitions(3)
766 ->ReportAggregatesOnly()
767 ->Unit(benchmark::kMicrosecond);
768 ADD_CASES(
769 TC_ConsoleOut,
770 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
771 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
772 {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
773 "]*3$"},
774 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
775 "]*3$"}});
776 ADD_CASES(TC_JSONOut,
777 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
778 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
779 {"\"family_index\": 21,$", MR_Next},
780 {"\"per_family_instance_index\": 0,$", MR_Next},
781 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
782 {"\"run_type\": \"aggregate\",$", MR_Next},
783 {"\"repetitions\": 3,$", MR_Next},
784 {"\"threads\": 1,$", MR_Next},
785 {"\"aggregate_name\": \"mean\",$", MR_Next},
786 {"\"aggregate_unit\": \"time\",$", MR_Next},
787 {"\"iterations\": 3,$", MR_Next},
788 {"\"time_unit\": \"us\",?$"},
789 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
790 {"\"family_index\": 21,$", MR_Next},
791 {"\"per_family_instance_index\": 0,$", MR_Next},
792 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
793 {"\"run_type\": \"aggregate\",$", MR_Next},
794 {"\"repetitions\": 3,$", MR_Next},
795 {"\"threads\": 1,$", MR_Next},
796 {"\"aggregate_name\": \"median\",$", MR_Next},
797 {"\"aggregate_unit\": \"time\",$", MR_Next},
798 {"\"iterations\": 3,$", MR_Next},
799 {"\"time_unit\": \"us\",?$"},
800 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
801 {"\"family_index\": 21,$", MR_Next},
802 {"\"per_family_instance_index\": 0,$", MR_Next},
803 {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
804 {"\"run_type\": \"aggregate\",$", MR_Next},
805 {"\"repetitions\": 3,$", MR_Next},
806 {"\"threads\": 1,$", MR_Next},
807 {"\"aggregate_name\": \"stddev\",$", MR_Next},
808 {"\"aggregate_unit\": \"time\",$", MR_Next},
809 {"\"iterations\": 3,$", MR_Next},
810 {"\"time_unit\": \"us\",?$"}});
811 ADD_CASES(TC_CSVOut,
812 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
813 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
814 {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
815 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
816
817 // ========================================================================= //
818 // -------------------- Testing user-provided statistics ------------------- //
819 // ========================================================================= //
820
__anon67cceb300102(const std::vector<double>& v) 821 const auto UserStatistics = [](const std::vector<double>& v) {
822 return v.back();
823 };
BM_UserStats(benchmark::State & state)824 void BM_UserStats(benchmark::State& state) {
825 for (auto _ : state) {
826 state.SetIterationTime(150 / 10e8);
827 }
828 }
829 // clang-format off
830 BENCHMARK(BM_UserStats)
831 ->Repetitions(3)
832 ->Iterations(5)
833 ->UseManualTime()
834 ->ComputeStatistics("", UserStatistics);
835 // clang-format on
836
837 // check that user-provided stats is calculated, and is after the default-ones
838 // empty string as name is intentional, it would sort before anything else
839 ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
840 "]* 150 ns %time [ ]*5$"},
841 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
842 "]* 150 ns %time [ ]*5$"},
843 {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
844 "]* 150 ns %time [ ]*5$"},
845 {"^BM_UserStats/iterations:5/repeats:3/"
846 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
847 {"^BM_UserStats/iterations:5/repeats:3/"
848 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
849 {"^BM_UserStats/iterations:5/repeats:3/"
850 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
851 {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
852 "[ ]* 150 ns %time [ ]*3$"}});
853 ADD_CASES(
854 TC_JSONOut,
855 {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
856 {"\"family_index\": 22,$", MR_Next},
857 {"\"per_family_instance_index\": 0,$", MR_Next},
858 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
859 MR_Next},
860 {"\"run_type\": \"iteration\",$", MR_Next},
861 {"\"repetitions\": 3,$", MR_Next},
862 {"\"repetition_index\": 0,$", MR_Next},
863 {"\"threads\": 1,$", MR_Next},
864 {"\"iterations\": 5,$", MR_Next},
865 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
866 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
867 {"\"family_index\": 22,$", MR_Next},
868 {"\"per_family_instance_index\": 0,$", MR_Next},
869 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
870 MR_Next},
871 {"\"run_type\": \"iteration\",$", MR_Next},
872 {"\"repetitions\": 3,$", MR_Next},
873 {"\"repetition_index\": 1,$", MR_Next},
874 {"\"threads\": 1,$", MR_Next},
875 {"\"iterations\": 5,$", MR_Next},
876 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
877 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
878 {"\"family_index\": 22,$", MR_Next},
879 {"\"per_family_instance_index\": 0,$", MR_Next},
880 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
881 MR_Next},
882 {"\"run_type\": \"iteration\",$", MR_Next},
883 {"\"repetitions\": 3,$", MR_Next},
884 {"\"repetition_index\": 2,$", MR_Next},
885 {"\"threads\": 1,$", MR_Next},
886 {"\"iterations\": 5,$", MR_Next},
887 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
888 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
889 {"\"family_index\": 22,$", MR_Next},
890 {"\"per_family_instance_index\": 0,$", MR_Next},
891 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
892 MR_Next},
893 {"\"run_type\": \"aggregate\",$", MR_Next},
894 {"\"repetitions\": 3,$", MR_Next},
895 {"\"threads\": 1,$", MR_Next},
896 {"\"aggregate_name\": \"mean\",$", MR_Next},
897 {"\"aggregate_unit\": \"time\",$", MR_Next},
898 {"\"iterations\": 3,$", MR_Next},
899 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
900 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
901 {"\"family_index\": 22,$", MR_Next},
902 {"\"per_family_instance_index\": 0,$", MR_Next},
903 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
904 MR_Next},
905 {"\"run_type\": \"aggregate\",$", MR_Next},
906 {"\"repetitions\": 3,$", MR_Next},
907 {"\"threads\": 1,$", MR_Next},
908 {"\"aggregate_name\": \"median\",$", MR_Next},
909 {"\"aggregate_unit\": \"time\",$", MR_Next},
910 {"\"iterations\": 3,$", MR_Next},
911 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
912 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
913 {"\"family_index\": 22,$", MR_Next},
914 {"\"per_family_instance_index\": 0,$", MR_Next},
915 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
916 MR_Next},
917 {"\"run_type\": \"aggregate\",$", MR_Next},
918 {"\"repetitions\": 3,$", MR_Next},
919 {"\"threads\": 1,$", MR_Next},
920 {"\"aggregate_name\": \"stddev\",$", MR_Next},
921 {"\"aggregate_unit\": \"time\",$", MR_Next},
922 {"\"iterations\": 3,$", MR_Next},
923 {"\"real_time\": %float,$", MR_Next},
924 {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
925 {"\"family_index\": 22,$", MR_Next},
926 {"\"per_family_instance_index\": 0,$", MR_Next},
927 {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
928 MR_Next},
929 {"\"run_type\": \"aggregate\",$", MR_Next},
930 {"\"repetitions\": 3,$", MR_Next},
931 {"\"threads\": 1,$", MR_Next},
932 {"\"aggregate_name\": \"\",$", MR_Next},
933 {"\"aggregate_unit\": \"time\",$", MR_Next},
934 {"\"iterations\": 3,$", MR_Next},
935 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
936 ADD_CASES(
937 TC_CSVOut,
938 {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
939 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
940 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
941 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
942 {"^\"BM_UserStats/iterations:5/repeats:3/"
943 "manual_time_median\",%csv_report$"},
944 {"^\"BM_UserStats/iterations:5/repeats:3/"
945 "manual_time_stddev\",%csv_report$"},
946 {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
947
948 // ========================================================================= //
949 // ------------- Testing relative standard deviation statistics ------------ //
950 // ========================================================================= //
951
__anon67cceb300202(const std::vector<double>&) 952 const auto UserPercentStatistics = [](const std::vector<double>&) {
953 return 1. / 100.;
954 };
BM_UserPercentStats(benchmark::State & state)955 void BM_UserPercentStats(benchmark::State& state) {
956 for (auto _ : state) {
957 state.SetIterationTime(150 / 10e8);
958 }
959 }
960 // clang-format off
961 BENCHMARK(BM_UserPercentStats)
962 ->Repetitions(3)
963 ->Iterations(5)
964 ->UseManualTime()
965 ->Unit(benchmark::TimeUnit::kNanosecond)
966 ->ComputeStatistics("", UserPercentStatistics, benchmark::StatisticUnit::kPercentage);
967 // clang-format on
968
969 // check that UserPercent-provided stats is calculated, and is after the
970 // default-ones empty string as name is intentional, it would sort before
971 // anything else
972 ADD_CASES(TC_ConsoleOut,
973 {{"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
974 "]* 150 ns %time [ ]*5$"},
975 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
976 "]* 150 ns %time [ ]*5$"},
977 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time [ "
978 "]* 150 ns %time [ ]*5$"},
979 {"^BM_UserPercentStats/iterations:5/repeats:3/"
980 "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
981 {"^BM_UserPercentStats/iterations:5/repeats:3/"
982 "manual_time_median [ ]* 150 ns %time [ ]*3$"},
983 {"^BM_UserPercentStats/iterations:5/repeats:3/"
984 "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
985 {"^BM_UserPercentStats/iterations:5/repeats:3/manual_time_ "
986 "[ ]* 1.00 % [ ]* 1.00 %[ ]*3$"}});
987 ADD_CASES(
988 TC_JSONOut,
989 {{"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
990 {"\"family_index\": 23,$", MR_Next},
991 {"\"per_family_instance_index\": 0,$", MR_Next},
992 {"\"run_name\": "
993 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
994 MR_Next},
995 {"\"run_type\": \"iteration\",$", MR_Next},
996 {"\"repetitions\": 3,$", MR_Next},
997 {"\"repetition_index\": 0,$", MR_Next},
998 {"\"threads\": 1,$", MR_Next},
999 {"\"iterations\": 5,$", MR_Next},
1000 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1001 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1002 {"\"family_index\": 23,$", MR_Next},
1003 {"\"per_family_instance_index\": 0,$", MR_Next},
1004 {"\"run_name\": "
1005 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1006 MR_Next},
1007 {"\"run_type\": \"iteration\",$", MR_Next},
1008 {"\"repetitions\": 3,$", MR_Next},
1009 {"\"repetition_index\": 1,$", MR_Next},
1010 {"\"threads\": 1,$", MR_Next},
1011 {"\"iterations\": 5,$", MR_Next},
1012 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1013 {"\"name\": \"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$"},
1014 {"\"family_index\": 23,$", MR_Next},
1015 {"\"per_family_instance_index\": 0,$", MR_Next},
1016 {"\"run_name\": "
1017 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1018 MR_Next},
1019 {"\"run_type\": \"iteration\",$", MR_Next},
1020 {"\"repetitions\": 3,$", MR_Next},
1021 {"\"repetition_index\": 2,$", MR_Next},
1022 {"\"threads\": 1,$", MR_Next},
1023 {"\"iterations\": 5,$", MR_Next},
1024 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1025 {"\"name\": "
1026 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_mean\",$"},
1027 {"\"family_index\": 23,$", MR_Next},
1028 {"\"per_family_instance_index\": 0,$", MR_Next},
1029 {"\"run_name\": "
1030 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1031 MR_Next},
1032 {"\"run_type\": \"aggregate\",$", MR_Next},
1033 {"\"repetitions\": 3,$", MR_Next},
1034 {"\"threads\": 1,$", MR_Next},
1035 {"\"aggregate_name\": \"mean\",$", MR_Next},
1036 {"\"aggregate_unit\": \"time\",$", MR_Next},
1037 {"\"iterations\": 3,$", MR_Next},
1038 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1039 {"\"name\": "
1040 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_median\",$"},
1041 {"\"family_index\": 23,$", MR_Next},
1042 {"\"per_family_instance_index\": 0,$", MR_Next},
1043 {"\"run_name\": "
1044 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1045 MR_Next},
1046 {"\"run_type\": \"aggregate\",$", MR_Next},
1047 {"\"repetitions\": 3,$", MR_Next},
1048 {"\"threads\": 1,$", MR_Next},
1049 {"\"aggregate_name\": \"median\",$", MR_Next},
1050 {"\"aggregate_unit\": \"time\",$", MR_Next},
1051 {"\"iterations\": 3,$", MR_Next},
1052 {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
1053 {"\"name\": "
1054 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_stddev\",$"},
1055 {"\"family_index\": 23,$", MR_Next},
1056 {"\"per_family_instance_index\": 0,$", MR_Next},
1057 {"\"run_name\": "
1058 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1059 MR_Next},
1060 {"\"run_type\": \"aggregate\",$", MR_Next},
1061 {"\"repetitions\": 3,$", MR_Next},
1062 {"\"threads\": 1,$", MR_Next},
1063 {"\"aggregate_name\": \"stddev\",$", MR_Next},
1064 {"\"aggregate_unit\": \"time\",$", MR_Next},
1065 {"\"iterations\": 3,$", MR_Next},
1066 {"\"real_time\": %float,$", MR_Next},
1067 {"\"name\": "
1068 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time_\",$"},
1069 {"\"family_index\": 23,$", MR_Next},
1070 {"\"per_family_instance_index\": 0,$", MR_Next},
1071 {"\"run_name\": "
1072 "\"BM_UserPercentStats/iterations:5/repeats:3/manual_time\",$",
1073 MR_Next},
1074 {"\"run_type\": \"aggregate\",$", MR_Next},
1075 {"\"repetitions\": 3,$", MR_Next},
1076 {"\"threads\": 1,$", MR_Next},
1077 {"\"aggregate_name\": \"\",$", MR_Next},
1078 {"\"aggregate_unit\": \"percentage\",$", MR_Next},
1079 {"\"iterations\": 3,$", MR_Next},
1080 {"\"real_time\": 1\\.(0)*e-(0)*2,$", MR_Next}});
1081 ADD_CASES(TC_CSVOut, {{"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1082 "manual_time\",%csv_report$"},
1083 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1084 "manual_time\",%csv_report$"},
1085 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1086 "manual_time\",%csv_report$"},
1087 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1088 "manual_time_mean\",%csv_report$"},
1089 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1090 "manual_time_median\",%csv_report$"},
1091 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1092 "manual_time_stddev\",%csv_report$"},
1093 {"^\"BM_UserPercentStats/iterations:5/repeats:3/"
1094 "manual_time_\",%csv_cv_report$"}});
1095
1096 // ========================================================================= //
1097 // ------------------------- Testing StrEscape JSON ------------------------ //
1098 // ========================================================================= //
1099 #if 0 // enable when csv testing code correctly handles multi-line fields
1100 void BM_JSON_Format(benchmark::State& state) {
1101 state.SkipWithError("val\b\f\n\r\t\\\"with\"es,capes");
1102 for (auto _ : state) {
1103 }
1104 }
1105 BENCHMARK(BM_JSON_Format);
1106 ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_JSON_Format\",$"},
1107 {"\"family_index\": 23,$", MR_Next},
1108 {"\"per_family_instance_index\": 0,$", MR_Next},
1109 {"\"run_name\": \"BM_JSON_Format\",$", MR_Next},
1110 {"\"run_type\": \"iteration\",$", MR_Next},
1111 {"\"repetitions\": 1,$", MR_Next},
1112 {"\"repetition_index\": 0,$", MR_Next},
1113 {"\"threads\": 1,$", MR_Next},
1114 {"\"error_occurred\": true,$", MR_Next},
1115 {R"("error_message": "val\\b\\f\\n\\r\\t\\\\\\"with\\"es,capes",$)", MR_Next}});
1116 #endif
1117 // ========================================================================= //
1118 // -------------------------- Testing CsvEscape ---------------------------- //
1119 // ========================================================================= //
1120
BM_CSV_Format(benchmark::State & state)1121 void BM_CSV_Format(benchmark::State& state) {
1122 state.SkipWithError("\"freedom\"");
1123 for (auto _ : state) {
1124 }
1125 }
1126 BENCHMARK(BM_CSV_Format);
1127 ADD_CASES(TC_CSVOut, {{"^\"BM_CSV_Format\",,,,,,,,true,\"\"\"freedom\"\"\"$"}});
1128
1129 // ========================================================================= //
1130 // --------------------------- TEST CASES END ------------------------------ //
1131 // ========================================================================= //
1132
main(int argc,char * argv[])1133 int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
1134