1 use criterion;
2 use serde_json;
3 
4 #[cfg(feature = "plotters")]
5 use criterion::SamplingMode;
6 use criterion::{
7     criterion_group, criterion_main, profiler::Profiler, BatchSize, BenchmarkId, Criterion,
8 };
9 use serde_json::value::Value;
10 use std::cell::{Cell, RefCell};
11 use std::cmp::max;
12 use std::fs::File;
13 use std::path::{Path, PathBuf};
14 use std::rc::Rc;
15 use std::time::{Duration, SystemTime};
16 use tempfile::{tempdir, TempDir};
17 use walkdir::WalkDir;
18 
19 /*
20  * Please note that these tests are not complete examples of how to use
21  * Criterion.rs. See the benches folder for actual examples.
22  */
temp_dir() -> TempDir23 fn temp_dir() -> TempDir {
24     tempdir().unwrap()
25 }
26 
27 // Configure a Criterion struct to perform really fast benchmarks. This is not
28 // recommended for real benchmarking, only for testing.
short_benchmark(dir: &TempDir) -> Criterion29 fn short_benchmark(dir: &TempDir) -> Criterion {
30     Criterion::default()
31         .output_directory(dir.path())
32         .warm_up_time(Duration::from_millis(250))
33         .measurement_time(Duration::from_millis(500))
34         .nresamples(2000)
35 }
36 
37 #[derive(Clone)]
38 struct Counter {
39     counter: Rc<RefCell<usize>>,
40 }
41 impl Counter {
count(&self)42     fn count(&self) {
43         *(*self.counter).borrow_mut() += 1;
44     }
45 
read(&self) -> usize46     fn read(&self) -> usize {
47         *(*self.counter).borrow()
48     }
49 }
50 impl Default for Counter {
default() -> Counter51     fn default() -> Counter {
52         Counter {
53             counter: Rc::new(RefCell::new(0)),
54         }
55     }
56 }
57 
verify_file(dir: &PathBuf, path: &str) -> PathBuf58 fn verify_file(dir: &PathBuf, path: &str) -> PathBuf {
59     let full_path = dir.join(path);
60     assert!(
61         full_path.is_file(),
62         "File {:?} does not exist or is not a file",
63         full_path
64     );
65     let metadata = full_path.metadata().unwrap();
66     assert!(metadata.len() > 0);
67     full_path
68 }
69 
verify_json(dir: &PathBuf, path: &str)70 fn verify_json(dir: &PathBuf, path: &str) {
71     let full_path = verify_file(dir, path);
72     let f = File::open(full_path).unwrap();
73     serde_json::from_reader::<File, Value>(f).unwrap();
74 }
75 
76 #[cfg(feature = "html_reports")]
verify_svg(dir: &PathBuf, path: &str)77 fn verify_svg(dir: &PathBuf, path: &str) {
78     verify_file(dir, path);
79 }
80 
81 #[cfg(feature = "html_reports")]
verify_html(dir: &PathBuf, path: &str)82 fn verify_html(dir: &PathBuf, path: &str) {
83     verify_file(dir, path);
84 }
85 
verify_stats(dir: &PathBuf, baseline: &str)86 fn verify_stats(dir: &PathBuf, baseline: &str) {
87     verify_json(&dir, &format!("{}/estimates.json", baseline));
88     verify_json(&dir, &format!("{}/sample.json", baseline));
89     verify_json(&dir, &format!("{}/tukey.json", baseline));
90     verify_json(&dir, &format!("{}/benchmark.json", baseline));
91     #[cfg(feature = "csv_output")]
92     verify_file(&dir, &format!("{}/raw.csv", baseline));
93 }
94 
verify_not_exists(dir: &PathBuf, path: &str)95 fn verify_not_exists(dir: &PathBuf, path: &str) {
96     assert!(!dir.join(path).exists());
97 }
98 
latest_modified(dir: &PathBuf) -> SystemTime99 fn latest_modified(dir: &PathBuf) -> SystemTime {
100     let mut newest_update: Option<SystemTime> = None;
101     for entry in WalkDir::new(dir) {
102         let entry = entry.unwrap();
103         let modified = entry.metadata().unwrap().modified().unwrap();
104         newest_update = match newest_update {
105             Some(latest) => Some(max(latest, modified)),
106             None => Some(modified),
107         };
108     }
109 
110     newest_update.expect("failed to find a single time in directory")
111 }
112 
113 #[test]
test_creates_directory()114 fn test_creates_directory() {
115     let dir = temp_dir();
116     short_benchmark(&dir).bench_function("test_creates_directory", |b| b.iter(|| 10));
117     assert!(dir.path().join("test_creates_directory").is_dir());
118 }
119 
120 #[test]
test_without_plots()121 fn test_without_plots() {
122     let dir = temp_dir();
123     short_benchmark(&dir)
124         .without_plots()
125         .bench_function("test_without_plots", |b| b.iter(|| 10));
126 
127     for entry in WalkDir::new(dir.path().join("test_without_plots")) {
128         let entry = entry.ok();
129         let is_svg = entry
130             .as_ref()
131             .and_then(|entry| entry.path().extension())
132             .and_then(|ext| ext.to_str())
133             .map(|ext| ext == "svg")
134             .unwrap_or(false);
135         assert!(
136             !is_svg,
137             "Found SVG file ({:?}) in output directory with plots disabled",
138             entry.unwrap().file_name()
139         );
140     }
141 }
142 
143 #[test]
test_save_baseline()144 fn test_save_baseline() {
145     let dir = temp_dir();
146     println!("tmp directory is {:?}", dir.path());
147     short_benchmark(&dir)
148         .save_baseline("some-baseline".to_owned())
149         .bench_function("test_save_baseline", |b| b.iter(|| 10));
150 
151     let dir = dir.path().join("test_save_baseline");
152     verify_stats(&dir, "some-baseline");
153 
154     verify_not_exists(&dir, "base");
155 }
156 
157 #[test]
test_retain_baseline()158 fn test_retain_baseline() {
159     // Initial benchmark to populate
160     let dir = temp_dir();
161     short_benchmark(&dir)
162         .save_baseline("some-baseline".to_owned())
163         .bench_function("test_retain_baseline", |b| b.iter(|| 10));
164 
165     let pre_modified = latest_modified(&dir.path().join("test_retain_baseline/some-baseline"));
166 
167     short_benchmark(&dir)
168         .retain_baseline("some-baseline".to_owned(), true)
169         .bench_function("test_retain_baseline", |b| b.iter(|| 10));
170 
171     let post_modified = latest_modified(&dir.path().join("test_retain_baseline/some-baseline"));
172 
173     assert_eq!(pre_modified, post_modified, "baseline modified by retain");
174 }
175 
176 #[test]
177 #[should_panic(expected = "Baseline 'some-baseline' must exist before comparison is allowed")]
test_compare_baseline_strict_panics_when_missing_baseline()178 fn test_compare_baseline_strict_panics_when_missing_baseline() {
179     let dir = temp_dir();
180     short_benchmark(&dir)
181         .retain_baseline("some-baseline".to_owned(), true)
182         .bench_function("test_compare_baseline", |b| b.iter(|| 10));
183 }
184 
185 #[test]
test_compare_baseline_lenient_when_missing_baseline()186 fn test_compare_baseline_lenient_when_missing_baseline() {
187     let dir = temp_dir();
188     short_benchmark(&dir)
189         .retain_baseline("some-baseline".to_owned(), false)
190         .bench_function("test_compare_baseline", |b| b.iter(|| 10));
191 }
192 
193 #[test]
test_sample_size()194 fn test_sample_size() {
195     let dir = temp_dir();
196     let counter = Counter::default();
197 
198     let clone = counter.clone();
199     short_benchmark(&dir)
200         .sample_size(50)
201         .bench_function("test_sample_size", move |b| {
202             clone.count();
203             b.iter(|| 10)
204         });
205 
206     // This function will be called more than sample_size times because of the
207     // warmup.
208     assert!(counter.read() > 50);
209 }
210 
211 #[test]
test_warmup_time()212 fn test_warmup_time() {
213     let dir = temp_dir();
214     let counter1 = Counter::default();
215 
216     let clone = counter1.clone();
217     short_benchmark(&dir)
218         .warm_up_time(Duration::from_millis(100))
219         .bench_function("test_warmup_time_1", move |b| {
220             clone.count();
221             b.iter(|| 10)
222         });
223 
224     let counter2 = Counter::default();
225     let clone = counter2.clone();
226     short_benchmark(&dir)
227         .warm_up_time(Duration::from_millis(2000))
228         .bench_function("test_warmup_time_2", move |b| {
229             clone.count();
230             b.iter(|| 10)
231         });
232 
233     assert!(counter1.read() < counter2.read());
234 }
235 
236 #[test]
test_measurement_time()237 fn test_measurement_time() {
238     let dir = temp_dir();
239     let counter1 = Counter::default();
240 
241     let clone = counter1.clone();
242     short_benchmark(&dir)
243         .measurement_time(Duration::from_millis(100))
244         .bench_function("test_meas_time_1", move |b| b.iter(|| clone.count()));
245 
246     let counter2 = Counter::default();
247     let clone = counter2.clone();
248     short_benchmark(&dir)
249         .measurement_time(Duration::from_millis(2000))
250         .bench_function("test_meas_time_2", move |b| b.iter(|| clone.count()));
251 
252     assert!(counter1.read() < counter2.read());
253 }
254 
255 #[test]
test_bench_function()256 fn test_bench_function() {
257     let dir = temp_dir();
258     short_benchmark(&dir).bench_function("test_bench_function", move |b| b.iter(|| 10));
259 }
260 
261 #[test]
test_filtering()262 fn test_filtering() {
263     let dir = temp_dir();
264     let counter = Counter::default();
265     let clone = counter.clone();
266 
267     short_benchmark(&dir)
268         .with_filter("Foo")
269         .bench_function("test_filtering", move |b| b.iter(|| clone.count()));
270 
271     assert_eq!(counter.read(), 0);
272     assert!(!dir.path().join("test_filtering").is_dir());
273 }
274 
275 #[test]
test_timing_loops()276 fn test_timing_loops() {
277     let dir = temp_dir();
278     let mut c = short_benchmark(&dir);
279     let mut group = c.benchmark_group("test_timing_loops");
280     group.bench_function("iter_with_setup", |b| {
281         b.iter_with_setup(|| vec![10], |v| v[0])
282     });
283     group.bench_function("iter_with_large_setup", |b| {
284         b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumBatches(1))
285     });
286     group.bench_function("iter_with_large_drop", |b| {
287         b.iter_with_large_drop(|| vec![10; 100])
288     });
289     group.bench_function("iter_batched_small", |b| {
290         b.iter_batched(|| vec![10], |v| v[0], BatchSize::SmallInput)
291     });
292     group.bench_function("iter_batched_large", |b| {
293         b.iter_batched(|| vec![10], |v| v[0], BatchSize::LargeInput)
294     });
295     group.bench_function("iter_batched_per_iteration", |b| {
296         b.iter_batched(|| vec![10], |v| v[0], BatchSize::PerIteration)
297     });
298     group.bench_function("iter_batched_one_batch", |b| {
299         b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumBatches(1))
300     });
301     group.bench_function("iter_batched_10_iterations", |b| {
302         b.iter_batched(|| vec![10], |v| v[0], BatchSize::NumIterations(10))
303     });
304     group.bench_function("iter_batched_ref_small", |b| {
305         b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::SmallInput)
306     });
307     group.bench_function("iter_batched_ref_large", |b| {
308         b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::LargeInput)
309     });
310     group.bench_function("iter_batched_ref_per_iteration", |b| {
311         b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::PerIteration)
312     });
313     group.bench_function("iter_batched_ref_one_batch", |b| {
314         b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumBatches(1))
315     });
316     group.bench_function("iter_batched_ref_10_iterations", |b| {
317         b.iter_batched_ref(|| vec![10], |v| v[0], BatchSize::NumIterations(10))
318     });
319 }
320 
321 // Verify that all expected output files are present
322 #[cfg(feature = "plotters")]
323 #[test]
test_output_files()324 fn test_output_files() {
325     let tempdir = temp_dir();
326     // Run benchmarks twice to produce comparisons
327     for _ in 0..2 {
328         let mut c = short_benchmark(&tempdir);
329         let mut group = c.benchmark_group("test_output");
330         group.sampling_mode(SamplingMode::Linear);
331         group.bench_function("output_1", |b| b.iter(|| 10));
332         group.bench_function("output_2", |b| b.iter(|| 20));
333         group.bench_function("output_\\/*\"?", |b| b.iter(|| 30));
334     }
335 
336     // For each benchmark, assert that the expected files are present.
337     for x in 0..3 {
338         let dir = if x == 2 {
339             // Check that certain special characters are replaced with underscores
340             tempdir.path().join("test_output/output______")
341         } else {
342             tempdir.path().join(format!("test_output/output_{}", x + 1))
343         };
344 
345         verify_stats(&dir, "new");
346         verify_stats(&dir, "base");
347         verify_json(&dir, "change/estimates.json");
348 
349         #[cfg(feature = "html_reports")]
350         {
351             verify_svg(&dir, "report/MAD.svg");
352             verify_svg(&dir, "report/mean.svg");
353             verify_svg(&dir, "report/median.svg");
354             verify_svg(&dir, "report/pdf.svg");
355             verify_svg(&dir, "report/regression.svg");
356             verify_svg(&dir, "report/SD.svg");
357             verify_svg(&dir, "report/slope.svg");
358             verify_svg(&dir, "report/typical.svg");
359             verify_svg(&dir, "report/both/pdf.svg");
360             verify_svg(&dir, "report/both/regression.svg");
361             verify_svg(&dir, "report/change/mean.svg");
362             verify_svg(&dir, "report/change/median.svg");
363             verify_svg(&dir, "report/change/t-test.svg");
364 
365             verify_svg(&dir, "report/pdf_small.svg");
366             verify_svg(&dir, "report/regression_small.svg");
367             verify_svg(&dir, "report/relative_pdf_small.svg");
368             verify_svg(&dir, "report/relative_regression_small.svg");
369             verify_html(&dir, "report/index.html");
370         }
371     }
372 
373     #[cfg(feature = "html_reports")]
374     {
375         // Check for overall report files
376         let dir = tempdir.path().join("test_output");
377 
378         verify_svg(&dir, "report/violin.svg");
379         verify_html(&dir, "report/index.html");
380     }
381 
382     // Run the final summary process and check for the report that produces
383     short_benchmark(&tempdir).final_summary();
384 
385     #[cfg(feature = "html_reports")]
386     {
387         let dir = tempdir.path().to_owned();
388         verify_html(&dir, "report/index.html");
389     }
390 }
391 
392 #[cfg(feature = "plotters")]
393 #[test]
test_output_files_flat_sampling()394 fn test_output_files_flat_sampling() {
395     let tempdir = temp_dir();
396     // Run benchmark twice to produce comparisons
397     for _ in 0..2 {
398         let mut c = short_benchmark(&tempdir);
399         let mut group = c.benchmark_group("test_output");
400         group.sampling_mode(SamplingMode::Flat);
401         group.bench_function("output_flat", |b| b.iter(|| 10));
402     }
403 
404     let dir = tempdir.path().join("test_output/output_flat");
405 
406     verify_stats(&dir, "new");
407     verify_stats(&dir, "base");
408     verify_json(&dir, "change/estimates.json");
409 
410     #[cfg(feature = "html_reports")]
411     {
412         verify_svg(&dir, "report/MAD.svg");
413         verify_svg(&dir, "report/mean.svg");
414         verify_svg(&dir, "report/median.svg");
415         verify_svg(&dir, "report/pdf.svg");
416         verify_svg(&dir, "report/iteration_times.svg");
417         verify_svg(&dir, "report/SD.svg");
418         verify_svg(&dir, "report/typical.svg");
419         verify_svg(&dir, "report/both/pdf.svg");
420         verify_svg(&dir, "report/both/iteration_times.svg");
421         verify_svg(&dir, "report/change/mean.svg");
422         verify_svg(&dir, "report/change/median.svg");
423         verify_svg(&dir, "report/change/t-test.svg");
424 
425         verify_svg(&dir, "report/pdf_small.svg");
426         verify_svg(&dir, "report/iteration_times_small.svg");
427         verify_svg(&dir, "report/relative_pdf_small.svg");
428         verify_svg(&dir, "report/relative_iteration_times_small.svg");
429         verify_html(&dir, "report/index.html");
430     }
431 }
432 
433 #[test]
434 #[should_panic(expected = "Benchmark function must call Bencher::iter or related method.")]
test_bench_with_no_iteration_panics()435 fn test_bench_with_no_iteration_panics() {
436     let dir = temp_dir();
437     short_benchmark(&dir).bench_function("no_iter", |_b| {});
438 }
439 
440 #[test]
test_benchmark_group_with_input()441 fn test_benchmark_group_with_input() {
442     let dir = temp_dir();
443     let mut c = short_benchmark(&dir);
444     let mut group = c.benchmark_group("Test Group");
445     for x in 0..2 {
446         group.bench_with_input(BenchmarkId::new("Test 1", x), &x, |b, i| b.iter(|| i));
447         group.bench_with_input(BenchmarkId::new("Test 2", x), &x, |b, i| b.iter(|| i));
448     }
449     group.finish();
450 }
451 
452 #[test]
test_benchmark_group_without_input()453 fn test_benchmark_group_without_input() {
454     let dir = temp_dir();
455     let mut c = short_benchmark(&dir);
456     let mut group = c.benchmark_group("Test Group 2");
457     group.bench_function("Test 1", |b| b.iter(|| 30));
458     group.bench_function("Test 2", |b| b.iter(|| 20));
459     group.finish();
460 }
461 
462 #[test]
test_criterion_doesnt_panic_if_measured_time_is_zero()463 fn test_criterion_doesnt_panic_if_measured_time_is_zero() {
464     let dir = temp_dir();
465     let mut c = short_benchmark(&dir);
466     c.bench_function("zero_time", |bencher| {
467         bencher.iter_custom(|_iters| Duration::new(0, 0))
468     });
469 }
470 
471 mod macros {
472     use super::{criterion, criterion_group, criterion_main};
473 
474     #[test]
475     #[should_panic(expected = "group executed")]
criterion_main()476     fn criterion_main() {
477         fn group() {}
478         fn group2() {
479             panic!("group executed");
480         }
481 
482         criterion_main!(group, group2);
483 
484         main();
485     }
486 
487     #[test]
criterion_main_trailing_comma()488     fn criterion_main_trailing_comma() {
489         // make this a compile-only check
490         // as the second logger initialization causes panic
491         #[allow(dead_code)]
492         fn group() {}
493         #[allow(dead_code)]
494         fn group2() {}
495 
496         criterion_main!(group, group2,);
497 
498         // silence dead_code warning
499         if false {
500             main()
501         }
502     }
503 
504     #[test]
505     #[should_panic(expected = "group executed")]
criterion_group()506     fn criterion_group() {
507         use self::criterion::Criterion;
508 
509         fn group(_crit: &mut Criterion) {}
510         fn group2(_crit: &mut Criterion) {
511             panic!("group executed");
512         }
513 
514         criterion_group!(test_group, group, group2);
515 
516         test_group();
517     }
518 
519     #[test]
520     #[should_panic(expected = "group executed")]
criterion_group_trailing_comma()521     fn criterion_group_trailing_comma() {
522         use self::criterion::Criterion;
523 
524         fn group(_crit: &mut Criterion) {}
525         fn group2(_crit: &mut Criterion) {
526             panic!("group executed");
527         }
528 
529         criterion_group!(test_group, group, group2,);
530 
531         test_group();
532     }
533 }
534 
535 struct TestProfiler {
536     started: Rc<Cell<u32>>,
537     stopped: Rc<Cell<u32>>,
538 }
539 impl Profiler for TestProfiler {
start_profiling(&mut self, benchmark_id: &str, _benchmark_path: &Path)540     fn start_profiling(&mut self, benchmark_id: &str, _benchmark_path: &Path) {
541         assert!(benchmark_id.contains("profile_test"));
542         self.started.set(self.started.get() + 1);
543     }
stop_profiling(&mut self, benchmark_id: &str, _benchmark_path: &Path)544     fn stop_profiling(&mut self, benchmark_id: &str, _benchmark_path: &Path) {
545         assert!(benchmark_id.contains("profile_test"));
546         self.stopped.set(self.stopped.get() + 1);
547     }
548 }
549 
550 // Verify that profilers are started and stopped as expected
551 #[test]
test_profiler_called()552 fn test_profiler_called() {
553     let started = Rc::new(Cell::new(0u32));
554     let stopped = Rc::new(Cell::new(0u32));
555     let profiler = TestProfiler {
556         started: started.clone(),
557         stopped: stopped.clone(),
558     };
559     let dir = temp_dir();
560     let mut criterion = short_benchmark(&dir)
561         .with_profiler(profiler)
562         .profile_time(Some(Duration::from_secs(1)));
563     criterion.bench_function("profile_test", |b| b.iter(|| 10));
564     assert_eq!(1, started.get());
565     assert_eq!(1, stopped.get());
566 }
567