mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-01 08:50:19 +00:00
Use inline snapshots for execution logs and benchmark reports
This commit is contained in:
David Peter committed
1 parent
258af076cb
commit
1e2e0be25c
2 files changed
+517
-343
No files matched your search
+213
-62
@@ -15,6 +15,16 @@ pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
cmd
|
||||
}
|
||||
|
||||
// Ignore trailing padding on report lines, including the space-only separators.
|
||||
// Keep indentation and internal spacing so snapshots still check the layout.
|
||||
fn stdout(output: &assert_cmd::assert::Assert) -> String {
|
||||
String::from_utf8_lossy(&output.get_output().stdout)
|
||||
.lines()
|
||||
.map(str::trim_end)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runs_successfully() {
|
||||
hyperfine()
|
||||
@@ -594,21 +604,38 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_relative_times() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times faster")
|
||||
.and(predicate::str::contains("3.00 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1.0 ran
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
3.00 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_same_time() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--command-name=A")
|
||||
.arg("--command-name=B")
|
||||
.arg("sleep 1.0")
|
||||
@@ -616,43 +643,98 @@ fn shows_benchmark_comparison_with_same_time() {
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 1000.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("As fast (1.00 ± 0.00) as")
|
||||
.and(predicate::str::contains("2.00 ± 0.00 times faster"))
|
||||
.and(predicate::str::contains("1000.00 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: A
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: B
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 1000.0
|
||||
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1000.000 s … 1000.000 s 10 runs
|
||||
|
||||
Summary
|
||||
A ran
|
||||
As fast (1.00 ± 0.00) as B
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
1000.00 ± 0.00 times faster than sleep 1000.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_relative_to_reference() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times slower")
|
||||
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 2.0 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: refabc123
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
refabc123 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
.arg("30")
|
||||
@@ -661,19 +743,37 @@ fn performs_all_benchmarks_in_parameter_scan() {
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 30")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 50").not()),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 30 ran
|
||||
1.17 ± 0.00 times faster than sleep 35
|
||||
1.33 ± 0.00 times faster than sleep 40
|
||||
1.50 ± 0.00 times faster than sleep 45
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 25")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
@@ -683,31 +783,56 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 25")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 30"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 25
|
||||
Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 25.000 s … 25.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 5: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 25 ran
|
||||
1.20 ± 0.00 times faster than sleep 30
|
||||
1.40 ± 0.00 times faster than sleep 35
|
||||
1.60 ± 0.00 times faster than sleep 40
|
||||
1.80 ± 0.00 times faster than sleep 45
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_results_are_not_exported_to_stdout() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=none") // To only see the Markdown export on stdout
|
||||
.arg("--export-markdown")
|
||||
.arg("-")
|
||||
.arg("sleep 1")
|
||||
.arg("sleep 2")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
(predicate::str::contains("sleep 1").count(1))
|
||||
.and(predicate::str::contains("sleep 2").count(1)),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||||
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -749,27 +874,53 @@ fn unused_parameters_are_shown_in_benchmark_name() {
|
||||
|
||||
#[test]
|
||||
fn speed_comparison_sort_order() {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
hyperfine_debug()
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}"))
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2",
|
||||
));
|
||||
insta::allow_duplicates! {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}"))
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1 ran
|
||||
2.00 ± 0.00 times faster than sleep 2
|
||||
");
|
||||
}
|
||||
}
|
||||
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg("--sort=command")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"2.00 ± 0.00 sleep 2\n 1.00 sleep 1",
|
||||
));
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Relative speed comparison
|
||||
2.00 ± 0.00 sleep 2
|
||||
1.00 sleep 1
|
||||
");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
|
||||
Reference in new issue
Block a user