mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-01 17:00:18 +00:00
Alternative performance metrics in the terminal (#986)
This commit is contained in:
David Peter authored and GitHub committed
1 parent
3b64ea1bf5
commit
bdbfae9907
43 files changed
+1872
-2056
No files matched your search
+179
-249
@@ -111,13 +111,23 @@ fn fails_with_zero_runs() {
|
||||
|
||||
#[test]
|
||||
fn min_runs_of_one_still_performs_one_run() {
|
||||
hyperfine_debug()
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--metrics=time_wall_clock")
|
||||
.arg("--min-runs=1")
|
||||
.arg("--warmup=0")
|
||||
.arg("sleep 4")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Time (abs ≡)"));
|
||||
.arg("sleep 4"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 4 (1 run)
|
||||
Value
|
||||
Wall Time 4.000 s
|
||||
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -159,7 +169,7 @@ fn fails_with_wrong_number_of_prepare_options() {
|
||||
.arg("--prepare=echo ref")
|
||||
.arg("--prepare=echo a")
|
||||
.arg("--prepare=echo b")
|
||||
.arg("--reference=echo ref")
|
||||
.arg("echo ref")
|
||||
.arg("echo a")
|
||||
.arg("echo b")
|
||||
.assert()
|
||||
@@ -182,7 +192,7 @@ fn fails_with_wrong_number_of_prepare_options() {
|
||||
.arg("--runs=1")
|
||||
.arg("--prepare=echo a")
|
||||
.arg("--prepare=echo b")
|
||||
.arg("--reference=echo ref")
|
||||
.arg("echo ref")
|
||||
.arg("echo a")
|
||||
.arg("echo b")
|
||||
.assert()
|
||||
@@ -208,7 +218,7 @@ fn fails_with_wrong_number_of_conclude_options() {
|
||||
.arg("--conclude=echo ref")
|
||||
.arg("--conclude=echo a")
|
||||
.arg("--conclude=echo b")
|
||||
.arg("--reference=echo ref")
|
||||
.arg("echo ref")
|
||||
.arg("echo a")
|
||||
.arg("echo b")
|
||||
.assert()
|
||||
@@ -231,7 +241,7 @@ fn fails_with_wrong_number_of_conclude_options() {
|
||||
.arg("--runs=1")
|
||||
.arg("--conclude=echo a")
|
||||
.arg("--conclude=echo b")
|
||||
.arg("--reference=echo ref")
|
||||
.arg("echo ref")
|
||||
.arg("echo a")
|
||||
.arg("echo b")
|
||||
.assert()
|
||||
@@ -509,29 +519,27 @@ fn returns_mean_time_in_correct_unit() {
|
||||
.arg("sleep 1.234")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Time (mean ± σ): 1.234 s ±"));
|
||||
.stdout(predicate::str::is_match(r"Wall Time\s+1\.234 s\s+±").unwrap());
|
||||
|
||||
hyperfine_debug()
|
||||
.arg("sleep 0.123")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Time (mean ± σ): 123.0 ms ±"));
|
||||
.stdout(predicate::str::is_match(r"Wall Time\s+123\.0 ms\s+±").unwrap());
|
||||
|
||||
hyperfine_debug()
|
||||
.arg("--time-unit=millisecond")
|
||||
.arg("--metrics=time_wall_clock:ms")
|
||||
.arg("sleep 1.234")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Time (mean ± σ): 1234.0 ms ±"));
|
||||
.stdout(predicate::str::is_match(r"Wall Time\s+1234\.0 ms\s+±").unwrap());
|
||||
|
||||
hyperfine_debug()
|
||||
.arg("--time-unit=microsecond")
|
||||
.arg("--metrics=time_wall_clock:us")
|
||||
.arg("sleep 1.234")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"Time (mean ± σ): 1234000.0 µs ±",
|
||||
));
|
||||
.stdout(predicate::str::is_match(r"Wall Time\s+1234000\.0 µs\s+±").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -625,32 +633,57 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_relative_times() {
|
||||
fn secondary_metrics_include_factors_for_large_changes() {
|
||||
// Mock CPU time is zero; secondary wall time still gets ratio annotations.
|
||||
let output = hyperfine_debug()
|
||||
.args([
|
||||
"--metrics=time_cpu,time_wall_clock",
|
||||
"--style=basic",
|
||||
"--runs=2",
|
||||
"sleep 1",
|
||||
"sleep 10",
|
||||
"sleep 0.1",
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "{:?}", output);
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
assert!(stdout.contains("+900.0%"), "{}", stdout);
|
||||
assert!(stdout.contains("-90.0%"), "{}", stdout);
|
||||
assert!(stdout.contains("+900.0% (10.0x slower)"), "{}", stdout);
|
||||
assert!(stdout.contains("-90.0% (10.0x faster)"), "{}", stdout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_multiple_metrics_relative_to_first_command() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 3.0"), @r"
|
||||
.arg("--metrics=time_wall_clock:ms,time_user:us,memory_peak_resident:MiB")
|
||||
.arg("sleep 0.0817")
|
||||
.arg("sleep 0.6155")
|
||||
.arg("sleep 0.1707"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
Benchmark 1: sleep 0.0817 (36 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 81.7 ms ± 0.0 ms 81.7 ms … 81.7 ms
|
||||
User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs
|
||||
Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB
|
||||
|
||||
Benchmark 2: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
Benchmark 2: sleep 0.6155 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 615.5 ms ± 0.0 ms 615.5 ms … 615.5 ms +653.4% (7.5x slower)
|
||||
User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs N/A
|
||||
Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB N/A
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
Benchmark 3: sleep 0.1707 (17 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 170.7 ms ± 0.0 ms 170.7 ms … 170.7 ms +108.9% (2.1x slower)
|
||||
User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs N/A
|
||||
Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB N/A
|
||||
|
||||
Summary
|
||||
sleep 1.0 ran
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
3.00 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
@@ -660,37 +693,37 @@ fn shows_benchmark_comparison_with_relative_times() {
|
||||
fn shows_benchmark_comparison_with_same_time() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--metrics=time_wall_clock,memory_peak_resident")
|
||||
.arg("--style=basic")
|
||||
.arg("--command-name=A")
|
||||
.arg("--command-name=B")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 1000.0"), @r"
|
||||
.arg("sleep 1000.0"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: A
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
Benchmark 1: A (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B
|
||||
|
||||
Benchmark 2: B
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
Benchmark 2: B (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s 0.0%
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Benchmark 3: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
Benchmark 3: sleep 2.0 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 2.000 s ± 0.000 s 2.000 s … 2.000 s +100.0% (2.0x slower)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Benchmark 4: sleep 1000.0
|
||||
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1000.000 s … 1000.000 s 10 runs
|
||||
Benchmark 4: sleep 1000.0 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 1000.000 s ± 0.000 s 1000.000 s … 1000.000 s +99900.0% (1000.0x slower)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Summary
|
||||
A ran
|
||||
As fast (1.00 ± 0.00) as B
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
1000.00 ± 0.00 times faster than sleep 1000.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
@@ -700,36 +733,36 @@ fn shows_benchmark_comparison_with_same_time() {
|
||||
fn shows_benchmark_comparison_relative_to_reference() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--metrics=time_wall_clock,memory_peak_resident")
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0"), @r"
|
||||
.arg("sleep 3.0"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
Benchmark 1: sleep 2.0 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 2.000 s ± 0.000 s 2.000 s … 2.000 s
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
Benchmark 2: sleep 1.0 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s -50.0% (2.0x faster)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
Benchmark 3: sleep 3.0 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 3.000 s ± 0.000 s 3.000 s … 3.000 s +50.0% (1.5x slower)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Summary
|
||||
sleep 2.0 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_sorted_comparison_and_markup_identify_reference() {
|
||||
fn comparison_and_markup_identify_first_command_as_reference() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let export_path = directory.path().join("results.md");
|
||||
@@ -737,34 +770,31 @@ fn command_sorted_comparison_and_markup_identify_reference() {
|
||||
.args([
|
||||
"--style=basic",
|
||||
"--runs=1",
|
||||
"--sort=command",
|
||||
"--reference",
|
||||
"sleep 2",
|
||||
"--reference-name",
|
||||
"baseline",
|
||||
"--command-name=baseline",
|
||||
"--export-markdown",
|
||||
])
|
||||
.arg(&export_path)
|
||||
.args(["sleep 1", "sleep 2", "sleep 3"])
|
||||
.args(["sleep 2", "sleep 1", "sleep 2", "sleep 3"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "{:?}", output);
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
|
||||
insta::assert_snapshot!(comparison, @r"
|
||||
1.00 baseline (reference)
|
||||
2.00 sleep 1 (faster)
|
||||
1.00 sleep 2 (same speed)
|
||||
1.50 sleep 3 (slower)
|
||||
");
|
||||
assert!(
|
||||
stdout.contains("Benchmark 1: baseline (1 run)"),
|
||||
"{}",
|
||||
stdout
|
||||
);
|
||||
assert!(stdout.contains("-50.0%"), "{}", stdout);
|
||||
assert!(stdout.contains("0.0%"), "{}", stdout);
|
||||
assert!(stdout.contains("+50.0%"), "{}", stdout);
|
||||
let markdown = std::fs::read_to_string(export_path).unwrap();
|
||||
insta::assert_snapshot!(markdown, @r"
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `baseline` | 2.000 | 2.000 | 2.000 | 1.00 (reference) |
|
||||
| `sleep 1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) |
|
||||
| `sleep 2` | 2.000 | 2.000 | 2.000 | 1.00 (same speed) |
|
||||
| `sleep 3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) |
|
||||
insta::assert_snapshot!(markdown, @"
|
||||
| Command | Mean Wall Time [s] | Change | Factor |
|
||||
|:---|---:|---:|:---|
|
||||
| `baseline` | 2.000 | | |
|
||||
| `sleep 1` | 1.000 | -50.0% | (2.0x faster) |
|
||||
| `sleep 2` | 2.000 | 0.0% | |
|
||||
| `sleep 3` | 3.000 | +50.0% | (1.5x slower) |
|
||||
");
|
||||
|
||||
// Equal zero times have no meaningful relative factor.
|
||||
@@ -773,10 +803,8 @@ fn command_sorted_comparison_and_markup_identify_reference() {
|
||||
"--style=none",
|
||||
"--runs=1",
|
||||
"--export-markdown=-",
|
||||
"--reference",
|
||||
"--command-name=baseline",
|
||||
"sleep 0",
|
||||
"--reference-name",
|
||||
"baseline",
|
||||
"sleep 0",
|
||||
])
|
||||
.output()
|
||||
@@ -787,46 +815,14 @@ fn command_sorted_comparison_and_markup_identify_reference() {
|
||||
.lines()
|
||||
.find(|line| line.contains("sleep 0"))
|
||||
.unwrap();
|
||||
assert!(row.ends_with("| N/A |"), "{}", row);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: refabc123
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
refabc123 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
assert!(row.ends_with("| N/A | |"), "{}", row);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--metrics=time_wall_clock,memory_peak_resident")
|
||||
.arg("--style=basic")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
@@ -834,31 +830,30 @@ fn performs_all_benchmarks_in_parameter_scan() {
|
||||
.arg("45")
|
||||
.arg("--parameter-step-size")
|
||||
.arg("5")
|
||||
.arg("sleep {time}"), @r"
|
||||
.arg("sleep {time}"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
Benchmark 1: sleep 30 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 30.000 s ± 0.000 s 30.000 s … 30.000 s
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B
|
||||
|
||||
Benchmark 2: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
Benchmark 2: sleep 35 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 35.000 s ± 0.000 s 35.000 s … 35.000 s +16.7%
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Benchmark 3: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
Benchmark 3: sleep 40 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 40.000 s ± 0.000 s 40.000 s … 40.000 s +33.3% (1.3x slower)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Benchmark 4: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
Benchmark 4: sleep 45 (10 runs)
|
||||
mean ± σ min … max
|
||||
Wall Time 45.000 s ± 0.000 s 45.000 s … 45.000 s +50.0% (1.5x slower)
|
||||
Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A
|
||||
|
||||
Summary
|
||||
sleep 30 ran
|
||||
1.17 ± 0.00 times faster than sleep 35
|
||||
1.33 ± 0.00 times faster than sleep 40
|
||||
1.50 ± 0.00 times faster than sleep 45
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
@@ -882,7 +877,7 @@ fn rejects_negative_parameter_steps() {
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn selects_reference_with_parameterized_prepare() {
|
||||
fn preserves_input_order_with_parameterized_prepare() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let csv_path = directory.path().join("results.csv");
|
||||
let json_path = directory.path().join("results.json");
|
||||
@@ -896,8 +891,6 @@ fn selects_reference_with_parameterized_prepare() {
|
||||
"0.2,0.4,0.6",
|
||||
"--prepare",
|
||||
"sleep {delay}",
|
||||
"--reference",
|
||||
"echo a (delay = 0.4)",
|
||||
"--export-csv",
|
||||
])
|
||||
.arg(&csv_path)
|
||||
@@ -912,7 +905,7 @@ fn selects_reference_with_parameterized_prepare() {
|
||||
for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() {
|
||||
assert!(
|
||||
stdout.contains(&format!(
|
||||
"Benchmark {}: echo a (delay = {})",
|
||||
"Benchmark {}: echo a (delay = {}) (1 run)",
|
||||
number + 1,
|
||||
delay
|
||||
)),
|
||||
@@ -920,7 +913,16 @@ fn selects_reference_with_parameterized_prepare() {
|
||||
stdout
|
||||
);
|
||||
}
|
||||
assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout);
|
||||
assert!(
|
||||
!stdout
|
||||
.split("Benchmark 2")
|
||||
.next()
|
||||
.unwrap()
|
||||
.contains("Change"),
|
||||
"{}",
|
||||
stdout
|
||||
);
|
||||
assert!(!stdout.contains("Change vs #1"));
|
||||
|
||||
let mut csv = csv::Reader::from_path(csv_path).unwrap();
|
||||
assert!(csv
|
||||
@@ -946,14 +948,21 @@ fn selects_reference_with_parameterized_prepare() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unmatched_reference_before_export_or_setup() {
|
||||
fn rejects_invalid_metric_before_export_or_setup() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let export_path = directory.path().join("results.csv");
|
||||
let marker_path = directory.path().join("setup-ran");
|
||||
std::fs::write(&export_path, "previous contents").unwrap();
|
||||
|
||||
hyperfine()
|
||||
.args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"])
|
||||
.args([
|
||||
"--metrics=memory_peak_resident:ms",
|
||||
"-P",
|
||||
"secs",
|
||||
"2",
|
||||
"3",
|
||||
"--setup",
|
||||
])
|
||||
.arg(format!("echo touched > \"{}\"", marker_path.display()))
|
||||
.arg("--export-csv")
|
||||
.arg(&export_path)
|
||||
@@ -962,7 +971,7 @@ fn rejects_unmatched_reference_before_export_or_setup() {
|
||||
.failure()
|
||||
.stdout(predicate::str::is_empty())
|
||||
.stderr(predicate::str::contains(
|
||||
"does not match any parameterized benchmark",
|
||||
"Unit 'ms' is not valid for metric 'memory_peak_resident'",
|
||||
));
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(export_path).unwrap(),
|
||||
@@ -971,29 +980,9 @@ fn rejects_unmatched_reference_before_export_or_setup() {
|
||||
assert!(!marker_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_ambiguous_parameterized_reference() {
|
||||
hyperfine_debug()
|
||||
.args([
|
||||
"-L",
|
||||
"x",
|
||||
"1,2",
|
||||
"--command-name",
|
||||
"case",
|
||||
"--reference",
|
||||
"case",
|
||||
"sleep {x}",
|
||||
])
|
||||
.assert()
|
||||
.failure()
|
||||
.stderr(predicate::str::contains(
|
||||
"matches multiple parameterized benchmarks",
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn intermediate_markdown_waits_for_selected_reference() {
|
||||
fn intermediate_markdown_retains_completed_reference() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let export_path = directory.path().join("results.md");
|
||||
hyperfine()
|
||||
@@ -1004,8 +993,6 @@ fn intermediate_markdown_waits_for_selected_reference() {
|
||||
"-L",
|
||||
"delay",
|
||||
"0.01,0.02,0.03",
|
||||
"--reference",
|
||||
"sleep 0.02",
|
||||
"--prepare",
|
||||
"true",
|
||||
"--prepare",
|
||||
@@ -1023,11 +1010,11 @@ fn intermediate_markdown_waits_for_selected_reference() {
|
||||
.lines()
|
||||
.find(|line| line.contains("sleep 0.01"))
|
||||
.unwrap();
|
||||
assert!(row.ends_with("| N/A |"), "{}", row);
|
||||
assert!(row.ends_with("| | |"), "{}", row);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markdown_export_uses_selected_parameterized_reference() {
|
||||
fn markdown_export_uses_first_parameterized_benchmark_as_reference() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.args([
|
||||
@@ -1040,19 +1027,17 @@ fn markdown_export_uses_selected_parameterized_reference() {
|
||||
"3",
|
||||
"--command-name",
|
||||
"case-{x}",
|
||||
"--reference",
|
||||
"case-2",
|
||||
"sleep {x}",
|
||||
]), @r"
|
||||
]), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `case-1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) |
|
||||
| `case-2` | 2.000 | 2.000 | 2.000 | 1.00 (reference) |
|
||||
| `case-3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) |
|
||||
| Command | Mean Wall Time [s] | Change | Factor |
|
||||
|:---|---:|---:|:---|
|
||||
| `case-1` | 1.000 | | |
|
||||
| `case-2` | 2.000 | +100.0% | (2.0x slower) |
|
||||
| `case-3` | 3.000 | +200.0% | (3.0x slower) |
|
||||
|
||||
|
||||
----- stderr -----
|
||||
@@ -1067,15 +1052,15 @@ fn intermediate_results_are_not_exported_to_stdout() {
|
||||
.arg("--export-markdown")
|
||||
.arg("-")
|
||||
.arg("sleep 1")
|
||||
.arg("sleep 2"), @r"
|
||||
.arg("sleep 2"), @"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||||
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
|
||||
| Command | Mean Wall Time [s] | Change | Factor |
|
||||
|:---|---:|---:|:---|
|
||||
| `sleep 1` | 1.000 ± 0.000 | | |
|
||||
| `sleep 2` | 2.000 ± 0.000 | +100.0% | (2.0x slower) |
|
||||
|
||||
|
||||
----- stderr -----
|
||||
@@ -1158,9 +1143,9 @@ fn markdown_export_preserves_backticks_and_pipes_in_command_names() {
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `` echo `uname` \| cat `` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||||
| Command | Mean Wall Time [s] | Change | Factor |
|
||||
|:---|---:|---:|:---|
|
||||
| `` echo `uname` \| cat `` | 1.000 ± 0.000 | | |
|
||||
|
||||
|
||||
----- stderr -----
|
||||
@@ -1183,60 +1168,6 @@ fn unused_parameters_are_shown_in_benchmark_name() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speed_comparison_sort_order() {
|
||||
let _settings = snapshot_settings().bind_to_scope();
|
||||
insta::allow_duplicates! {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}")), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1 ran
|
||||
2.00 ± 0.00 times faster than sleep 2
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
}
|
||||
|
||||
assert_cmd_snapshot!(hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg("--sort=command"), @r"
|
||||
success: true
|
||||
exit_code: 0
|
||||
----- stdout -----
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Relative speed comparison
|
||||
2.00 ± 0.00 sleep 2
|
||||
1.00 sleep 1
|
||||
|
||||
----- stderr -----
|
||||
");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_quote_args() {
|
||||
@@ -1308,8 +1239,8 @@ fn json_export_basic() {
|
||||
"--export-json=-",
|
||||
"--runs=2",
|
||||
"--warmup=1",
|
||||
"--reference-name=one second",
|
||||
"--reference=sleep 1",
|
||||
"--command-name=one second",
|
||||
"sleep 1",
|
||||
"--command-name=sleep 2",
|
||||
"sleep 2",
|
||||
]));
|
||||
@@ -1338,7 +1269,6 @@ fn json_export_parameterized_with_reference() {
|
||||
"duration",
|
||||
"1,2",
|
||||
"--command-name=sleep for {duration} seconds",
|
||||
"--reference=sleep for 2 seconds",
|
||||
"sleep {duration}",
|
||||
]));
|
||||
}
|
||||
Reference in new issue
Block a user