mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-10-31 00:40:18 +00:00
1519 lines
43 KiB
Rust
1519 lines
43 KiB
Rust
mod common;
|
||
use common::{hyperfine, hyperfine_raw_command};
|
||
|
||
use assert_cmd::assert::OutputAssertExt;
|
||
use insta_cmd::assert_cmd_snapshot;
|
||
use predicates::prelude::*;
|
||
|
||
/// Platform-specific I/O utility.
|
||
/// - On Unix-like systems, defaults to `cat`.
|
||
/// - On Windows, uses `findstr` as an alternative.
|
||
/// See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
|
||
const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
|
||
|
||
pub fn hyperfine_debug() -> std::process::Command {
|
||
let mut cmd = hyperfine_raw_command();
|
||
cmd.arg("--debug-mode");
|
||
cmd
|
||
}
|
||
|
||
fn snapshot_settings() -> insta::Settings {
|
||
let mut settings = insta::Settings::clone_current();
|
||
// Ignore trailing report padding while preserving indentation and internal spacing.
|
||
settings.add_filter(r"(?m)[ \t]+$", "");
|
||
settings
|
||
}
|
||
|
||
#[test]
|
||
fn runs_successfully() {
|
||
hyperfine()
|
||
.arg("--runs=2")
|
||
.arg("echo dummy benchmark")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
#[test]
|
||
fn one_run_is_supported() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("echo dummy benchmark")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
/// Regression test: hyperfine must not panic when writing to a closed
|
||
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
|
||
/// that terminates early). A `BrokenPipe` error should result in a quiet
|
||
/// exit with code 0 instead of a `println!` panic.
|
||
#[test]
|
||
fn exits_quietly_when_stdout_is_closed() {
|
||
use std::process::Stdio;
|
||
|
||
for args in [
|
||
vec!["--runs=1", "echo dummy benchmark"],
|
||
vec!["--runs=1", "--export-json=-", "echo dummy benchmark"],
|
||
vec!["--runs=1", "echo dummy benchmark", "echo second benchmark"],
|
||
] {
|
||
// A pipe whose read end has been closed before hyperfine is even
|
||
// started: every write to it will fail with `BrokenPipe`.
|
||
let (reader, writer) = std::io::pipe().expect("failed to create pipe");
|
||
drop(reader);
|
||
|
||
let output = hyperfine_raw_command()
|
||
.args(&args)
|
||
.stdout(writer)
|
||
.stderr(Stdio::piped())
|
||
.spawn()
|
||
.expect("failed to spawn hyperfine")
|
||
.wait_with_output()
|
||
.expect("failed to wait for hyperfine");
|
||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||
|
||
assert!(
|
||
!stderr.contains("panicked"),
|
||
"hyperfine panicked while writing to a closed pipe:\n{}",
|
||
stderr
|
||
);
|
||
assert!(
|
||
output.status.success(),
|
||
"hyperfine did not exit cleanly; stderr:\n{}",
|
||
stderr
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn fails_with_zero_runs() {
|
||
for option in ["--runs", "--min-runs", "--max-runs"] {
|
||
hyperfine()
|
||
.arg(option)
|
||
.arg("0")
|
||
.arg("echo dummy benchmark")
|
||
.assert()
|
||
.code(2)
|
||
.stderr(predicate::str::contains(format!(
|
||
"invalid value '0' for '{option} <NUM>'"
|
||
)));
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn min_runs_of_one_still_performs_one_run() {
|
||
hyperfine_debug()
|
||
.arg("--min-runs=1")
|
||
.arg("--warmup=0")
|
||
.arg("sleep 4")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("Time (abs ≡)"));
|
||
}
|
||
|
||
#[test]
|
||
fn can_run_commands_without_a_shell() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--show-output")
|
||
.arg("--shell=none")
|
||
.arg("echo 'hello world' argument2")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("hello world argument2"));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_with_wrong_number_of_command_name_arguments() {
|
||
hyperfine()
|
||
.arg("--command-name=a")
|
||
.arg("--command-name=b")
|
||
.arg("echo a")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains("Too many --command-name options"));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_with_wrong_number_of_prepare_options() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--prepare=echo a")
|
||
.arg("--prepare=echo b")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--prepare=echo ref")
|
||
.arg("--prepare=echo a")
|
||
.arg("--prepare=echo b")
|
||
.arg("--reference=echo ref")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--prepare=echo a")
|
||
.arg("--prepare=echo b")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.arg("echo c")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The '--prepare' option has to be provided",
|
||
));
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--prepare=echo a")
|
||
.arg("--prepare=echo b")
|
||
.arg("--reference=echo ref")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The '--prepare' option has to be provided",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_with_wrong_number_of_conclude_options() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--conclude=echo a")
|
||
.arg("--conclude=echo b")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--conclude=echo ref")
|
||
.arg("--conclude=echo a")
|
||
.arg("--conclude=echo b")
|
||
.arg("--reference=echo ref")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--conclude=echo a")
|
||
.arg("--conclude=echo b")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.arg("echo c")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The '--conclude' option has to be provided",
|
||
));
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--conclude=echo a")
|
||
.arg("--conclude=echo b")
|
||
.arg("--reference=echo ref")
|
||
.arg("echo a")
|
||
.arg("echo b")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The '--conclude' option has to be provided",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_with_duplicate_parameter_names() {
|
||
hyperfine()
|
||
.arg("--parameter-list")
|
||
.arg("x")
|
||
.arg("1,2,3")
|
||
.arg("--parameter-list")
|
||
.arg("x")
|
||
.arg("a,b,c")
|
||
.arg("echo test")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains("Duplicate parameter names: x"));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_command() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Command terminated with non-zero exit code",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_command_without_shell() {
|
||
hyperfine()
|
||
.arg("--shell=none")
|
||
.arg("--runs=1")
|
||
.arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Failed to run command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577'",
|
||
));
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn fails_for_failing_command_without_shell() {
|
||
hyperfine()
|
||
.arg("--shell=none")
|
||
.arg("--runs=1")
|
||
.arg("false")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Command terminated with non-zero exit code",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_setup_command() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--setup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.arg("echo test")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The setup command terminated with a non-zero exit code.",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_cleanup_command() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--cleanup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.arg("echo test")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The cleanup command terminated with a non-zero exit code.",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_prepare_command() {
|
||
hyperfine()
|
||
.arg("--prepare=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.arg("echo test")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The preparation command terminated with a non-zero exit code.",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_for_unknown_conclude_command() {
|
||
hyperfine()
|
||
.arg("--conclude=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
|
||
.arg("echo test")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The conclusion command terminated with a non-zero exit code.",
|
||
));
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn can_run_failing_commands_with_ignore_failure_option() {
|
||
hyperfine()
|
||
.arg("false")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Command terminated with non-zero exit code",
|
||
));
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure")
|
||
.arg("false")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn can_ignore_specific_exit_codes() {
|
||
// Test that specifying exit code 1 ignores it
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1")
|
||
.arg("exit 1")
|
||
.assert()
|
||
.success();
|
||
|
||
// Test that other exit codes still fail
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1")
|
||
.arg("exit 2")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Command terminated with non-zero exit code 2",
|
||
));
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn can_ignore_multiple_exit_codes() {
|
||
// Test that all specified exit codes are ignored
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1,2,3")
|
||
.arg("exit 1")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1,2,3")
|
||
.arg("exit 2")
|
||
.assert()
|
||
.success();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1,2,3")
|
||
.arg("exit 3")
|
||
.assert()
|
||
.success();
|
||
|
||
// Test that other exit codes still fail
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=1,2,3")
|
||
.arg("exit 4")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"Command terminated with non-zero exit code 4",
|
||
));
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn ignore_failure_with_all_non_zero() {
|
||
// Test explicit "all-non-zero" mode
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--ignore-failure=all-non-zero")
|
||
.arg("exit 5")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
#[test]
|
||
fn shows_output_of_benchmarked_command() {
|
||
hyperfine()
|
||
.arg("--runs=2")
|
||
.arg("--command-name=dummy")
|
||
.arg("--show-output")
|
||
.arg("echo 4fd47015")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("4fd47015").count(2));
|
||
}
|
||
|
||
#[test]
|
||
fn runs_commands_using_user_defined_shell() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--show-output")
|
||
.arg("--shell")
|
||
.arg("echo 'custom_shell' '--shell-arg'")
|
||
.arg("echo benchmark")
|
||
.assert()
|
||
.success()
|
||
.stdout(
|
||
predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or(
|
||
predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),
|
||
),
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn can_pass_input_to_command_from_a_file() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--input=example_input_file.txt")
|
||
.arg("--show-output")
|
||
.arg(STDIN_READ_COMMAND)
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("This text is part of a file"));
|
||
}
|
||
|
||
#[test]
|
||
fn fails_if_invalid_stdin_data_file_provided() {
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--input=example_non_existent_file.txt")
|
||
.arg("--show-output")
|
||
.arg(STDIN_READ_COMMAND)
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The file 'example_non_existent_file.txt' specified as '--input' does not exist",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn returns_mean_time_in_correct_unit() {
|
||
hyperfine_debug()
|
||
.arg("sleep 1.234")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("Time (mean ± σ): 1.234 s ±"));
|
||
|
||
hyperfine_debug()
|
||
.arg("sleep 0.123")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("Time (mean ± σ): 123.0 ms ±"));
|
||
|
||
hyperfine_debug()
|
||
.arg("--time-unit=millisecond")
|
||
.arg("sleep 1.234")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("Time (mean ± σ): 1234.0 ms ±"));
|
||
|
||
hyperfine_debug()
|
||
.arg("--time-unit=microsecond")
|
||
.arg("sleep 1.234")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains(
|
||
"Time (mean ± σ): 1234000.0 µs ±",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn performs_ten_runs_for_slow_commands() {
|
||
hyperfine_debug()
|
||
.arg("sleep 0.5")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("10 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn performs_three_seconds_of_benchmarking_for_fast_commands() {
|
||
hyperfine_debug()
|
||
.arg("sleep 0.01")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("300 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn takes_shell_spawning_time_into_account_for_computing_number_of_runs() {
|
||
hyperfine_debug()
|
||
.arg("--shell=sleep 0.02")
|
||
.arg("sleep 0.01")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("100 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn takes_preparation_command_into_account_for_computing_number_of_runs() {
|
||
hyperfine_debug()
|
||
.arg("--prepare=sleep 0.02")
|
||
.arg("sleep 0.01")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("100 runs"));
|
||
|
||
// Shell overhead needs to be added to both the prepare command and the actual command,
|
||
// leading to a total benchmark time of (prepare + shell + cmd + shell = 0.1 s)
|
||
hyperfine_debug()
|
||
.arg("--shell=sleep 0.01")
|
||
.arg("--prepare=sleep 0.03")
|
||
.arg("sleep 0.05")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("30 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn takes_conclusion_command_into_account_for_computing_number_of_runs() {
|
||
hyperfine_debug()
|
||
.arg("--conclude=sleep 0.02")
|
||
.arg("sleep 0.01")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("100 runs"));
|
||
|
||
// Shell overhead needs to be added to both the conclude command and the actual command,
|
||
// leading to a total benchmark time of (cmd + shell + conclude + shell = 0.1 s)
|
||
hyperfine_debug()
|
||
.arg("--shell=sleep 0.01")
|
||
.arg("--conclude=sleep 0.03")
|
||
.arg("sleep 0.05")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("30 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn takes_both_preparation_and_conclusion_command_into_account_for_computing_number_of_runs() {
|
||
hyperfine_debug()
|
||
.arg("--prepare=sleep 0.01")
|
||
.arg("--conclude=sleep 0.01")
|
||
.arg("sleep 0.01")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("100 runs"));
|
||
|
||
// Shell overhead needs to be added to both the prepare, conclude and the actual command,
|
||
// leading to a total benchmark time of (prepare + shell + cmd + shell + conclude + shell = 0.1 s)
|
||
hyperfine_debug()
|
||
.arg("--shell=sleep 0.01")
|
||
.arg("--prepare=sleep 0.01")
|
||
.arg("--conclude=sleep 0.01")
|
||
.arg("sleep 0.05")
|
||
.assert()
|
||
.success()
|
||
.stdout(predicate::str::contains("30 runs"));
|
||
}
|
||
|
||
#[test]
|
||
fn shows_benchmark_comparison_with_relative_times() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("sleep 1.0")
|
||
.arg("sleep 2.0")
|
||
.arg("sleep 3.0"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: sleep 1.0
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 2.0
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 3: sleep 3.0
|
||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||
|
||
Summary
|
||
sleep 1.0 ran
|
||
2.00 ± 0.00 times faster than sleep 2.0
|
||
3.00 ± 0.00 times faster than sleep 3.0
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn shows_benchmark_comparison_with_same_time() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("--command-name=A")
|
||
.arg("--command-name=B")
|
||
.arg("sleep 1.0")
|
||
.arg("sleep 1.0")
|
||
.arg("sleep 2.0")
|
||
.arg("sleep 1000.0"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: A
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Benchmark 2: B
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Benchmark 3: sleep 2.0
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 4: sleep 1000.0
|
||
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1000.000 s … 1000.000 s 10 runs
|
||
|
||
Summary
|
||
A ran
|
||
As fast (1.00 ± 0.00) as B
|
||
2.00 ± 0.00 times faster than sleep 2.0
|
||
1000.00 ± 0.00 times faster than sleep 1000.0
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn shows_benchmark_comparison_relative_to_reference() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("--reference=sleep 2.0")
|
||
.arg("sleep 1.0")
|
||
.arg("sleep 3.0"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: sleep 2.0
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 1.0
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Benchmark 3: sleep 3.0
|
||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||
|
||
Summary
|
||
sleep 2.0 ran
|
||
2.00 ± 0.00 times slower than sleep 1.0
|
||
1.50 ± 0.00 times faster than sleep 3.0
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn command_sorted_comparison_labels_explicit_reference() {
|
||
for (args, expected) in [
|
||
(
|
||
vec!["--reference", "sleep 2", "sleep 1", "sleep 3"],
|
||
vec![
|
||
"1.00 sleep 2 (reference)",
|
||
"2.00 ± 0.00 sleep 1 (faster)",
|
||
"1.50 ± 0.00 sleep 3 (slower)",
|
||
],
|
||
),
|
||
(
|
||
vec!["--reference", "sleep 2", "sleep 2"],
|
||
vec![
|
||
"1.00 sleep 2 (reference)",
|
||
"1.00 ± 0.00 sleep 2 (same speed)",
|
||
],
|
||
),
|
||
] {
|
||
let output = hyperfine_debug()
|
||
.args(["--style=basic", "--runs=2", "--sort=command"])
|
||
.args(args)
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
|
||
assert_eq!(
|
||
comparison.lines().map(str::trim).collect::<Vec<_>>(),
|
||
expected
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn shows_reference_name() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("--reference=sleep 2.0")
|
||
.arg("--reference-name=refabc123")
|
||
.arg("sleep 1.0")
|
||
.arg("sleep 3.0"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: refabc123
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 1.0
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Benchmark 3: sleep 3.0
|
||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||
|
||
Summary
|
||
refabc123 ran
|
||
2.00 ± 0.00 times slower than sleep 1.0
|
||
1.50 ± 0.00 times faster than sleep 3.0
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn performs_all_benchmarks_in_parameter_scan() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("--parameter-scan")
|
||
.arg("time")
|
||
.arg("30")
|
||
.arg("45")
|
||
.arg("--parameter-step-size")
|
||
.arg("5")
|
||
.arg("sleep {time}"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: sleep 30
|
||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 35
|
||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||
|
||
Benchmark 3: sleep 40
|
||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||
|
||
Benchmark 4: sleep 45
|
||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||
|
||
Summary
|
||
sleep 30 ran
|
||
1.17 ± 0.00 times faster than sleep 35
|
||
1.33 ± 0.00 times faster than sleep 40
|
||
1.50 ± 0.00 times faster than sleep 45
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn rejects_negative_parameter_steps() {
|
||
for step in ["-1", "-1.0"] {
|
||
hyperfine()
|
||
.args(["--parameter-scan", "n", "0", "1"])
|
||
.arg(format!("--parameter-step-size={step}"))
|
||
.arg("echo {n}")
|
||
.assert()
|
||
.code(1)
|
||
.stdout(predicate::str::is_empty())
|
||
.stderr(predicate::str::contains(
|
||
"Parameter step size must be positive",
|
||
));
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn selects_reference_from_parameter_scan() {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=basic",
|
||
"--reference=sleep 35",
|
||
"--parameter-scan",
|
||
"time",
|
||
"30",
|
||
"45",
|
||
"--parameter-step-size",
|
||
"5",
|
||
"sleep {time}",
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
let benchmarks: Vec<_> = stdout
|
||
.lines()
|
||
.filter(|line| line.starts_with("Benchmark "))
|
||
.collect();
|
||
assert_eq!(
|
||
benchmarks,
|
||
[
|
||
"Benchmark 1: sleep 30",
|
||
"Benchmark 2: sleep 35",
|
||
"Benchmark 3: sleep 40",
|
||
"Benchmark 4: sleep 45",
|
||
]
|
||
);
|
||
assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout);
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn selects_reference_with_parameterized_prepare() {
|
||
let directory = tempfile::tempdir().unwrap();
|
||
let csv_path = directory.path().join("results.csv");
|
||
let json_path = directory.path().join("results.json");
|
||
let output = hyperfine_raw_command()
|
||
.args([
|
||
"--style=basic",
|
||
"--shell=none",
|
||
"--runs=1",
|
||
"-L",
|
||
"delay",
|
||
"0.2,0.4,0.6",
|
||
"--prepare",
|
||
"sleep {delay}",
|
||
"--reference",
|
||
"echo a (delay = 0.4)",
|
||
"--export-csv",
|
||
])
|
||
.arg(&csv_path)
|
||
.arg("--export-json")
|
||
.arg(&json_path)
|
||
.arg("echo a")
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
assert_eq!(stdout.matches("Benchmark ").count(), 3);
|
||
assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout);
|
||
|
||
let mut csv = csv::Reader::from_path(csv_path).unwrap();
|
||
assert!(csv
|
||
.headers()
|
||
.unwrap()
|
||
.iter()
|
||
.any(|header| header == "parameter_delay"));
|
||
let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect();
|
||
assert_eq!(records.len(), 3);
|
||
let json: serde_json::Value =
|
||
serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap();
|
||
let results = json["results"].as_array().unwrap();
|
||
assert_eq!(results.len(), 3);
|
||
assert_eq!(
|
||
results
|
||
.iter()
|
||
.map(|result| result["parameters"]["delay"].as_str().unwrap())
|
||
.collect::<Vec<_>>(),
|
||
["0.2", "0.4", "0.6"]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn selects_reference_by_custom_benchmark_name() {
|
||
for (command, name, reference) in [
|
||
("sleep {x}", "case-{x}", "case-2"),
|
||
("sleep 1", "case-{x}", "case-2 (x = 2)"),
|
||
] {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=basic",
|
||
"--runs=1",
|
||
"-L",
|
||
"x",
|
||
"1,2",
|
||
"--command-name",
|
||
name,
|
||
"--reference",
|
||
reference,
|
||
command,
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
assert_eq!(stdout.matches("Benchmark ").count(), 2);
|
||
assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn selects_reference_from_multiple_commands_and_parameters() {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=basic",
|
||
"--runs=1",
|
||
"-L",
|
||
"x",
|
||
"1,2",
|
||
"-L",
|
||
"y",
|
||
"3,4",
|
||
"--reference",
|
||
"sleep 4 (x = 2)",
|
||
"sleep {x}",
|
||
"sleep {y}",
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
assert_eq!(stdout.matches("Benchmark ").count(), 8);
|
||
assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout);
|
||
}
|
||
|
||
#[test]
|
||
fn rejects_unmatched_parameterized_reference_before_creating_export() {
|
||
let directory = tempfile::tempdir().unwrap();
|
||
let export_path = directory.path().join("results.csv");
|
||
let marker_path = directory.path().join("setup-ran");
|
||
for (args, benchmark) in [
|
||
(vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"),
|
||
(
|
||
vec!["--reference", "sleep 1", "-P", "secs", "2", "3"],
|
||
"sleep {secs}",
|
||
),
|
||
] {
|
||
std::fs::write(&export_path, "previous contents").unwrap();
|
||
hyperfine()
|
||
.args(args)
|
||
.arg("--setup")
|
||
.arg(format!("echo touched > \"{}\"", marker_path.display()))
|
||
.arg("--export-csv")
|
||
.arg(&export_path)
|
||
.arg(benchmark)
|
||
.assert()
|
||
.failure()
|
||
.stdout(predicate::str::is_empty())
|
||
.stderr(predicate::str::contains(
|
||
"does not match any parameterized benchmark",
|
||
));
|
||
|
||
assert_eq!(
|
||
std::fs::read_to_string(&export_path).unwrap(),
|
||
"previous contents"
|
||
);
|
||
assert!(!marker_path.exists());
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn rejects_ambiguous_parameterized_reference() {
|
||
for args in [
|
||
vec![
|
||
"-L",
|
||
"x",
|
||
"1,2",
|
||
"--command-name",
|
||
"case",
|
||
"--reference",
|
||
"case",
|
||
"sleep {x}",
|
||
],
|
||
vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"],
|
||
] {
|
||
hyperfine_debug()
|
||
.args(args)
|
||
.assert()
|
||
.failure()
|
||
.stdout(predicate::str::is_empty())
|
||
.stderr(predicate::str::contains(
|
||
"matches multiple parameterized benchmarks",
|
||
));
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn rejects_reference_name_with_parameterized_reference() {
|
||
hyperfine_debug()
|
||
.args([
|
||
"-L",
|
||
"x",
|
||
"1,2",
|
||
"--reference",
|
||
"sleep 1",
|
||
"--reference-name",
|
||
"baseline",
|
||
"sleep {x}",
|
||
])
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"--reference-name cannot be used with a parameterized reference",
|
||
));
|
||
}
|
||
|
||
#[test]
|
||
fn parameterized_reference_preserves_per_benchmark_hook_order() {
|
||
let directory = tempfile::tempdir().unwrap();
|
||
let log_path = directory.path().join("hooks.log");
|
||
let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display());
|
||
let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display());
|
||
|
||
hyperfine()
|
||
.args(["--runs=1", "-L", "x", "1,2,3"])
|
||
.arg("--command-name")
|
||
.arg("case-{x}")
|
||
.arg("--reference")
|
||
.arg("case-2")
|
||
.arg("--prepare")
|
||
.arg(prepare("prepare-1"))
|
||
.arg("--prepare")
|
||
.arg(prepare("prepare-2"))
|
||
.arg("--prepare")
|
||
.arg(prepare("prepare-3"))
|
||
.arg(command)
|
||
.assert()
|
||
.success();
|
||
|
||
let contents = std::fs::read_to_string(log_path).unwrap();
|
||
assert_eq!(
|
||
contents.lines().map(str::trim).collect::<Vec<_>>(),
|
||
[
|
||
"prepare-1",
|
||
"benchmark-1",
|
||
"prepare-2",
|
||
"benchmark-2",
|
||
"prepare-3",
|
||
"benchmark-3"
|
||
]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn parameterized_reference_preserves_per_benchmark_output_order() {
|
||
let directory = tempfile::tempdir().unwrap();
|
||
let output_paths = [
|
||
directory.path().join("one.txt"),
|
||
directory.path().join("two.txt"),
|
||
directory.path().join("three.txt"),
|
||
];
|
||
let mut command = hyperfine();
|
||
command.args([
|
||
"--runs=1",
|
||
"-L",
|
||
"x",
|
||
"1,2,3",
|
||
"--command-name",
|
||
"case-{x}",
|
||
"--reference",
|
||
"case-2",
|
||
]);
|
||
for path in &output_paths {
|
||
command.arg("--output").arg(path);
|
||
}
|
||
command.arg("echo benchmark-{x}").assert().success();
|
||
|
||
for (path, expected) in output_paths
|
||
.iter()
|
||
.zip(["benchmark-1", "benchmark-2", "benchmark-3"])
|
||
{
|
||
assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected);
|
||
}
|
||
}
|
||
|
||
#[cfg(unix)]
|
||
#[test]
|
||
fn intermediate_markdown_waits_for_selected_reference() {
|
||
let directory = tempfile::tempdir().unwrap();
|
||
for (reference, preparations, reference_available) in [
|
||
("sleep 0.03", ["true", "false", "true"], false),
|
||
("sleep 0.02", ["true", "true", "false"], true),
|
||
] {
|
||
let export_path = directory.path().join("results.md");
|
||
let mut command = hyperfine();
|
||
command.args([
|
||
"--style=none",
|
||
"--shell=none",
|
||
"--runs=1",
|
||
"-L",
|
||
"delay",
|
||
"0.01,0.02,0.03",
|
||
"--reference",
|
||
reference,
|
||
]);
|
||
for preparation in preparations {
|
||
command.arg("--prepare").arg(preparation);
|
||
}
|
||
command
|
||
.arg("--export-markdown")
|
||
.arg(&export_path)
|
||
.arg("sleep {delay}")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(
|
||
"The preparation command terminated",
|
||
));
|
||
let markdown = std::fs::read_to_string(&export_path).unwrap();
|
||
if reference_available {
|
||
assert!(!markdown.contains("N/A"), "{}", markdown);
|
||
let row = markdown
|
||
.lines()
|
||
.find(|line| line.contains("sleep 0.02"))
|
||
.unwrap();
|
||
assert!(row.ends_with("| 1.00 (reference) |"), "{}", row);
|
||
} else {
|
||
let row = markdown
|
||
.lines()
|
||
.find(|line| line.contains("sleep 0.01"))
|
||
.unwrap();
|
||
assert!(row.ends_with("| N/A |"), "{}", row);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_export_uses_selected_parameterized_reference() {
|
||
for sort_order in ["command", "mean-time"] {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=none",
|
||
"--runs=1",
|
||
"--time-unit=second",
|
||
"--export-markdown=-",
|
||
"--sort",
|
||
sort_order,
|
||
"-L",
|
||
"x",
|
||
"3,2,1",
|
||
"--reference",
|
||
"sleep 2",
|
||
"sleep {x}",
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
let rows: Vec<_> = stdout
|
||
.lines()
|
||
.filter(|line| line.contains("sleep "))
|
||
.collect();
|
||
if sort_order == "command" {
|
||
assert!(rows[0].contains("sleep 3"));
|
||
assert!(rows[1].contains("sleep 2"));
|
||
assert!(rows[2].contains("sleep 1"));
|
||
} else {
|
||
assert!(rows[0].contains("sleep 1"));
|
||
assert!(rows[1].contains("sleep 2"));
|
||
assert!(rows[2].contains("sleep 3"));
|
||
}
|
||
for (command, relative) in [
|
||
("sleep 1", "2.00 (faster)"),
|
||
("sleep 2", "1.00 (reference)"),
|
||
("sleep 3", "1.50 (slower)"),
|
||
] {
|
||
let row = rows.iter().find(|line| line.contains(command)).unwrap();
|
||
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_export_uses_standalone_reference() {
|
||
for sort_order in ["command", "mean-time"] {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=none",
|
||
"--runs=1",
|
||
"--time-unit=second",
|
||
"--export-markdown=-",
|
||
"--sort",
|
||
sort_order,
|
||
"--reference",
|
||
"sleep 2",
|
||
"--reference-name",
|
||
"baseline",
|
||
"sleep 1",
|
||
"sleep 2",
|
||
"sleep 3",
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
for (command, relative) in [
|
||
("baseline", "1.00 (reference)"),
|
||
("sleep 1", "2.00 (faster)"),
|
||
("sleep 2", "1.00 (same speed)"),
|
||
("sleep 3", "1.50 (slower)"),
|
||
] {
|
||
let row = stdout.lines().find(|line| line.contains(command)).unwrap();
|
||
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_export_without_explicit_reference_is_unchanged() {
|
||
for sort_order in ["command", "mean-time"] {
|
||
let output = hyperfine_debug()
|
||
.args([
|
||
"--style=none",
|
||
"--runs=1",
|
||
"--time-unit=second",
|
||
"--export-markdown=-",
|
||
"--sort",
|
||
sort_order,
|
||
"sleep 3",
|
||
"sleep 1",
|
||
"sleep 2",
|
||
])
|
||
.output()
|
||
.unwrap();
|
||
assert!(output.status.success(), "{:?}", output);
|
||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||
for (command, relative) in [
|
||
("sleep 1", "1.00"),
|
||
("sleep 2", "2.00"),
|
||
("sleep 3", "3.00"),
|
||
] {
|
||
let row = stdout.lines().find(|line| line.contains(command)).unwrap();
|
||
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn intermediate_results_are_not_exported_to_stdout() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=none") // To only see the Markdown export on stdout
|
||
.arg("--export-markdown")
|
||
.arg("-")
|
||
.arg("sleep 1")
|
||
.arg("sleep 2"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
|
||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||
|:---|---:|---:|---:|---:|
|
||
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
|
||
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
#[cfg(unix)]
|
||
fn exports_intermediate_results_to_file() {
|
||
use tempfile::tempdir;
|
||
|
||
let tempdir = tempdir().unwrap();
|
||
let export_path = tempdir.path().join("results.md");
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--export-markdown")
|
||
.arg(&export_path)
|
||
.arg("true")
|
||
.arg("false")
|
||
.assert()
|
||
.failure();
|
||
|
||
let contents = std::fs::read_to_string(export_path).unwrap();
|
||
assert!(contents.contains("true"));
|
||
}
|
||
|
||
#[test]
|
||
fn invalid_command_options_preserve_export_files() {
|
||
use tempfile::tempdir;
|
||
|
||
for (option, values) in [
|
||
("--prepare", ["echo first", "echo second"]),
|
||
("--conclude", ["echo first", "echo second"]),
|
||
("--output", ["null", "pipe"]),
|
||
] {
|
||
let directory = tempdir().unwrap();
|
||
let existing_export = directory.path().join("previous.json");
|
||
let new_export = directory.path().join("new.csv");
|
||
let previous_results =
|
||
b"{\"results\":[{\"command\":\"previous benchmark\",\"mean\":1.0}]}\n";
|
||
std::fs::write(&existing_export, previous_results).unwrap();
|
||
|
||
hyperfine()
|
||
.arg("--runs=1")
|
||
.arg("--export-json")
|
||
.arg(&existing_export)
|
||
.arg("--export-csv")
|
||
.arg(&new_export)
|
||
.args([option, values[0], option, values[1]])
|
||
.arg("echo benchmark")
|
||
.assert()
|
||
.failure()
|
||
.stderr(predicate::str::contains(format!(
|
||
"The '{option}' option has to be provided just once or N times"
|
||
)));
|
||
|
||
assert_eq!(
|
||
std::fs::read(&existing_export).unwrap(),
|
||
previous_results,
|
||
"{option} validation must preserve previous results"
|
||
);
|
||
assert!(
|
||
!new_export.exists(),
|
||
"{} validation must not create export files",
|
||
option
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_export_preserves_backticks_and_pipes_in_command_names() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=none")
|
||
.arg("--export-markdown=-")
|
||
.arg("--command-name=echo `uname` | cat")
|
||
.arg("sleep 1"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
|
||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||
|:---|---:|---:|---:|---:|
|
||
| `` echo `uname` \| cat `` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[test]
|
||
fn unused_parameters_are_shown_in_benchmark_name() {
|
||
hyperfine()
|
||
.arg("--runs=2")
|
||
.arg("--parameter-list")
|
||
.arg("branch")
|
||
.arg("master,feature")
|
||
.arg("echo test")
|
||
.assert()
|
||
.success()
|
||
.stdout(
|
||
predicate::str::contains("echo test (branch = master)")
|
||
.and(predicate::str::contains("echo test (branch = feature)")),
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn speed_comparison_sort_order() {
|
||
let _settings = snapshot_settings().bind_to_scope();
|
||
insta::allow_duplicates! {
|
||
for sort_order in ["auto", "mean-time"] {
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("sleep 2")
|
||
.arg("sleep 1")
|
||
.arg(format!("--sort={sort_order}")), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: sleep 2
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 1
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Summary
|
||
sleep 1 ran
|
||
2.00 ± 0.00 times faster than sleep 2
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
}
|
||
|
||
assert_cmd_snapshot!(hyperfine_debug()
|
||
.arg("--style=basic")
|
||
.arg("sleep 2")
|
||
.arg("sleep 1")
|
||
.arg("--sort=command"), @r"
|
||
success: true
|
||
exit_code: 0
|
||
----- stdout -----
|
||
Benchmark 1: sleep 2
|
||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||
|
||
Benchmark 2: sleep 1
|
||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||
|
||
Relative speed comparison
|
||
2.00 ± 0.00 sleep 2
|
||
1.00 sleep 1
|
||
|
||
----- stderr -----
|
||
");
|
||
}
|
||
|
||
#[cfg(windows)]
|
||
#[test]
|
||
fn windows_quote_args() {
|
||
hyperfine()
|
||
.arg("more \"example_input_file.txt\"")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
#[cfg(windows)]
|
||
#[test]
|
||
fn windows_quote_before_quote_args() {
|
||
hyperfine()
|
||
.arg("dir \"..\\src\\\" \"..\\tests\\\"")
|
||
.assert()
|
||
.success();
|
||
}
|
||
|
||
#[test]
|
||
fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
|
||
let tempdir = tempfile::tempdir().unwrap();
|
||
let output_path = tempdir.path().join("iteration output.txt");
|
||
let iteration = if cfg!(windows) {
|
||
"%HYPERFINE_ITERATION%"
|
||
} else {
|
||
"$HYPERFINE_ITERATION"
|
||
};
|
||
let command = |phase| format!(r#"echo {phase}:{iteration} >> "iteration output.txt""#);
|
||
|
||
hyperfine()
|
||
.current_dir(tempdir.path())
|
||
.arg("--runs=2")
|
||
.arg("--warmup=1")
|
||
.arg("--prepare")
|
||
.arg(command("prepare"))
|
||
.arg(command("benchmark"))
|
||
.arg("--conclude")
|
||
.arg(command("conclude"))
|
||
.assert()
|
||
.success();
|
||
|
||
let contents = std::fs::read_to_string(output_path).unwrap();
|
||
// cmd.exe's echo includes the space before the redirection operator.
|
||
let lines: Vec<&str> = contents.lines().map(str::trim_end).collect();
|
||
assert_eq!(
|
||
lines,
|
||
[
|
||
"prepare:warmup-0",
|
||
"benchmark:warmup-0",
|
||
"conclude:warmup-0",
|
||
"prepare:0",
|
||
"benchmark:0",
|
||
"conclude:0",
|
||
"prepare:1",
|
||
"benchmark:1",
|
||
"conclude:1",
|
||
]
|
||
);
|
||
}
|