vsndrg/hyperfine
1
0 Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-11-01 08:50:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify parameterized reference handling

Browse Source
This commit is contained in:
David Peter committed 2026-10-05 15:07:46 +02:00
1 parent f7429e2fb2
commit 463a889638
8 files changed
+210 -693
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/execution_order_tests.rs
-18
View File
@@ -383,24 +383,6 @@ fn reference_is_executed_first() {
.run();
}
#[test]
fn parameterized_reference_keeps_parameter_order() {
ExecutionOrderTest::new()
.arg("--runs=2")
.reference("command 2")
.arg("--parameter-list")
.arg("number")
.arg("1,2,3")
.command("command {number}")
.expect_output("command 1")
.expect_output("command 1")
.expect_output("command 2")
.expect_output("command 2")
.expect_output("command 3")
.expect_output("command 3")
.run();
}
#[test]
fn reference_is_executed_separately_from_commands() {
ExecutionOrderTest::new()
tests/integration_tests.rs
+148 -387
View File
@@ -706,37 +706,71 @@ fn shows_benchmark_comparison_relative_to_reference() {
}
#[test]
fn command_sorted_comparison_labels_explicit_reference() {
for (args, expected) in [
(
vec!["--reference", "sleep 2", "sleep 1", "sleep 3"],
vec![
"1.00 sleep 2 (reference)",
"2.00 ± 0.00 sleep 1 (faster)",
"1.50 ± 0.00 sleep 3 (slower)",
],
),
(
vec!["--reference", "sleep 2", "sleep 2"],
vec![
"1.00 sleep 2 (reference)",
"1.00 ± 0.00 sleep 2 (same speed)",
],
),
fn command_sorted_comparison_and_markup_identify_reference() {
let directory = tempfile::tempdir().unwrap();
let export_path = directory.path().join("results.md");
let output = hyperfine_debug()
.args([
"--style=basic",
"--runs=1",
"--sort=command",
"--reference",
"sleep 2",
"--reference-name",
"baseline",
"--export-markdown",
])
.arg(&export_path)
.args(["sleep 1", "sleep 2", "sleep 3"])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
for (name, factor) in [
("baseline (reference)", "1.00"),
("sleep 1 (faster)", "2.00"),
("sleep 2 (same speed)", "1.00"),
("sleep 3 (slower)", "1.50"),
] {
let output = hyperfine_debug()
.args(["--style=basic", "--runs=2", "--sort=command"])
.args(args)
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
assert_eq!(
comparison.lines().map(str::trim).collect::<Vec<_>>(),
expected
);
let row = comparison.lines().find(|line| line.contains(name)).unwrap();
assert!(row.trim_start().starts_with(factor), "{}", row);
}
let markdown = std::fs::read_to_string(export_path).unwrap();
for (command, relative) in [
("baseline", "1.00 (reference)"),
("sleep 1", "2.00 (faster)"),
("sleep 2", "1.00 (same speed)"),
("sleep 3", "1.50 (slower)"),
] {
let row = markdown
.lines()
.find(|line| line.contains(command))
.unwrap();
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
}
// Equal zero times have no meaningful relative factor.
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--export-markdown=-",
"--reference",
"sleep 0",
"--reference-name",
"baseline",
"sleep 0",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let markdown = String::from_utf8(output.stdout).unwrap();
let row = markdown
.lines()
.find(|line| line.contains("sleep 0"))
.unwrap();
assert!(row.ends_with("| N/A |"), "{}", row);
}
#[test]
@@ -829,40 +863,6 @@ fn rejects_negative_parameter_steps() {
}
}
#[test]
fn selects_reference_from_parameter_scan() {
let output = hyperfine_debug()
.args([
"--style=basic",
"--reference=sleep 35",
"--parameter-scan",
"time",
"30",
"45",
"--parameter-step-size",
"5",
"sleep {time}",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
let benchmarks: Vec<_> = stdout
.lines()
.filter(|line| line.starts_with("Benchmark "))
.collect();
assert_eq!(
benchmarks,
[
"Benchmark 1: sleep 30",
"Benchmark 2: sleep 35",
"Benchmark 3: sleep 40",
"Benchmark 4: sleep 45",
]
);
assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout);
}
#[cfg(unix)]
#[test]
fn selects_reference_with_parameterized_prepare() {
@@ -892,6 +892,17 @@ fn selects_reference_with_parameterized_prepare() {
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.matches("Benchmark ").count(), 3);
for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() {
assert!(
stdout.contains(&format!(
"Benchmark {}: echo a (delay = {})",
number + 1,
delay
)),
"{}",
stdout
);
}
assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout);
let mut csv = csv::Reader::from_path(csv_path).unwrap();
@@ -900,8 +911,10 @@ fn selects_reference_with_parameterized_prepare() {
.unwrap()
.iter()
.any(|header| header == "parameter_delay"));
let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect();
assert_eq!(records.len(), 3);
assert_eq!(
csv.records().collect::<Result<Vec<_>, _>>().unwrap().len(),
3
);
let json: serde_json::Value =
serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap();
let results = json["results"].as_array().unwrap();
@@ -916,97 +929,35 @@ fn selects_reference_with_parameterized_prepare() {
}
#[test]
fn selects_reference_by_custom_benchmark_name() {
for (command, name, reference) in [
("sleep {x}", "case-{x}", "case-2"),
("sleep 1", "case-{x}", "case-2 (x = 2)"),
] {
let output = hyperfine_debug()
.args([
"--style=basic",
"--runs=1",
"-L",
"x",
"1,2",
"--command-name",
name,
"--reference",
reference,
command,
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.matches("Benchmark ").count(), 2);
assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout);
}
}
#[test]
fn selects_reference_from_multiple_commands_and_parameters() {
let output = hyperfine_debug()
.args([
"--style=basic",
"--runs=1",
"-L",
"x",
"1,2",
"-L",
"y",
"3,4",
"--reference",
"sleep 4 (x = 2)",
"sleep {x}",
"sleep {y}",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(stdout.matches("Benchmark ").count(), 8);
assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout);
}
#[test]
fn rejects_unmatched_parameterized_reference_before_creating_export() {
fn rejects_unmatched_reference_before_export_or_setup() {
let directory = tempfile::tempdir().unwrap();
let export_path = directory.path().join("results.csv");
let marker_path = directory.path().join("setup-ran");
for (args, benchmark) in [
(vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"),
(
vec!["--reference", "sleep 1", "-P", "secs", "2", "3"],
"sleep {secs}",
),
] {
std::fs::write(&export_path, "previous contents").unwrap();
hyperfine()
.args(args)
.arg("--setup")
.arg(format!("echo touched > \"{}\"", marker_path.display()))
.arg("--export-csv")
.arg(&export_path)
.arg(benchmark)
.assert()
.failure()
.stdout(predicate::str::is_empty())
.stderr(predicate::str::contains(
"does not match any parameterized benchmark",
));
std::fs::write(&export_path, "previous contents").unwrap();
assert_eq!(
std::fs::read_to_string(&export_path).unwrap(),
"previous contents"
);
assert!(!marker_path.exists());
}
hyperfine()
.args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"])
.arg(format!("echo touched > \"{}\"", marker_path.display()))
.arg("--export-csv")
.arg(&export_path)
.arg("sleep {secs}")
.assert()
.failure()
.stdout(predicate::str::is_empty())
.stderr(predicate::str::contains(
"does not match any parameterized benchmark",
));
assert_eq!(
std::fs::read_to_string(export_path).unwrap(),
"previous contents"
);
assert!(!marker_path.exists());
}
#[test]
fn rejects_ambiguous_parameterized_reference() {
for args in [
vec![
hyperfine_debug()
.args([
"-L",
"x",
"1,2",
@@ -1015,120 +966,21 @@ fn rejects_ambiguous_parameterized_reference() {
"--reference",
"case",
"sleep {x}",
],
vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"],
] {
hyperfine_debug()
.args(args)
.assert()
.failure()
.stdout(predicate::str::is_empty())
.stderr(predicate::str::contains(
"matches multiple parameterized benchmarks",
));
}
}
#[test]
fn rejects_reference_name_with_parameterized_reference() {
hyperfine_debug()
.args([
"-L",
"x",
"1,2",
"--reference",
"sleep 1",
"--reference-name",
"baseline",
"sleep {x}",
])
.assert()
.failure()
.stderr(predicate::str::contains(
"--reference-name cannot be used with a parameterized reference",
"matches multiple parameterized benchmarks",
));
}
#[test]
fn parameterized_reference_preserves_per_benchmark_hook_order() {
let directory = tempfile::tempdir().unwrap();
let log_path = directory.path().join("hooks.log");
let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display());
let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display());
hyperfine()
.args(["--runs=1", "-L", "x", "1,2,3"])
.arg("--command-name")
.arg("case-{x}")
.arg("--reference")
.arg("case-2")
.arg("--prepare")
.arg(prepare("prepare-1"))
.arg("--prepare")
.arg(prepare("prepare-2"))
.arg("--prepare")
.arg(prepare("prepare-3"))
.arg(command)
.assert()
.success();
let contents = std::fs::read_to_string(log_path).unwrap();
assert_eq!(
contents.lines().map(str::trim).collect::<Vec<_>>(),
[
"prepare-1",
"benchmark-1",
"prepare-2",
"benchmark-2",
"prepare-3",
"benchmark-3"
]
);
}
#[test]
fn parameterized_reference_preserves_per_benchmark_output_order() {
let directory = tempfile::tempdir().unwrap();
let output_paths = [
directory.path().join("one.txt"),
directory.path().join("two.txt"),
directory.path().join("three.txt"),
];
let mut command = hyperfine();
command.args([
"--runs=1",
"-L",
"x",
"1,2,3",
"--command-name",
"case-{x}",
"--reference",
"case-2",
]);
for path in &output_paths {
command.arg("--output").arg(path);
}
command.arg("echo benchmark-{x}").assert().success();
for (path, expected) in output_paths
.iter()
.zip(["benchmark-1", "benchmark-2", "benchmark-3"])
{
assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected);
}
}
#[cfg(unix)]
#[test]
fn intermediate_markdown_waits_for_selected_reference() {
let directory = tempfile::tempdir().unwrap();
for (reference, preparations, reference_available) in [
("sleep 0.03", ["true", "false", "true"], false),
("sleep 0.02", ["true", "true", "false"], true),
] {
let export_path = directory.path().join("results.md");
let mut command = hyperfine();
command.args([
let export_path = directory.path().join("results.md");
hyperfine()
.args([
"--style=none",
"--shell=none",
"--runs=1",
@@ -1136,146 +988,55 @@ fn intermediate_markdown_waits_for_selected_reference() {
"delay",
"0.01,0.02,0.03",
"--reference",
reference,
]);
for preparation in preparations {
command.arg("--prepare").arg(preparation);
}
command
.arg("--export-markdown")
.arg(&export_path)
.arg("sleep {delay}")
.assert()
.failure()
.stderr(predicate::str::contains(
"The preparation command terminated",
));
let markdown = std::fs::read_to_string(&export_path).unwrap();
if reference_available {
assert!(!markdown.contains("N/A"), "{}", markdown);
let row = markdown
.lines()
.find(|line| line.contains("sleep 0.02"))
.unwrap();
assert!(row.ends_with("| 1.00 (reference) |"), "{}", row);
} else {
let row = markdown
.lines()
.find(|line| line.contains("sleep 0.01"))
.unwrap();
assert!(row.ends_with("| N/A |"), "{}", row);
}
}
"sleep 0.02",
"--prepare",
"true",
"--prepare",
"false",
"--prepare",
"true",
"--export-markdown",
])
.arg(&export_path)
.arg("sleep {delay}")
.assert()
.failure();
let markdown = std::fs::read_to_string(export_path).unwrap();
let row = markdown
.lines()
.find(|line| line.contains("sleep 0.01"))
.unwrap();
assert!(row.ends_with("| N/A |"), "{}", row);
}
#[test]
fn markdown_export_uses_selected_parameterized_reference() {
for sort_order in ["command", "mean-time"] {
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--time-unit=second",
"--export-markdown=-",
"--sort",
sort_order,
"-L",
"x",
"3,2,1",
"--reference",
"sleep 2",
"sleep {x}",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
let rows: Vec<_> = stdout
.lines()
.filter(|line| line.contains("sleep "))
.collect();
if sort_order == "command" {
assert!(rows[0].contains("sleep 3"));
assert!(rows[1].contains("sleep 2"));
assert!(rows[2].contains("sleep 1"));
} else {
assert!(rows[0].contains("sleep 1"));
assert!(rows[1].contains("sleep 2"));
assert!(rows[2].contains("sleep 3"));
}
for (command, relative) in [
("sleep 1", "2.00 (faster)"),
("sleep 2", "1.00 (reference)"),
("sleep 3", "1.50 (slower)"),
] {
let row = rows.iter().find(|line| line.contains(command)).unwrap();
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
}
}
}
#[test]
fn markdown_export_uses_standalone_reference() {
for sort_order in ["command", "mean-time"] {
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--time-unit=second",
"--export-markdown=-",
"--sort",
sort_order,
"--reference",
"sleep 2",
"--reference-name",
"baseline",
"sleep 1",
"sleep 2",
"sleep 3",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
for (command, relative) in [
("baseline", "1.00 (reference)"),
("sleep 1", "2.00 (faster)"),
("sleep 2", "1.00 (same speed)"),
("sleep 3", "1.50 (slower)"),
] {
let row = stdout.lines().find(|line| line.contains(command)).unwrap();
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
}
}
}
#[test]
fn markdown_export_without_explicit_reference_is_unchanged() {
for sort_order in ["command", "mean-time"] {
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--time-unit=second",
"--export-markdown=-",
"--sort",
sort_order,
"sleep 3",
"sleep 1",
"sleep 2",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
for (command, relative) in [
("sleep 1", "1.00"),
("sleep 2", "2.00"),
("sleep 3", "3.00"),
] {
let row = stdout.lines().find(|line| line.contains(command)).unwrap();
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
}
let output = hyperfine_debug()
.args([
"--style=none",
"--runs=1",
"--export-markdown=-",
"-P",
"x",
"1",
"3",
"--command-name",
"case-{x}",
"--reference",
"case-2",
"sleep {x}",
])
.output()
.unwrap();
assert!(output.status.success(), "{:?}", output);
let stdout = String::from_utf8(output.stdout).unwrap();
for (command, relative) in [
("case-1", "2.00 (faster)"),
("case-2", "1.00 (reference)"),
("case-3", "1.50 (slower)"),
] {
let row = stdout.lines().find(|line| line.contains(command)).unwrap();
assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
}
}