From 463a8896383d34c997df767554ca5e11a6c17f5f Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 15:07:46 +0200 Subject: [PATCH] Simplify parameterized reference handling --- README.md | 10 - src/benchmark/relative_speed.rs | 74 ++--- src/benchmark/scheduler.rs | 27 +- src/export/markup.rs | 43 +-- src/export/tests.rs | 167 +--------- src/options.rs | 29 +- tests/execution_order_tests.rs | 18 -- tests/integration_tests.rs | 535 +++++++++----------------------- 8 files changed, 210 insertions(+), 693 deletions(-) diff --git a/README.md b/README.md index f92d4bc..ca4f06c 100644 --- a/README.md +++ b/README.md @@ -98,16 +98,6 @@ hyperfine \ 'python main.py' ``` -To compare a parameterized benchmark against a particular run, pass its exact printed name to -`--reference`. For example: -```sh -hyperfine -L delay 0.2,0.4,0.6 --prepare 'sleep {delay}' --reference 'echo a (delay = 0.4)' 'echo a' -``` -The name must identify exactly one benchmark, including any `--command-name` and displayed unused -parameters. `--reference-name` cannot be used with parameterized benchmarks; use `--command-name` -to name them instead. Without `--parameter-scan` or `--parameter-list`, `--reference` runs a -separate reference command. - If you need a unique value for each individual run of a benchmark command, hyperfine also exposes the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself: ```sh diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 4b03c3e..e62a591 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -16,6 +16,20 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { pub relative_ordering: Ordering, } +impl BenchmarkResultWithRelativeSpeed<'_> { + pub fn reference_label(&self) -> &'static str { + if self.is_reference { + " (reference)" + } else { + match self.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Equal => " (same speed)", + Ordering::Greater => " (slower)", + } + } + } +} + pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean_wall_clock_time() .partial_cmp(&r.mean_wall_clock_time()) @@ -29,7 +43,7 @@ pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { .expect("at least one benchmark result") } -pub fn compute_relative_speeds<'a>( +fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, @@ -125,14 +139,14 @@ pub fn compute_with_check( Some(compute_relative_speeds(results, fastest, sort_order)) } -/// Same as compute_with_check, potentially resulting in relative speeds of infinity -pub fn compute( - results: &[BenchmarkResult], +/// Compute relative speeds against the given reference, or the fastest result. +pub fn compute<'a>( + results: &'a [BenchmarkResult], sort_order: SortOrder, -) -> Vec> { - let fastest = fastest_of(results); - - compute_relative_speeds(results, fastest, sort_order) + reference: Option<&'a BenchmarkResult>, +) -> Vec> { + let reference = reference.unwrap_or_else(|| fastest_of(results)); + compute_relative_speeds(results, reference, sort_order) } #[cfg(test)] @@ -205,25 +219,6 @@ fn reference_identity_distinguishes_equal_results() { assert!(!entries[1].is_reference); } -#[test] -fn reference_ratios_indicate_faster_and_slower_runs() { - use approx::assert_relative_eq; - - let results = vec![ - create_result("faster", 1.0), - create_result("reference", 2.0), - create_result("slower", 3.0), - ]; - let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); - - assert_relative_eq!(entries[0].relative_speed, 2.0); - assert_relative_eq!(entries[1].relative_speed, 1.0); - assert_relative_eq!(entries[2].relative_speed, 1.5); - assert!(!entries[0].is_reference); - assert!(entries[1].is_reference); - assert!(!entries[2].is_reference); -} - #[test] fn reference_ratios_handle_zero_times() { let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; @@ -239,28 +234,3 @@ fn reference_ratios_handle_zero_times() { assert_eq!(entries[0].relative_speed_stddev, None); assert_eq!(entries[1].relative_speed, 1.0); } - -#[test] -fn reference_ratio_propagates_uncertainty() { - use crate::benchmark::measurement::Measurement; - use crate::quantity::second; - use approx::assert_relative_eq; - - let mut results = vec![ - create_result("faster", 1.0), - create_result("reference", 3.0), - ]; - for (result, times) in results.iter_mut().zip([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0]]) { - result.measurements.measurements = times - .iter() - .copied() - .map(|seconds| Measurement { - time_wall_clock: Time::new::(seconds), - ..Default::default() - }) - .collect(); - } - let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); - assert_relative_eq!(entries[0].relative_speed, 2.0); - assert_relative_eq!(entries[0].relative_speed_stddev.unwrap(), 1.118033988749895); -} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 76cba13..db036d6 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -44,7 +44,6 @@ impl<'a> Scheduler<'a> { .options .reference_command .as_ref() - .filter(|_| !self.options.parameterized_mode) .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd)); executor.calibrate()?; @@ -56,7 +55,7 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. self.export_manager - .write_results(&self.results, true, self.reference_index())?; + .write_results(&self.results, true, self.options.reference_index)?; } Ok(()) @@ -72,7 +71,8 @@ impl<'a> Scheduler<'a> { } let reference = self - .reference_index() + .options + .reference_index .map(|index| &self.results[index]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); @@ -130,16 +130,10 @@ impl<'a> Scheduler<'a> { console_writeln!(stdout, "{}", "Relative speed comparison".bold())?; for item in annotated_results { - let relationship = if self.reference_index().is_none() { - "" - } else if item.is_reference { - " (reference)" + let relationship = if self.options.reference_index.is_some() { + item.reference_label() } else { - match item.relative_ordering { - Ordering::Less => " (faster)", - Ordering::Equal => " (same speed)", - Ordering::Greater => " (slower)", - } + "" }; console_writeln!( stdout, @@ -174,16 +168,9 @@ impl<'a> Scheduler<'a> { Ok(()) } - fn reference_index(&self) -> Option { - self.options - .reference_command - .as_ref() - .map(|_| self.options.parameterized_reference_index.unwrap_or(0)) - } - pub fn final_export(&self) -> Result<()> { self.export_manager - .write_results(&self.results, false, self.reference_index()) + .write_results(&self.results, false, self.options.reference_index) } } diff --git a/src/export/markup.rs b/src/export/markup.rs index d5b2daa..0eea533 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -67,28 +67,19 @@ pub trait MarkupExporter { && !entry.is_reference && entry.relative_ordering == Ordering::Equal && measurement.mean_wall_clock_time() == Time::zero()); - let rel_str = if relative_unavailable { + let relative = if relative_unavailable { "N/A".to_string() } else { - format!("{:.2}", entry.relative_speed) - }; - let rel_stddev_str = if relative_unavailable || entry.is_reference { - "".into() - } else if let Some(stddev) = entry.relative_speed_stddev { - format!(" ± {stddev:.2}") - } else { - "".into() - }; - let rel_direction = if relative_unavailable || !explicit_reference { - "" - } else if entry.is_reference { - " (reference)" - } else { - match entry.relative_ordering { - Ordering::Less => " (faster)", - Ordering::Greater => " (slower)", - Ordering::Equal => " (same speed)", - } + let stddev = match (entry.is_reference, entry.relative_speed_stddev) { + (false, Some(stddev)) => format!(" ± {stddev:.2}"), + _ => String::new(), + }; + let label = if explicit_reference { + entry.reference_label() + } else { + "" + }; + format!("{:.2}{stddev}{label}", entry.relative_speed) }; // prepare table row entries @@ -97,7 +88,7 @@ pub trait MarkupExporter { &format!("{mean_str}{stddev_str}"), &min_str, &max_str, - &format!("{rel_str}{rel_stddev_str}{rel_direction}"), + &relative, ])) } @@ -144,11 +135,11 @@ impl Exporter for T { // Do not report ratios against another benchmark while the selected // reference is still pending in an intermediate export. let reference_pending = reference_index.is_some_and(|i| i >= results.len()); - let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) { - relative_speed::compute_relative_speeds(results, reference, sort_order) - } else { - relative_speed::compute(results, sort_order) - }; + let entries = relative_speed::compute( + results, + sort_order, + reference_index.and_then(|i| results.get(i)), + ); let table = self.table_results(&entries, unit, reference_pending, reference_index.is_some()); diff --git a/src/export/tests.rs b/src/export/tests.rs index 098226f..abe340a 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -12,174 +12,9 @@ fn get_output( results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, -) -> String { - get_output_with_reference::(results, unit, sort_order, None) -} - -fn get_output_with_reference( - results: &[BenchmarkResult], - unit: Option, - sort_order: SortOrder, - reference_index: Option, ) -> String { let exporter = E::default(); - String::from_utf8( - exporter - .serialize(results, unit, sort_order, reference_index) - .unwrap(), - ) - .unwrap() -} - -#[test] -fn test_markup_exports_identify_explicit_reference_and_direction() { - let benchmark = |name: &str, seconds| { - let measurement = Measurement { - time_wall_clock: Time::new::(seconds), - ..Measurement::default() - }; - BenchmarkResult { - command: name.to_owned(), - command_with_unused_parameters: name.to_owned(), - measurements: Measurements::new(vec![measurement, measurement]), - ..BenchmarkResult::default() - } - }; - let results = [ - benchmark("faster", 1.0), - benchmark("reference", 2.0), - benchmark("equal", 2.0), - benchmark("slower", 3.0), - ]; - - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - ]; - for output in outputs { - for expected in [ - "2.00 ± 0.00 (faster)", - "1.00 (reference)", - "1.00 ± 0.00 (same speed)", - "1.50 ± 0.00 (slower)", - ] { - assert!(output.contains(expected), "{}", output); - } - } - } -} - -#[test] -fn test_markup_exports_leave_pending_reference_ratios_unavailable() { - let results = [BenchmarkResult { - command: "sleep 1".into(), - command_with_unused_parameters: "sleep 1".into(), - measurements: Measurements::new(vec![Measurement { - time_wall_clock: Time::new::(1.0), - ..Measurement::default() - }]), - ..BenchmarkResult::default() - }]; - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - ]; - for output in outputs { - assert!(output.contains("N/A"), "{}", output); - assert!(!output.contains("(reference)"), "{}", output); - } - } -} - -#[test] -fn test_markup_exports_with_zero_reference() { - let benchmark = |name: &str, seconds| BenchmarkResult { - command: name.to_owned(), - command_with_unused_parameters: name.to_owned(), - measurements: Measurements::new(vec![Measurement { - time_wall_clock: Time::new::(seconds), - ..Measurement::default() - }]), - ..BenchmarkResult::default() - }; - let results = [ - benchmark("other zero", 0.0), - benchmark("reference", 0.0), - benchmark("slower", 1.0), - ]; - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - for reference in [Some(1), Some(3), None] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - ]; - for output in outputs { - match reference { - Some(1) => { - assert_eq!(output.matches("N/A").count(), 1, "{}", output); - assert!(output.contains("1.00 (reference)"), "{}", output); - assert!(output.contains("inf (slower)"), "{}", output); - } - Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output), - None => { - assert!(!output.contains("N/A"), "{}", output); - assert!(output.contains("inf"), "{}", output); - assert!(!output.contains("(reference)"), "{}", output); - } - } - } - } - } + String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap() } /// Ensure the makrup output includes the table header and the multiple diff --git a/src/options.rs b/src/options.rs index 659b976..f5a3285 100644 --- a/src/options.rs +++ b/src/options.rs @@ -208,17 +208,14 @@ pub struct Options { /// Whether or not to ignore non-zero exit codes pub command_failure_action: CmdFailureAction, - // Command to use as a reference for relative speed comparison + // Standalone reference command, cleared when an existing benchmark is selected. pub reference_command: Option, // Name of the reference command pub reference_name: Option, - // Whether commands are expanded from --parameter-scan or --parameter-list. - pub parameterized_mode: bool, - - // Index of the selected command in a parameterized run, if any. - pub parameterized_reference_index: Option, + // Index of the reference in the benchmark sequence, if any. + pub reference_index: Option, /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -263,8 +260,7 @@ impl Default for Options { command_failure_action: CmdFailureAction::RaiseError, reference_command: None, reference_name: None, - parameterized_mode: false, - parameterized_reference_index: None, + reference_index: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -317,8 +313,6 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); - options.parameterized_mode = matches.get_many::("parameter-scan").is_some() - || matches.get_many::("parameter-list").is_some(); options.reference_command = matches.get_one::("reference").map(String::from); options.reference_name = matches .get_one::("reference-name") @@ -474,8 +468,12 @@ impl Options { } pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { - if self.parameterized_mode { - if let Some(reference) = &self.reference_command { + if let Some(reference) = &self.reference_command { + if commands + .iter() + .next() + .is_some_and(|command| !command.get_parameters().is_empty()) + { ensure!( self.reference_name.is_none(), "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark" @@ -493,10 +491,13 @@ impl Options { matches.next().is_none(), "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names." ); - self.parameterized_reference_index = Some(index); + self.reference_index = Some(index); + self.reference_command = None; + } else { + self.reference_index = Some(0); } } - let has_reference_command = self.reference_command.is_some() && !self.parameterized_mode; + let has_reference_command = self.reference_command.is_some(); let num_commands = commands.num_commands(has_reference_command); if let Some(preparation_command) = &self.preparation_command { diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index b097e98..e09eb63 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -383,24 +383,6 @@ fn reference_is_executed_first() { .run(); } -#[test] -fn parameterized_reference_keeps_parameter_order() { - ExecutionOrderTest::new() - .arg("--runs=2") - .reference("command 2") - .arg("--parameter-list") - .arg("number") - .arg("1,2,3") - .command("command {number}") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") - .run(); -} - #[test] fn reference_is_executed_separately_from_commands() { ExecutionOrderTest::new() diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2256585..3bf25ff 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -706,37 +706,71 @@ fn shows_benchmark_comparison_relative_to_reference() { } #[test] -fn command_sorted_comparison_labels_explicit_reference() { - for (args, expected) in [ - ( - vec!["--reference", "sleep 2", "sleep 1", "sleep 3"], - vec![ - "1.00 sleep 2 (reference)", - "2.00 ± 0.00 sleep 1 (faster)", - "1.50 ± 0.00 sleep 3 (slower)", - ], - ), - ( - vec!["--reference", "sleep 2", "sleep 2"], - vec![ - "1.00 sleep 2 (reference)", - "1.00 ± 0.00 sleep 2 (same speed)", - ], - ), +fn command_sorted_comparison_and_markup_identify_reference() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.md"); + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "--sort=command", + "--reference", + "sleep 2", + "--reference-name", + "baseline", + "--export-markdown", + ]) + .arg(&export_path) + .args(["sleep 1", "sleep 2", "sleep 3"]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; + for (name, factor) in [ + ("baseline (reference)", "1.00"), + ("sleep 1 (faster)", "2.00"), + ("sleep 2 (same speed)", "1.00"), + ("sleep 3 (slower)", "1.50"), ] { - let output = hyperfine_debug() - .args(["--style=basic", "--runs=2", "--sort=command"]) - .args(args) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; - assert_eq!( - comparison.lines().map(str::trim).collect::>(), - expected - ); + let row = comparison.lines().find(|line| line.contains(name)).unwrap(); + assert!(row.trim_start().starts_with(factor), "{}", row); } + let markdown = std::fs::read_to_string(export_path).unwrap(); + for (command, relative) in [ + ("baseline", "1.00 (reference)"), + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (same speed)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = markdown + .lines() + .find(|line| line.contains(command)) + .unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + + // Equal zero times have no meaningful relative factor. + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "--reference", + "sleep 0", + "--reference-name", + "baseline", + "sleep 0", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let markdown = String::from_utf8(output.stdout).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); } #[test] @@ -829,40 +863,6 @@ fn rejects_negative_parameter_steps() { } } -#[test] -fn selects_reference_from_parameter_scan() { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--reference=sleep 35", - "--parameter-scan", - "time", - "30", - "45", - "--parameter-step-size", - "5", - "sleep {time}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let benchmarks: Vec<_> = stdout - .lines() - .filter(|line| line.starts_with("Benchmark ")) - .collect(); - assert_eq!( - benchmarks, - [ - "Benchmark 1: sleep 30", - "Benchmark 2: sleep 35", - "Benchmark 3: sleep 40", - "Benchmark 4: sleep 45", - ] - ); - assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout); -} - #[cfg(unix)] #[test] fn selects_reference_with_parameterized_prepare() { @@ -892,6 +892,17 @@ fn selects_reference_with_parameterized_prepare() { assert!(output.status.success(), "{:?}", output); let stdout = String::from_utf8(output.stdout).unwrap(); assert_eq!(stdout.matches("Benchmark ").count(), 3); + for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() { + assert!( + stdout.contains(&format!( + "Benchmark {}: echo a (delay = {})", + number + 1, + delay + )), + "{}", + stdout + ); + } assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout); let mut csv = csv::Reader::from_path(csv_path).unwrap(); @@ -900,8 +911,10 @@ fn selects_reference_with_parameterized_prepare() { .unwrap() .iter() .any(|header| header == "parameter_delay")); - let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect(); - assert_eq!(records.len(), 3); + assert_eq!( + csv.records().collect::, _>>().unwrap().len(), + 3 + ); let json: serde_json::Value = serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap(); let results = json["results"].as_array().unwrap(); @@ -916,97 +929,35 @@ fn selects_reference_with_parameterized_prepare() { } #[test] -fn selects_reference_by_custom_benchmark_name() { - for (command, name, reference) in [ - ("sleep {x}", "case-{x}", "case-2"), - ("sleep 1", "case-{x}", "case-2 (x = 2)"), - ] { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--runs=1", - "-L", - "x", - "1,2", - "--command-name", - name, - "--reference", - reference, - command, - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - assert_eq!(stdout.matches("Benchmark ").count(), 2); - assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout); - } -} - -#[test] -fn selects_reference_from_multiple_commands_and_parameters() { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--runs=1", - "-L", - "x", - "1,2", - "-L", - "y", - "3,4", - "--reference", - "sleep 4 (x = 2)", - "sleep {x}", - "sleep {y}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - assert_eq!(stdout.matches("Benchmark ").count(), 8); - assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout); -} - -#[test] -fn rejects_unmatched_parameterized_reference_before_creating_export() { +fn rejects_unmatched_reference_before_export_or_setup() { let directory = tempfile::tempdir().unwrap(); let export_path = directory.path().join("results.csv"); let marker_path = directory.path().join("setup-ran"); - for (args, benchmark) in [ - (vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"), - ( - vec!["--reference", "sleep 1", "-P", "secs", "2", "3"], - "sleep {secs}", - ), - ] { - std::fs::write(&export_path, "previous contents").unwrap(); - hyperfine() - .args(args) - .arg("--setup") - .arg(format!("echo touched > \"{}\"", marker_path.display())) - .arg("--export-csv") - .arg(&export_path) - .arg(benchmark) - .assert() - .failure() - .stdout(predicate::str::is_empty()) - .stderr(predicate::str::contains( - "does not match any parameterized benchmark", - )); + std::fs::write(&export_path, "previous contents").unwrap(); - assert_eq!( - std::fs::read_to_string(&export_path).unwrap(), - "previous contents" - ); - assert!(!marker_path.exists()); - } + hyperfine() + .args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"]) + .arg(format!("echo touched > \"{}\"", marker_path.display())) + .arg("--export-csv") + .arg(&export_path) + .arg("sleep {secs}") + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "does not match any parameterized benchmark", + )); + assert_eq!( + std::fs::read_to_string(export_path).unwrap(), + "previous contents" + ); + assert!(!marker_path.exists()); } #[test] fn rejects_ambiguous_parameterized_reference() { - for args in [ - vec![ + hyperfine_debug() + .args([ "-L", "x", "1,2", @@ -1015,120 +966,21 @@ fn rejects_ambiguous_parameterized_reference() { "--reference", "case", "sleep {x}", - ], - vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"], - ] { - hyperfine_debug() - .args(args) - .assert() - .failure() - .stdout(predicate::str::is_empty()) - .stderr(predicate::str::contains( - "matches multiple parameterized benchmarks", - )); - } -} - -#[test] -fn rejects_reference_name_with_parameterized_reference() { - hyperfine_debug() - .args([ - "-L", - "x", - "1,2", - "--reference", - "sleep 1", - "--reference-name", - "baseline", - "sleep {x}", ]) .assert() .failure() .stderr(predicate::str::contains( - "--reference-name cannot be used with a parameterized reference", + "matches multiple parameterized benchmarks", )); } -#[test] -fn parameterized_reference_preserves_per_benchmark_hook_order() { - let directory = tempfile::tempdir().unwrap(); - let log_path = directory.path().join("hooks.log"); - let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display()); - let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display()); - - hyperfine() - .args(["--runs=1", "-L", "x", "1,2,3"]) - .arg("--command-name") - .arg("case-{x}") - .arg("--reference") - .arg("case-2") - .arg("--prepare") - .arg(prepare("prepare-1")) - .arg("--prepare") - .arg(prepare("prepare-2")) - .arg("--prepare") - .arg(prepare("prepare-3")) - .arg(command) - .assert() - .success(); - - let contents = std::fs::read_to_string(log_path).unwrap(); - assert_eq!( - contents.lines().map(str::trim).collect::>(), - [ - "prepare-1", - "benchmark-1", - "prepare-2", - "benchmark-2", - "prepare-3", - "benchmark-3" - ] - ); -} - -#[test] -fn parameterized_reference_preserves_per_benchmark_output_order() { - let directory = tempfile::tempdir().unwrap(); - let output_paths = [ - directory.path().join("one.txt"), - directory.path().join("two.txt"), - directory.path().join("three.txt"), - ]; - let mut command = hyperfine(); - command.args([ - "--runs=1", - "-L", - "x", - "1,2,3", - "--command-name", - "case-{x}", - "--reference", - "case-2", - ]); - for path in &output_paths { - command.arg("--output").arg(path); - } - command.arg("echo benchmark-{x}").assert().success(); - - for (path, expected) in output_paths - .iter() - .zip(["benchmark-1", "benchmark-2", "benchmark-3"]) - { - assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected); - } -} - #[cfg(unix)] #[test] fn intermediate_markdown_waits_for_selected_reference() { let directory = tempfile::tempdir().unwrap(); - for (reference, preparations, reference_available) in [ - ("sleep 0.03", ["true", "false", "true"], false), - ("sleep 0.02", ["true", "true", "false"], true), - ] { - let export_path = directory.path().join("results.md"); - let mut command = hyperfine(); - command.args([ + let export_path = directory.path().join("results.md"); + hyperfine() + .args([ "--style=none", "--shell=none", "--runs=1", @@ -1136,146 +988,55 @@ fn intermediate_markdown_waits_for_selected_reference() { "delay", "0.01,0.02,0.03", "--reference", - reference, - ]); - for preparation in preparations { - command.arg("--prepare").arg(preparation); - } - command - .arg("--export-markdown") - .arg(&export_path) - .arg("sleep {delay}") - .assert() - .failure() - .stderr(predicate::str::contains( - "The preparation command terminated", - )); - let markdown = std::fs::read_to_string(&export_path).unwrap(); - if reference_available { - assert!(!markdown.contains("N/A"), "{}", markdown); - let row = markdown - .lines() - .find(|line| line.contains("sleep 0.02")) - .unwrap(); - assert!(row.ends_with("| 1.00 (reference) |"), "{}", row); - } else { - let row = markdown - .lines() - .find(|line| line.contains("sleep 0.01")) - .unwrap(); - assert!(row.ends_with("| N/A |"), "{}", row); - } - } + "sleep 0.02", + "--prepare", + "true", + "--prepare", + "false", + "--prepare", + "true", + "--export-markdown", + ]) + .arg(&export_path) + .arg("sleep {delay}") + .assert() + .failure(); + let markdown = std::fs::read_to_string(export_path).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.01")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); } #[test] fn markdown_export_uses_selected_parameterized_reference() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "-L", - "x", - "3,2,1", - "--reference", - "sleep 2", - "sleep {x}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let rows: Vec<_> = stdout - .lines() - .filter(|line| line.contains("sleep ")) - .collect(); - if sort_order == "command" { - assert!(rows[0].contains("sleep 3")); - assert!(rows[1].contains("sleep 2")); - assert!(rows[2].contains("sleep 1")); - } else { - assert!(rows[0].contains("sleep 1")); - assert!(rows[1].contains("sleep 2")); - assert!(rows[2].contains("sleep 3")); - } - for (command, relative) in [ - ("sleep 1", "2.00 (faster)"), - ("sleep 2", "1.00 (reference)"), - ("sleep 3", "1.50 (slower)"), - ] { - let row = rows.iter().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } - } -} - -#[test] -fn markdown_export_uses_standalone_reference() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "--reference", - "sleep 2", - "--reference-name", - "baseline", - "sleep 1", - "sleep 2", - "sleep 3", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - for (command, relative) in [ - ("baseline", "1.00 (reference)"), - ("sleep 1", "2.00 (faster)"), - ("sleep 2", "1.00 (same speed)"), - ("sleep 3", "1.50 (slower)"), - ] { - let row = stdout.lines().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } - } -} - -#[test] -fn markdown_export_without_explicit_reference_is_unchanged() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "sleep 3", - "sleep 1", - "sleep 2", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - for (command, relative) in [ - ("sleep 1", "1.00"), - ("sleep 2", "2.00"), - ("sleep 3", "3.00"), - ] { - let row = stdout.lines().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "-P", + "x", + "1", + "3", + "--command-name", + "case-{x}", + "--reference", + "case-2", + "sleep {x}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("case-1", "2.00 (faster)"), + ("case-2", "1.00 (reference)"), + ("case-3", "1.50 (slower)"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); } }