From f7429e2fb25b8cddcdc1742231a8bff37bb85639 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:59:18 +0200 Subject: [PATCH 1/4] Use parameterized benchmarks as references --- README.md | 10 + doc/hyperfine.1 | 8 +- src/benchmark/relative_speed.rs | 82 ++++- src/benchmark/scheduler.rs | 34 ++- src/cli.rs | 10 +- src/export/csv.rs | 3 +- src/export/json.rs | 1 + src/export/markup.rs | 45 ++- src/export/mod.rs | 12 +- src/export/tests.rs | 167 ++++++++++- src/options.rs | 36 ++- tests/execution_order_tests.rs | 6 +- tests/integration_tests.rs | 513 +++++++++++++++++++++++++++++--- 13 files changed, 856 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index ca4f06c..f92d4bc 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,16 @@ hyperfine \ 'python main.py' ``` +To compare a parameterized benchmark against a particular run, pass its exact printed name to +`--reference`. For example: +```sh +hyperfine -L delay 0.2,0.4,0.6 --prepare 'sleep {delay}' --reference 'echo a (delay = 0.4)' 'echo a' +``` +The name must identify exactly one benchmark, including any `--command-name` and displayed unused +parameters. `--reference-name` cannot be used with parameterized benchmarks; use `--command-name` +to name them instead. Without `--parameter-scan` or `--parameter-list`, `--reference` runs a +separate reference command. + If you need a unique value for each individual run of a benchmark command, hyperfine also exposes the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself: ```sh diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index d565dd3..0136ff8 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -116,13 +116,17 @@ not every time as would happen with the \fB\-\-prepare\fR option. .HP \fB\-\-reference\fR \fICMD\fP .IP -Benchmark \fICMD\fP as the reference for the relative comparison of results. -If this option is not specified, the fastest command is used as the reference. +Use \fICMD\fP as the reference for the relative comparison of results. Without +parameters, benchmark \fICMD\fP as a separate reference command. If this option +is not specified, the fastest command is used as the reference. +With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing +benchmark by its exact printed name. The name must match exactly one benchmark. .HP \fB\-\-reference\-name\fR \fINAME\fP .IP Give a meaningful \fINAME\fP to the reference command. Requires \fB\-\-reference\fR. +Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead. .HP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP .IP diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 711ca2f..4b03c3e 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -29,7 +29,7 @@ pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { .expect("at least one benchmark result") } -fn compute_relative_speeds<'a>( +pub fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, @@ -37,10 +37,13 @@ fn compute_relative_speeds<'a>( let mut results: Vec<_> = results .iter() .map(|result| { - let is_reference = result == reference; + // Separate benchmarks can have identical names and measurements. + let is_reference = std::ptr::eq(result, reference); let relative_ordering = compare_mean_time(result, reference); - if result.mean_wall_clock_time() == Time::zero() { + if result.mean_wall_clock_time() == Time::zero() + || reference.mean_wall_clock_time() == Time::zero() + { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, @@ -64,11 +67,11 @@ fn compute_relative_speeds<'a>( result.measurements.stddev(), reference.measurements.stddev(), ) { - (Some(result_stddev), Some(fastest_stddev)) => Some( + (Some(result_stddev), Some(reference_stddev)) => Some( ratio * ((result_stddev / result.mean_wall_clock_time()) .powi(uom::typenum::P2::new()) - + (fastest_stddev / reference.mean_wall_clock_time()) + + (reference_stddev / reference.mean_wall_clock_time()) .powi(uom::typenum::P2::new())) .sqrt(), ), @@ -192,3 +195,72 @@ fn test_compute_relative_speed_for_zero_times() { assert!(annotated_results.is_none()); } + +#[test] +fn reference_identity_distinguishes_equal_results() { + let results = vec![create_result("same", 1.0), create_result("same", 1.0)]; + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + + assert!(entries[0].is_reference); + assert!(!entries[1].is_reference); +} + +#[test] +fn reference_ratios_indicate_faster_and_slower_runs() { + use approx::assert_relative_eq; + + let results = vec![ + create_result("faster", 1.0), + create_result("reference", 2.0), + create_result("slower", 3.0), + ]; + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + + assert_relative_eq!(entries[0].relative_speed, 2.0); + assert_relative_eq!(entries[1].relative_speed, 1.0); + assert_relative_eq!(entries[2].relative_speed, 1.5); + assert!(!entries[0].is_reference); + assert!(entries[1].is_reference); + assert!(!entries[2].is_reference); +} + +#[test] +fn reference_ratios_handle_zero_times() { + let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; + for result in &mut results { + let measurement = result.measurements.measurements[0]; + result.measurements.measurements.push(measurement); + } + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + assert_eq!(entries[1].relative_speed, f64::INFINITY); + + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + assert_eq!(entries[0].relative_speed, f64::INFINITY); + assert_eq!(entries[0].relative_speed_stddev, None); + assert_eq!(entries[1].relative_speed, 1.0); +} + +#[test] +fn reference_ratio_propagates_uncertainty() { + use crate::benchmark::measurement::Measurement; + use crate::quantity::second; + use approx::assert_relative_eq; + + let mut results = vec![ + create_result("faster", 1.0), + create_result("reference", 3.0), + ]; + for (result, times) in results.iter_mut().zip([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0]]) { + result.measurements.measurements = times + .iter() + .copied() + .map(|seconds| Measurement { + time_wall_clock: Time::new::(seconds), + ..Default::default() + }) + .collect(); + } + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + assert_relative_eq!(entries[0].relative_speed, 2.0); + assert_relative_eq!(entries[0].relative_speed_stddev.unwrap(), 1.118033988749895); +} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index ce5fd81..76cba13 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -44,6 +44,7 @@ impl<'a> Scheduler<'a> { .options .reference_command .as_ref() + .filter(|_| !self.options.parameterized_mode) .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd)); executor.calibrate()?; @@ -54,7 +55,8 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. - self.export_manager.write_results(&self.results, true)?; + self.export_manager + .write_results(&self.results, true, self.reference_index())?; } Ok(()) @@ -70,10 +72,8 @@ impl<'a> Scheduler<'a> { } let reference = self - .options - .reference_command - .as_ref() - .map(|_| &self.results[0]) + .reference_index() + .map(|index| &self.results[index]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( @@ -130,9 +130,20 @@ impl<'a> Scheduler<'a> { console_writeln!(stdout, "{}", "Relative speed comparison".bold())?; for item in annotated_results { + let relationship = if self.reference_index().is_none() { + "" + } else if item.is_reference { + " (reference)" + } else { + match item.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Equal => " (same speed)", + Ordering::Greater => " (slower)", + } + }; console_writeln!( stdout, - " {}{} {}", + " {}{} {}{}", format!("{:10.2}", item.relative_speed).bold().green(), if item.is_reference { " ".into() @@ -142,6 +153,7 @@ impl<'a> Scheduler<'a> { " ".into() }, item.result.command_with_unused_parameters, + relationship, )?; } } @@ -162,8 +174,16 @@ impl<'a> Scheduler<'a> { Ok(()) } + fn reference_index(&self) -> Option { + self.options + .reference_command + .as_ref() + .map(|_| self.options.parameterized_reference_index.unwrap_or(0)) + } + pub fn final_export(&self) -> Result<()> { - self.export_manager.write_results(&self.results, false) + self.export_manager + .write_results(&self.results, false, self.reference_index()) } } diff --git a/src/cli.rs b/src/cli.rs index 39a2238..0a38675 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -105,8 +105,11 @@ fn build_command() -> Command { .action(ArgAction::Set) .value_name("CMD") .help( - "The reference command for the relative comparison of results. \ - If this is unset, results are compared with the fastest command as reference." + "The reference for the relative comparison of results. Without parameters, \ + CMD is run as a separate reference command. With --parameter-scan or \ + --parameter-list, CMD must exactly match one unique benchmark name as \ + shown in the output. If this is unset, results are compared with the \ + fastest command as reference." ) ) .arg( @@ -114,7 +117,8 @@ fn build_command() -> Command { .long("reference-name") .action(ArgAction::Set) .value_name("CMD") - .help("Give a meaningful name to the reference command.") + .help("Give a meaningful name to the reference command. This cannot be used \ + with parameterized benchmarks; use --command-name instead.") .requires("reference") ) .arg( diff --git a/src/export/csv.rs b/src/export/csv.rs index 3932f92..15cc3a5 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -18,6 +18,7 @@ impl Exporter for CsvExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { const CSV_UNIT: TimeUnit = TimeUnit::Second; const CSV_PRECISION: usize = 6; @@ -172,7 +173,7 @@ fn test_csv() { let actual = String::from_utf8( exporter - .serialize(&results, Some(TimeUnit::Second), SortOrder::Command) + .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None) .unwrap(), ) .unwrap(); diff --git a/src/export/json.rs b/src/export/json.rs index 6c9b21d..c5b6a84 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -22,6 +22,7 @@ impl Exporter for JsonExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); if let Ok(ref mut content) = output { diff --git a/src/export/markup.rs b/src/export/markup.rs index f3f8711..d5b2daa 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -1,10 +1,11 @@ use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed; use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed}; use crate::options::SortOrder; -use crate::quantity::{FormatQuantity, IsUnit, TimeUnit}; +use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero}; use super::Exporter; use anyhow::Result; +use std::cmp::Ordering; pub enum Alignment { Left, @@ -16,6 +17,8 @@ pub trait MarkupExporter { &self, entries: &[BenchmarkResultWithRelativeSpeed], unit: TimeUnit, + reference_pending: bool, + explicit_reference: bool, ) -> String { // prepare table header strings let notation = format!("[{}]", unit.short_name()); @@ -58,14 +61,35 @@ pub trait MarkupExporter { }; let min_str = measurement.measurements.min().format_value(unit); let max_str = measurement.measurements.max().format_value(unit); - let rel_str = format!("{:.2}", entry.relative_speed); - let rel_stddev_str = if entry.is_reference { + // The ratio of two zero times is undefined, even if they compare equal. + let relative_unavailable = reference_pending + || (explicit_reference + && !entry.is_reference + && entry.relative_ordering == Ordering::Equal + && measurement.mean_wall_clock_time() == Time::zero()); + let rel_str = if relative_unavailable { + "N/A".to_string() + } else { + format!("{:.2}", entry.relative_speed) + }; + let rel_stddev_str = if relative_unavailable || entry.is_reference { "".into() } else if let Some(stddev) = entry.relative_speed_stddev { format!(" ± {stddev:.2}") } else { "".into() }; + let rel_direction = if relative_unavailable || !explicit_reference { + "" + } else if entry.is_reference { + " (reference)" + } else { + match entry.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Greater => " (slower)", + Ordering::Equal => " (same speed)", + } + }; // prepare table row entries table.push_str(&self.table_row(&[ @@ -73,7 +97,7 @@ pub trait MarkupExporter { &format!("{mean_str}{stddev_str}"), &min_str, &max_str, - &format!("{rel_str}{rel_stddev_str}"), + &format!("{rel_str}{rel_stddev_str}{rel_direction}"), ])) } @@ -114,11 +138,20 @@ impl Exporter for T { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result> { let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); - let entries = relative_speed::compute(results, sort_order); + // Do not report ratios against another benchmark while the selected + // reference is still pending in an intermediate export. + let reference_pending = reference_index.is_some_and(|i| i >= results.len()); + let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) { + relative_speed::compute_relative_speeds(results, reference, sort_order) + } else { + relative_speed::compute(results, sort_order) + }; - let table = self.table_results(&entries, unit); + let table = + self.table_results(&entries, unit, reference_pending, reference_index.is_some()); Ok(table.as_bytes().to_vec()) } } diff --git a/src/export/mod.rs b/src/export/mod.rs index c8d51d7..c8dd9a0 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -51,6 +51,7 @@ trait Exporter { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result>; } @@ -130,11 +131,18 @@ impl ExportManager { /// results are written to all file targets (to always have them up to date, even /// if a benchmark fails). In the latter case, we only print to stdout targets (in /// order not to clutter the output of hyperfine with intermediate results). - pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { + /// `reference_index` refers to the complete benchmark sequence, so it can + /// point past `results` while that benchmark is still pending. + pub fn write_results( + &self, + results: &[BenchmarkResult], + intermediate: bool, + reference_index: Option, + ) -> Result<()> { for e in &self.exporters { let content = || { e.exporter - .serialize(results, self.time_unit, self.sort_order) + .serialize(results, self.time_unit, self.sort_order, reference_index) }; match e.target { diff --git a/src/export/tests.rs b/src/export/tests.rs index 9566f9d..098226f 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -12,9 +12,174 @@ fn get_output( results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, +) -> String { + get_output_with_reference::(results, unit, sort_order, None) +} + +fn get_output_with_reference( + results: &[BenchmarkResult], + unit: Option, + sort_order: SortOrder, + reference_index: Option, ) -> String { let exporter = E::default(); - String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap() + String::from_utf8( + exporter + .serialize(results, unit, sort_order, reference_index) + .unwrap(), + ) + .unwrap() +} + +#[test] +fn test_markup_exports_identify_explicit_reference_and_direction() { + let benchmark = |name: &str, seconds| { + let measurement = Measurement { + time_wall_clock: Time::new::(seconds), + ..Measurement::default() + }; + BenchmarkResult { + command: name.to_owned(), + command_with_unused_parameters: name.to_owned(), + measurements: Measurements::new(vec![measurement, measurement]), + ..BenchmarkResult::default() + } + }; + let results = [ + benchmark("faster", 1.0), + benchmark("reference", 2.0), + benchmark("equal", 2.0), + benchmark("slower", 3.0), + ]; + + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + ]; + for output in outputs { + for expected in [ + "2.00 ± 0.00 (faster)", + "1.00 (reference)", + "1.00 ± 0.00 (same speed)", + "1.50 ± 0.00 (slower)", + ] { + assert!(output.contains(expected), "{}", output); + } + } + } +} + +#[test] +fn test_markup_exports_leave_pending_reference_ratios_unavailable() { + let results = [BenchmarkResult { + command: "sleep 1".into(), + command_with_unused_parameters: "sleep 1".into(), + measurements: Measurements::new(vec![Measurement { + time_wall_clock: Time::new::(1.0), + ..Measurement::default() + }]), + ..BenchmarkResult::default() + }]; + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + ]; + for output in outputs { + assert!(output.contains("N/A"), "{}", output); + assert!(!output.contains("(reference)"), "{}", output); + } + } +} + +#[test] +fn test_markup_exports_with_zero_reference() { + let benchmark = |name: &str, seconds| BenchmarkResult { + command: name.to_owned(), + command_with_unused_parameters: name.to_owned(), + measurements: Measurements::new(vec![Measurement { + time_wall_clock: Time::new::(seconds), + ..Measurement::default() + }]), + ..BenchmarkResult::default() + }; + let results = [ + benchmark("other zero", 0.0), + benchmark("reference", 0.0), + benchmark("slower", 1.0), + ]; + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + for reference in [Some(1), Some(3), None] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + ]; + for output in outputs { + match reference { + Some(1) => { + assert_eq!(output.matches("N/A").count(), 1, "{}", output); + assert!(output.contains("1.00 (reference)"), "{}", output); + assert!(output.contains("inf (slower)"), "{}", output); + } + Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output), + None => { + assert!(!output.contains("N/A"), "{}", output); + assert!(output.contains("inf"), "{}", output); + assert!(!output.contains("(reference)"), "{}", output); + } + } + } + } + } } /// Ensure the makrup output includes the table header and the multiple diff --git a/src/options.rs b/src/options.rs index 0b902d5..659b976 100644 --- a/src/options.rs +++ b/src/options.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::{cmp, env, fmt, io}; -use anyhow::ensure; +use anyhow::{bail, ensure}; use clap::ArgMatches; use crate::command::Commands; @@ -214,6 +214,12 @@ pub struct Options { // Name of the reference command pub reference_name: Option, + // Whether commands are expanded from --parameter-scan or --parameter-list. + pub parameterized_mode: bool, + + // Index of the selected command in a parameterized run, if any. + pub parameterized_reference_index: Option, + /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -257,6 +263,8 @@ impl Default for Options { command_failure_action: CmdFailureAction::RaiseError, reference_command: None, reference_name: None, + parameterized_mode: false, + parameterized_reference_index: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -309,6 +317,8 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); + options.parameterized_mode = matches.get_many::("parameter-scan").is_some() + || matches.get_many::("parameter-list").is_some(); options.reference_command = matches.get_one::("reference").map(String::from); options.reference_name = matches .get_one::("reference-name") @@ -464,7 +474,29 @@ impl Options { } pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { - let has_reference_command = self.reference_command.is_some(); + if self.parameterized_mode { + if let Some(reference) = &self.reference_command { + ensure!( + self.reference_name.is_none(), + "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark" + ); + let mut matches = commands + .iter() + .enumerate() + .filter(|(_, command)| command.get_name_with_unused_parameters() == *reference); + let Some((index, _)) = matches.next() else { + bail!( + "Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name." + ); + }; + ensure!( + matches.next().is_none(), + "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names." + ); + self.parameterized_reference_index = Some(index); + } + } + let has_reference_command = self.reference_command.is_some() && !self.parameterized_mode; let num_commands = commands.num_commands(has_reference_command); if let Some(preparation_command) = &self.preparation_command { diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5..b097e98 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -384,16 +384,14 @@ fn reference_is_executed_first() { } #[test] -fn reference_is_executed_first_parameter_value() { +fn parameterized_reference_keeps_parameter_order() { ExecutionOrderTest::new() .arg("--runs=2") - .reference("reference") + .reference("command 2") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("reference") - .expect_output("reference") .expect_output("command 1") .expect_output("command 1") .expect_output("command 2") diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e6832b2..2256585 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -705,6 +705,40 @@ fn shows_benchmark_comparison_relative_to_reference() { "); } +#[test] +fn command_sorted_comparison_labels_explicit_reference() { + for (args, expected) in [ + ( + vec!["--reference", "sleep 2", "sleep 1", "sleep 3"], + vec![ + "1.00 sleep 2 (reference)", + "2.00 ± 0.00 sleep 1 (faster)", + "1.50 ± 0.00 sleep 3 (slower)", + ], + ), + ( + vec!["--reference", "sleep 2", "sleep 2"], + vec![ + "1.00 sleep 2 (reference)", + "1.00 ± 0.00 sleep 2 (same speed)", + ], + ), + ] { + let output = hyperfine_debug() + .args(["--style=basic", "--runs=2", "--sort=command"]) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; + assert_eq!( + comparison.lines().map(str::trim).collect::>(), + expected + ); + } +} + #[test] fn shows_reference_name() { let _settings = snapshot_settings().bind_to_scope(); @@ -796,50 +830,453 @@ fn rejects_negative_parameter_steps() { } #[test] -fn performs_reference_and_all_benchmarks_in_parameter_scan() { - let _settings = snapshot_settings().bind_to_scope(); - assert_cmd_snapshot!(hyperfine_debug() - .arg("--style=basic") - .arg("--reference=sleep 25") - .arg("--parameter-scan") - .arg("time") - .arg("30") - .arg("45") - .arg("--parameter-step-size") - .arg("5") - .arg("sleep {time}"), @r" - success: true - exit_code: 0 - ----- stdout ----- - Benchmark 1: sleep 25 - Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 25.000 s … 25.000 s 10 runs +fn selects_reference_from_parameter_scan() { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--reference=sleep 35", + "--parameter-scan", + "time", + "30", + "45", + "--parameter-step-size", + "5", + "sleep {time}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let benchmarks: Vec<_> = stdout + .lines() + .filter(|line| line.starts_with("Benchmark ")) + .collect(); + assert_eq!( + benchmarks, + [ + "Benchmark 1: sleep 30", + "Benchmark 2: sleep 35", + "Benchmark 3: sleep 40", + "Benchmark 4: sleep 45", + ] + ); + assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout); +} - Benchmark 2: sleep 30 - Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 30.000 s … 30.000 s 10 runs +#[cfg(unix)] +#[test] +fn selects_reference_with_parameterized_prepare() { + let directory = tempfile::tempdir().unwrap(); + let csv_path = directory.path().join("results.csv"); + let json_path = directory.path().join("results.json"); + let output = hyperfine_raw_command() + .args([ + "--style=basic", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.2,0.4,0.6", + "--prepare", + "sleep {delay}", + "--reference", + "echo a (delay = 0.4)", + "--export-csv", + ]) + .arg(&csv_path) + .arg("--export-json") + .arg(&json_path) + .arg("echo a") + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 3); + assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout); - Benchmark 3: sleep 35 - Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 35.000 s … 35.000 s 10 runs + let mut csv = csv::Reader::from_path(csv_path).unwrap(); + assert!(csv + .headers() + .unwrap() + .iter() + .any(|header| header == "parameter_delay")); + let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect(); + assert_eq!(records.len(), 3); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap(); + let results = json["results"].as_array().unwrap(); + assert_eq!(results.len(), 3); + assert_eq!( + results + .iter() + .map(|result| result["parameters"]["delay"].as_str().unwrap()) + .collect::>(), + ["0.2", "0.4", "0.6"] + ); +} - Benchmark 4: sleep 40 - Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 40.000 s … 40.000 s 10 runs +#[test] +fn selects_reference_by_custom_benchmark_name() { + for (command, name, reference) in [ + ("sleep {x}", "case-{x}", "case-2"), + ("sleep 1", "case-{x}", "case-2 (x = 2)"), + ] { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "-L", + "x", + "1,2", + "--command-name", + name, + "--reference", + reference, + command, + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 2); + assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout); + } +} - Benchmark 5: sleep 45 - Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 45.000 s … 45.000 s 10 runs +#[test] +fn selects_reference_from_multiple_commands_and_parameters() { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "-L", + "x", + "1,2", + "-L", + "y", + "3,4", + "--reference", + "sleep 4 (x = 2)", + "sleep {x}", + "sleep {y}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 8); + assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout); +} - Summary - sleep 25 ran - 1.20 ± 0.00 times faster than sleep 30 - 1.40 ± 0.00 times faster than sleep 35 - 1.60 ± 0.00 times faster than sleep 40 - 1.80 ± 0.00 times faster than sleep 45 +#[test] +fn rejects_unmatched_parameterized_reference_before_creating_export() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.csv"); + let marker_path = directory.path().join("setup-ran"); + for (args, benchmark) in [ + (vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"), + ( + vec!["--reference", "sleep 1", "-P", "secs", "2", "3"], + "sleep {secs}", + ), + ] { + std::fs::write(&export_path, "previous contents").unwrap(); + hyperfine() + .args(args) + .arg("--setup") + .arg(format!("echo touched > \"{}\"", marker_path.display())) + .arg("--export-csv") + .arg(&export_path) + .arg(benchmark) + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "does not match any parameterized benchmark", + )); - ----- stderr ----- - "); + assert_eq!( + std::fs::read_to_string(&export_path).unwrap(), + "previous contents" + ); + assert!(!marker_path.exists()); + } +} + +#[test] +fn rejects_ambiguous_parameterized_reference() { + for args in [ + vec![ + "-L", + "x", + "1,2", + "--command-name", + "case", + "--reference", + "case", + "sleep {x}", + ], + vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"], + ] { + hyperfine_debug() + .args(args) + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "matches multiple parameterized benchmarks", + )); + } +} + +#[test] +fn rejects_reference_name_with_parameterized_reference() { + hyperfine_debug() + .args([ + "-L", + "x", + "1,2", + "--reference", + "sleep 1", + "--reference-name", + "baseline", + "sleep {x}", + ]) + .assert() + .failure() + .stderr(predicate::str::contains( + "--reference-name cannot be used with a parameterized reference", + )); +} + +#[test] +fn parameterized_reference_preserves_per_benchmark_hook_order() { + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("hooks.log"); + let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display()); + let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display()); + + hyperfine() + .args(["--runs=1", "-L", "x", "1,2,3"]) + .arg("--command-name") + .arg("case-{x}") + .arg("--reference") + .arg("case-2") + .arg("--prepare") + .arg(prepare("prepare-1")) + .arg("--prepare") + .arg(prepare("prepare-2")) + .arg("--prepare") + .arg(prepare("prepare-3")) + .arg(command) + .assert() + .success(); + + let contents = std::fs::read_to_string(log_path).unwrap(); + assert_eq!( + contents.lines().map(str::trim).collect::>(), + [ + "prepare-1", + "benchmark-1", + "prepare-2", + "benchmark-2", + "prepare-3", + "benchmark-3" + ] + ); +} + +#[test] +fn parameterized_reference_preserves_per_benchmark_output_order() { + let directory = tempfile::tempdir().unwrap(); + let output_paths = [ + directory.path().join("one.txt"), + directory.path().join("two.txt"), + directory.path().join("three.txt"), + ]; + let mut command = hyperfine(); + command.args([ + "--runs=1", + "-L", + "x", + "1,2,3", + "--command-name", + "case-{x}", + "--reference", + "case-2", + ]); + for path in &output_paths { + command.arg("--output").arg(path); + } + command.arg("echo benchmark-{x}").assert().success(); + + for (path, expected) in output_paths + .iter() + .zip(["benchmark-1", "benchmark-2", "benchmark-3"]) + { + assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected); + } +} + +#[cfg(unix)] +#[test] +fn intermediate_markdown_waits_for_selected_reference() { + let directory = tempfile::tempdir().unwrap(); + for (reference, preparations, reference_available) in [ + ("sleep 0.03", ["true", "false", "true"], false), + ("sleep 0.02", ["true", "true", "false"], true), + ] { + let export_path = directory.path().join("results.md"); + let mut command = hyperfine(); + command.args([ + "--style=none", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.01,0.02,0.03", + "--reference", + reference, + ]); + for preparation in preparations { + command.arg("--prepare").arg(preparation); + } + command + .arg("--export-markdown") + .arg(&export_path) + .arg("sleep {delay}") + .assert() + .failure() + .stderr(predicate::str::contains( + "The preparation command terminated", + )); + let markdown = std::fs::read_to_string(&export_path).unwrap(); + if reference_available { + assert!(!markdown.contains("N/A"), "{}", markdown); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.02")) + .unwrap(); + assert!(row.ends_with("| 1.00 (reference) |"), "{}", row); + } else { + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.01")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); + } + } +} + +#[test] +fn markdown_export_uses_selected_parameterized_reference() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "-L", + "x", + "3,2,1", + "--reference", + "sleep 2", + "sleep {x}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let rows: Vec<_> = stdout + .lines() + .filter(|line| line.contains("sleep ")) + .collect(); + if sort_order == "command" { + assert!(rows[0].contains("sleep 3")); + assert!(rows[1].contains("sleep 2")); + assert!(rows[2].contains("sleep 1")); + } else { + assert!(rows[0].contains("sleep 1")); + assert!(rows[1].contains("sleep 2")); + assert!(rows[2].contains("sleep 3")); + } + for (command, relative) in [ + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (reference)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = rows.iter().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } +} + +#[test] +fn markdown_export_uses_standalone_reference() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "--reference", + "sleep 2", + "--reference-name", + "baseline", + "sleep 1", + "sleep 2", + "sleep 3", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("baseline", "1.00 (reference)"), + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (same speed)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } +} + +#[test] +fn markdown_export_without_explicit_reference_is_unchanged() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "sleep 3", + "sleep 1", + "sleep 2", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("sleep 1", "1.00"), + ("sleep 2", "2.00"), + ("sleep 3", "3.00"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } } #[test] From 463a8896383d34c997df767554ca5e11a6c17f5f Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 15:07:46 +0200 Subject: [PATCH 2/4] Simplify parameterized reference handling --- README.md | 10 - src/benchmark/relative_speed.rs | 74 ++--- src/benchmark/scheduler.rs | 27 +- src/export/markup.rs | 43 +-- src/export/tests.rs | 167 +--------- src/options.rs | 29 +- tests/execution_order_tests.rs | 18 -- tests/integration_tests.rs | 535 +++++++++----------------------- 8 files changed, 210 insertions(+), 693 deletions(-) diff --git a/README.md b/README.md index f92d4bc..ca4f06c 100644 --- a/README.md +++ b/README.md @@ -98,16 +98,6 @@ hyperfine \ 'python main.py' ``` -To compare a parameterized benchmark against a particular run, pass its exact printed name to -`--reference`. For example: -```sh -hyperfine -L delay 0.2,0.4,0.6 --prepare 'sleep {delay}' --reference 'echo a (delay = 0.4)' 'echo a' -``` -The name must identify exactly one benchmark, including any `--command-name` and displayed unused -parameters. `--reference-name` cannot be used with parameterized benchmarks; use `--command-name` -to name them instead. Without `--parameter-scan` or `--parameter-list`, `--reference` runs a -separate reference command. - If you need a unique value for each individual run of a benchmark command, hyperfine also exposes the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself: ```sh diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 4b03c3e..e62a591 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -16,6 +16,20 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { pub relative_ordering: Ordering, } +impl BenchmarkResultWithRelativeSpeed<'_> { + pub fn reference_label(&self) -> &'static str { + if self.is_reference { + " (reference)" + } else { + match self.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Equal => " (same speed)", + Ordering::Greater => " (slower)", + } + } + } +} + pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean_wall_clock_time() .partial_cmp(&r.mean_wall_clock_time()) @@ -29,7 +43,7 @@ pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { .expect("at least one benchmark result") } -pub fn compute_relative_speeds<'a>( +fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, @@ -125,14 +139,14 @@ pub fn compute_with_check( Some(compute_relative_speeds(results, fastest, sort_order)) } -/// Same as compute_with_check, potentially resulting in relative speeds of infinity -pub fn compute( - results: &[BenchmarkResult], +/// Compute relative speeds against the given reference, or the fastest result. +pub fn compute<'a>( + results: &'a [BenchmarkResult], sort_order: SortOrder, -) -> Vec> { - let fastest = fastest_of(results); - - compute_relative_speeds(results, fastest, sort_order) + reference: Option<&'a BenchmarkResult>, +) -> Vec> { + let reference = reference.unwrap_or_else(|| fastest_of(results)); + compute_relative_speeds(results, reference, sort_order) } #[cfg(test)] @@ -205,25 +219,6 @@ fn reference_identity_distinguishes_equal_results() { assert!(!entries[1].is_reference); } -#[test] -fn reference_ratios_indicate_faster_and_slower_runs() { - use approx::assert_relative_eq; - - let results = vec![ - create_result("faster", 1.0), - create_result("reference", 2.0), - create_result("slower", 3.0), - ]; - let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); - - assert_relative_eq!(entries[0].relative_speed, 2.0); - assert_relative_eq!(entries[1].relative_speed, 1.0); - assert_relative_eq!(entries[2].relative_speed, 1.5); - assert!(!entries[0].is_reference); - assert!(entries[1].is_reference); - assert!(!entries[2].is_reference); -} - #[test] fn reference_ratios_handle_zero_times() { let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; @@ -239,28 +234,3 @@ fn reference_ratios_handle_zero_times() { assert_eq!(entries[0].relative_speed_stddev, None); assert_eq!(entries[1].relative_speed, 1.0); } - -#[test] -fn reference_ratio_propagates_uncertainty() { - use crate::benchmark::measurement::Measurement; - use crate::quantity::second; - use approx::assert_relative_eq; - - let mut results = vec![ - create_result("faster", 1.0), - create_result("reference", 3.0), - ]; - for (result, times) in results.iter_mut().zip([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0]]) { - result.measurements.measurements = times - .iter() - .copied() - .map(|seconds| Measurement { - time_wall_clock: Time::new::(seconds), - ..Default::default() - }) - .collect(); - } - let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); - assert_relative_eq!(entries[0].relative_speed, 2.0); - assert_relative_eq!(entries[0].relative_speed_stddev.unwrap(), 1.118033988749895); -} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 76cba13..db036d6 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -44,7 +44,6 @@ impl<'a> Scheduler<'a> { .options .reference_command .as_ref() - .filter(|_| !self.options.parameterized_mode) .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd)); executor.calibrate()?; @@ -56,7 +55,7 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. self.export_manager - .write_results(&self.results, true, self.reference_index())?; + .write_results(&self.results, true, self.options.reference_index)?; } Ok(()) @@ -72,7 +71,8 @@ impl<'a> Scheduler<'a> { } let reference = self - .reference_index() + .options + .reference_index .map(|index| &self.results[index]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); @@ -130,16 +130,10 @@ impl<'a> Scheduler<'a> { console_writeln!(stdout, "{}", "Relative speed comparison".bold())?; for item in annotated_results { - let relationship = if self.reference_index().is_none() { - "" - } else if item.is_reference { - " (reference)" + let relationship = if self.options.reference_index.is_some() { + item.reference_label() } else { - match item.relative_ordering { - Ordering::Less => " (faster)", - Ordering::Equal => " (same speed)", - Ordering::Greater => " (slower)", - } + "" }; console_writeln!( stdout, @@ -174,16 +168,9 @@ impl<'a> Scheduler<'a> { Ok(()) } - fn reference_index(&self) -> Option { - self.options - .reference_command - .as_ref() - .map(|_| self.options.parameterized_reference_index.unwrap_or(0)) - } - pub fn final_export(&self) -> Result<()> { self.export_manager - .write_results(&self.results, false, self.reference_index()) + .write_results(&self.results, false, self.options.reference_index) } } diff --git a/src/export/markup.rs b/src/export/markup.rs index d5b2daa..0eea533 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -67,28 +67,19 @@ pub trait MarkupExporter { && !entry.is_reference && entry.relative_ordering == Ordering::Equal && measurement.mean_wall_clock_time() == Time::zero()); - let rel_str = if relative_unavailable { + let relative = if relative_unavailable { "N/A".to_string() } else { - format!("{:.2}", entry.relative_speed) - }; - let rel_stddev_str = if relative_unavailable || entry.is_reference { - "".into() - } else if let Some(stddev) = entry.relative_speed_stddev { - format!(" ± {stddev:.2}") - } else { - "".into() - }; - let rel_direction = if relative_unavailable || !explicit_reference { - "" - } else if entry.is_reference { - " (reference)" - } else { - match entry.relative_ordering { - Ordering::Less => " (faster)", - Ordering::Greater => " (slower)", - Ordering::Equal => " (same speed)", - } + let stddev = match (entry.is_reference, entry.relative_speed_stddev) { + (false, Some(stddev)) => format!(" ± {stddev:.2}"), + _ => String::new(), + }; + let label = if explicit_reference { + entry.reference_label() + } else { + "" + }; + format!("{:.2}{stddev}{label}", entry.relative_speed) }; // prepare table row entries @@ -97,7 +88,7 @@ pub trait MarkupExporter { &format!("{mean_str}{stddev_str}"), &min_str, &max_str, - &format!("{rel_str}{rel_stddev_str}{rel_direction}"), + &relative, ])) } @@ -144,11 +135,11 @@ impl Exporter for T { // Do not report ratios against another benchmark while the selected // reference is still pending in an intermediate export. let reference_pending = reference_index.is_some_and(|i| i >= results.len()); - let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) { - relative_speed::compute_relative_speeds(results, reference, sort_order) - } else { - relative_speed::compute(results, sort_order) - }; + let entries = relative_speed::compute( + results, + sort_order, + reference_index.and_then(|i| results.get(i)), + ); let table = self.table_results(&entries, unit, reference_pending, reference_index.is_some()); diff --git a/src/export/tests.rs b/src/export/tests.rs index 098226f..abe340a 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -12,174 +12,9 @@ fn get_output( results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, -) -> String { - get_output_with_reference::(results, unit, sort_order, None) -} - -fn get_output_with_reference( - results: &[BenchmarkResult], - unit: Option, - sort_order: SortOrder, - reference_index: Option, ) -> String { let exporter = E::default(); - String::from_utf8( - exporter - .serialize(results, unit, sort_order, reference_index) - .unwrap(), - ) - .unwrap() -} - -#[test] -fn test_markup_exports_identify_explicit_reference_and_direction() { - let benchmark = |name: &str, seconds| { - let measurement = Measurement { - time_wall_clock: Time::new::(seconds), - ..Measurement::default() - }; - BenchmarkResult { - command: name.to_owned(), - command_with_unused_parameters: name.to_owned(), - measurements: Measurements::new(vec![measurement, measurement]), - ..BenchmarkResult::default() - } - }; - let results = [ - benchmark("faster", 1.0), - benchmark("reference", 2.0), - benchmark("equal", 2.0), - benchmark("slower", 3.0), - ]; - - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - ]; - for output in outputs { - for expected in [ - "2.00 ± 0.00 (faster)", - "1.00 (reference)", - "1.00 ± 0.00 (same speed)", - "1.50 ± 0.00 (slower)", - ] { - assert!(output.contains(expected), "{}", output); - } - } - } -} - -#[test] -fn test_markup_exports_leave_pending_reference_ratios_unavailable() { - let results = [BenchmarkResult { - command: "sleep 1".into(), - command_with_unused_parameters: "sleep 1".into(), - measurements: Measurements::new(vec![Measurement { - time_wall_clock: Time::new::(1.0), - ..Measurement::default() - }]), - ..BenchmarkResult::default() - }]; - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - Some(1), - ), - ]; - for output in outputs { - assert!(output.contains("N/A"), "{}", output); - assert!(!output.contains("(reference)"), "{}", output); - } - } -} - -#[test] -fn test_markup_exports_with_zero_reference() { - let benchmark = |name: &str, seconds| BenchmarkResult { - command: name.to_owned(), - command_with_unused_parameters: name.to_owned(), - measurements: Measurements::new(vec![Measurement { - time_wall_clock: Time::new::(seconds), - ..Measurement::default() - }]), - ..BenchmarkResult::default() - }; - let results = [ - benchmark("other zero", 0.0), - benchmark("reference", 0.0), - benchmark("slower", 1.0), - ]; - for sort_order in [SortOrder::Command, SortOrder::MeanTime] { - for reference in [Some(1), Some(3), None] { - let outputs = [ - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - get_output_with_reference::( - &results, - Some(TimeUnit::Second), - sort_order, - reference, - ), - ]; - for output in outputs { - match reference { - Some(1) => { - assert_eq!(output.matches("N/A").count(), 1, "{}", output); - assert!(output.contains("1.00 (reference)"), "{}", output); - assert!(output.contains("inf (slower)"), "{}", output); - } - Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output), - None => { - assert!(!output.contains("N/A"), "{}", output); - assert!(output.contains("inf"), "{}", output); - assert!(!output.contains("(reference)"), "{}", output); - } - } - } - } - } + String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap() } /// Ensure the makrup output includes the table header and the multiple diff --git a/src/options.rs b/src/options.rs index 659b976..f5a3285 100644 --- a/src/options.rs +++ b/src/options.rs @@ -208,17 +208,14 @@ pub struct Options { /// Whether or not to ignore non-zero exit codes pub command_failure_action: CmdFailureAction, - // Command to use as a reference for relative speed comparison + // Standalone reference command, cleared when an existing benchmark is selected. pub reference_command: Option, // Name of the reference command pub reference_name: Option, - // Whether commands are expanded from --parameter-scan or --parameter-list. - pub parameterized_mode: bool, - - // Index of the selected command in a parameterized run, if any. - pub parameterized_reference_index: Option, + // Index of the reference in the benchmark sequence, if any. + pub reference_index: Option, /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -263,8 +260,7 @@ impl Default for Options { command_failure_action: CmdFailureAction::RaiseError, reference_command: None, reference_name: None, - parameterized_mode: false, - parameterized_reference_index: None, + reference_index: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -317,8 +313,6 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); - options.parameterized_mode = matches.get_many::("parameter-scan").is_some() - || matches.get_many::("parameter-list").is_some(); options.reference_command = matches.get_one::("reference").map(String::from); options.reference_name = matches .get_one::("reference-name") @@ -474,8 +468,12 @@ impl Options { } pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { - if self.parameterized_mode { - if let Some(reference) = &self.reference_command { + if let Some(reference) = &self.reference_command { + if commands + .iter() + .next() + .is_some_and(|command| !command.get_parameters().is_empty()) + { ensure!( self.reference_name.is_none(), "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark" @@ -493,10 +491,13 @@ impl Options { matches.next().is_none(), "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names." ); - self.parameterized_reference_index = Some(index); + self.reference_index = Some(index); + self.reference_command = None; + } else { + self.reference_index = Some(0); } } - let has_reference_command = self.reference_command.is_some() && !self.parameterized_mode; + let has_reference_command = self.reference_command.is_some(); let num_commands = commands.num_commands(has_reference_command); if let Some(preparation_command) = &self.preparation_command { diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index b097e98..e09eb63 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -383,24 +383,6 @@ fn reference_is_executed_first() { .run(); } -#[test] -fn parameterized_reference_keeps_parameter_order() { - ExecutionOrderTest::new() - .arg("--runs=2") - .reference("command 2") - .arg("--parameter-list") - .arg("number") - .arg("1,2,3") - .command("command {number}") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") - .run(); -} - #[test] fn reference_is_executed_separately_from_commands() { ExecutionOrderTest::new() diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2256585..3bf25ff 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -706,37 +706,71 @@ fn shows_benchmark_comparison_relative_to_reference() { } #[test] -fn command_sorted_comparison_labels_explicit_reference() { - for (args, expected) in [ - ( - vec!["--reference", "sleep 2", "sleep 1", "sleep 3"], - vec![ - "1.00 sleep 2 (reference)", - "2.00 ± 0.00 sleep 1 (faster)", - "1.50 ± 0.00 sleep 3 (slower)", - ], - ), - ( - vec!["--reference", "sleep 2", "sleep 2"], - vec![ - "1.00 sleep 2 (reference)", - "1.00 ± 0.00 sleep 2 (same speed)", - ], - ), +fn command_sorted_comparison_and_markup_identify_reference() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.md"); + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "--sort=command", + "--reference", + "sleep 2", + "--reference-name", + "baseline", + "--export-markdown", + ]) + .arg(&export_path) + .args(["sleep 1", "sleep 2", "sleep 3"]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; + for (name, factor) in [ + ("baseline (reference)", "1.00"), + ("sleep 1 (faster)", "2.00"), + ("sleep 2 (same speed)", "1.00"), + ("sleep 3 (slower)", "1.50"), ] { - let output = hyperfine_debug() - .args(["--style=basic", "--runs=2", "--sort=command"]) - .args(args) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; - assert_eq!( - comparison.lines().map(str::trim).collect::>(), - expected - ); + let row = comparison.lines().find(|line| line.contains(name)).unwrap(); + assert!(row.trim_start().starts_with(factor), "{}", row); } + let markdown = std::fs::read_to_string(export_path).unwrap(); + for (command, relative) in [ + ("baseline", "1.00 (reference)"), + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (same speed)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = markdown + .lines() + .find(|line| line.contains(command)) + .unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + + // Equal zero times have no meaningful relative factor. + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "--reference", + "sleep 0", + "--reference-name", + "baseline", + "sleep 0", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let markdown = String::from_utf8(output.stdout).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); } #[test] @@ -829,40 +863,6 @@ fn rejects_negative_parameter_steps() { } } -#[test] -fn selects_reference_from_parameter_scan() { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--reference=sleep 35", - "--parameter-scan", - "time", - "30", - "45", - "--parameter-step-size", - "5", - "sleep {time}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let benchmarks: Vec<_> = stdout - .lines() - .filter(|line| line.starts_with("Benchmark ")) - .collect(); - assert_eq!( - benchmarks, - [ - "Benchmark 1: sleep 30", - "Benchmark 2: sleep 35", - "Benchmark 3: sleep 40", - "Benchmark 4: sleep 45", - ] - ); - assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout); -} - #[cfg(unix)] #[test] fn selects_reference_with_parameterized_prepare() { @@ -892,6 +892,17 @@ fn selects_reference_with_parameterized_prepare() { assert!(output.status.success(), "{:?}", output); let stdout = String::from_utf8(output.stdout).unwrap(); assert_eq!(stdout.matches("Benchmark ").count(), 3); + for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() { + assert!( + stdout.contains(&format!( + "Benchmark {}: echo a (delay = {})", + number + 1, + delay + )), + "{}", + stdout + ); + } assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout); let mut csv = csv::Reader::from_path(csv_path).unwrap(); @@ -900,8 +911,10 @@ fn selects_reference_with_parameterized_prepare() { .unwrap() .iter() .any(|header| header == "parameter_delay")); - let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect(); - assert_eq!(records.len(), 3); + assert_eq!( + csv.records().collect::, _>>().unwrap().len(), + 3 + ); let json: serde_json::Value = serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap(); let results = json["results"].as_array().unwrap(); @@ -916,97 +929,35 @@ fn selects_reference_with_parameterized_prepare() { } #[test] -fn selects_reference_by_custom_benchmark_name() { - for (command, name, reference) in [ - ("sleep {x}", "case-{x}", "case-2"), - ("sleep 1", "case-{x}", "case-2 (x = 2)"), - ] { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--runs=1", - "-L", - "x", - "1,2", - "--command-name", - name, - "--reference", - reference, - command, - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - assert_eq!(stdout.matches("Benchmark ").count(), 2); - assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout); - } -} - -#[test] -fn selects_reference_from_multiple_commands_and_parameters() { - let output = hyperfine_debug() - .args([ - "--style=basic", - "--runs=1", - "-L", - "x", - "1,2", - "-L", - "y", - "3,4", - "--reference", - "sleep 4 (x = 2)", - "sleep {x}", - "sleep {y}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - assert_eq!(stdout.matches("Benchmark ").count(), 8); - assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout); -} - -#[test] -fn rejects_unmatched_parameterized_reference_before_creating_export() { +fn rejects_unmatched_reference_before_export_or_setup() { let directory = tempfile::tempdir().unwrap(); let export_path = directory.path().join("results.csv"); let marker_path = directory.path().join("setup-ran"); - for (args, benchmark) in [ - (vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"), - ( - vec!["--reference", "sleep 1", "-P", "secs", "2", "3"], - "sleep {secs}", - ), - ] { - std::fs::write(&export_path, "previous contents").unwrap(); - hyperfine() - .args(args) - .arg("--setup") - .arg(format!("echo touched > \"{}\"", marker_path.display())) - .arg("--export-csv") - .arg(&export_path) - .arg(benchmark) - .assert() - .failure() - .stdout(predicate::str::is_empty()) - .stderr(predicate::str::contains( - "does not match any parameterized benchmark", - )); + std::fs::write(&export_path, "previous contents").unwrap(); - assert_eq!( - std::fs::read_to_string(&export_path).unwrap(), - "previous contents" - ); - assert!(!marker_path.exists()); - } + hyperfine() + .args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"]) + .arg(format!("echo touched > \"{}\"", marker_path.display())) + .arg("--export-csv") + .arg(&export_path) + .arg("sleep {secs}") + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "does not match any parameterized benchmark", + )); + assert_eq!( + std::fs::read_to_string(export_path).unwrap(), + "previous contents" + ); + assert!(!marker_path.exists()); } #[test] fn rejects_ambiguous_parameterized_reference() { - for args in [ - vec![ + hyperfine_debug() + .args([ "-L", "x", "1,2", @@ -1015,120 +966,21 @@ fn rejects_ambiguous_parameterized_reference() { "--reference", "case", "sleep {x}", - ], - vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"], - ] { - hyperfine_debug() - .args(args) - .assert() - .failure() - .stdout(predicate::str::is_empty()) - .stderr(predicate::str::contains( - "matches multiple parameterized benchmarks", - )); - } -} - -#[test] -fn rejects_reference_name_with_parameterized_reference() { - hyperfine_debug() - .args([ - "-L", - "x", - "1,2", - "--reference", - "sleep 1", - "--reference-name", - "baseline", - "sleep {x}", ]) .assert() .failure() .stderr(predicate::str::contains( - "--reference-name cannot be used with a parameterized reference", + "matches multiple parameterized benchmarks", )); } -#[test] -fn parameterized_reference_preserves_per_benchmark_hook_order() { - let directory = tempfile::tempdir().unwrap(); - let log_path = directory.path().join("hooks.log"); - let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display()); - let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display()); - - hyperfine() - .args(["--runs=1", "-L", "x", "1,2,3"]) - .arg("--command-name") - .arg("case-{x}") - .arg("--reference") - .arg("case-2") - .arg("--prepare") - .arg(prepare("prepare-1")) - .arg("--prepare") - .arg(prepare("prepare-2")) - .arg("--prepare") - .arg(prepare("prepare-3")) - .arg(command) - .assert() - .success(); - - let contents = std::fs::read_to_string(log_path).unwrap(); - assert_eq!( - contents.lines().map(str::trim).collect::>(), - [ - "prepare-1", - "benchmark-1", - "prepare-2", - "benchmark-2", - "prepare-3", - "benchmark-3" - ] - ); -} - -#[test] -fn parameterized_reference_preserves_per_benchmark_output_order() { - let directory = tempfile::tempdir().unwrap(); - let output_paths = [ - directory.path().join("one.txt"), - directory.path().join("two.txt"), - directory.path().join("three.txt"), - ]; - let mut command = hyperfine(); - command.args([ - "--runs=1", - "-L", - "x", - "1,2,3", - "--command-name", - "case-{x}", - "--reference", - "case-2", - ]); - for path in &output_paths { - command.arg("--output").arg(path); - } - command.arg("echo benchmark-{x}").assert().success(); - - for (path, expected) in output_paths - .iter() - .zip(["benchmark-1", "benchmark-2", "benchmark-3"]) - { - assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected); - } -} - #[cfg(unix)] #[test] fn intermediate_markdown_waits_for_selected_reference() { let directory = tempfile::tempdir().unwrap(); - for (reference, preparations, reference_available) in [ - ("sleep 0.03", ["true", "false", "true"], false), - ("sleep 0.02", ["true", "true", "false"], true), - ] { - let export_path = directory.path().join("results.md"); - let mut command = hyperfine(); - command.args([ + let export_path = directory.path().join("results.md"); + hyperfine() + .args([ "--style=none", "--shell=none", "--runs=1", @@ -1136,146 +988,55 @@ fn intermediate_markdown_waits_for_selected_reference() { "delay", "0.01,0.02,0.03", "--reference", - reference, - ]); - for preparation in preparations { - command.arg("--prepare").arg(preparation); - } - command - .arg("--export-markdown") - .arg(&export_path) - .arg("sleep {delay}") - .assert() - .failure() - .stderr(predicate::str::contains( - "The preparation command terminated", - )); - let markdown = std::fs::read_to_string(&export_path).unwrap(); - if reference_available { - assert!(!markdown.contains("N/A"), "{}", markdown); - let row = markdown - .lines() - .find(|line| line.contains("sleep 0.02")) - .unwrap(); - assert!(row.ends_with("| 1.00 (reference) |"), "{}", row); - } else { - let row = markdown - .lines() - .find(|line| line.contains("sleep 0.01")) - .unwrap(); - assert!(row.ends_with("| N/A |"), "{}", row); - } - } + "sleep 0.02", + "--prepare", + "true", + "--prepare", + "false", + "--prepare", + "true", + "--export-markdown", + ]) + .arg(&export_path) + .arg("sleep {delay}") + .assert() + .failure(); + let markdown = std::fs::read_to_string(export_path).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.01")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); } #[test] fn markdown_export_uses_selected_parameterized_reference() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "-L", - "x", - "3,2,1", - "--reference", - "sleep 2", - "sleep {x}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - let rows: Vec<_> = stdout - .lines() - .filter(|line| line.contains("sleep ")) - .collect(); - if sort_order == "command" { - assert!(rows[0].contains("sleep 3")); - assert!(rows[1].contains("sleep 2")); - assert!(rows[2].contains("sleep 1")); - } else { - assert!(rows[0].contains("sleep 1")); - assert!(rows[1].contains("sleep 2")); - assert!(rows[2].contains("sleep 3")); - } - for (command, relative) in [ - ("sleep 1", "2.00 (faster)"), - ("sleep 2", "1.00 (reference)"), - ("sleep 3", "1.50 (slower)"), - ] { - let row = rows.iter().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } - } -} - -#[test] -fn markdown_export_uses_standalone_reference() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "--reference", - "sleep 2", - "--reference-name", - "baseline", - "sleep 1", - "sleep 2", - "sleep 3", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - for (command, relative) in [ - ("baseline", "1.00 (reference)"), - ("sleep 1", "2.00 (faster)"), - ("sleep 2", "1.00 (same speed)"), - ("sleep 3", "1.50 (slower)"), - ] { - let row = stdout.lines().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } - } -} - -#[test] -fn markdown_export_without_explicit_reference_is_unchanged() { - for sort_order in ["command", "mean-time"] { - let output = hyperfine_debug() - .args([ - "--style=none", - "--runs=1", - "--time-unit=second", - "--export-markdown=-", - "--sort", - sort_order, - "sleep 3", - "sleep 1", - "sleep 2", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - for (command, relative) in [ - ("sleep 1", "1.00"), - ("sleep 2", "2.00"), - ("sleep 3", "3.00"), - ] { - let row = stdout.lines().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "-P", + "x", + "1", + "3", + "--command-name", + "case-{x}", + "--reference", + "case-2", + "sleep {x}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("case-1", "2.00 (faster)"), + ("case-2", "1.00 (reference)"), + ("case-3", "1.50 (slower)"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); } } From 92330ce75d001ec0fdfe28482b2384649b6d4a3b Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 15:33:13 +0200 Subject: [PATCH 3/4] Snapshot reference comparison output --- tests/integration_tests.rs | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 3bf25ff..fdc2124 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -707,6 +707,7 @@ fn shows_benchmark_comparison_relative_to_reference() { #[test] fn command_sorted_comparison_and_markup_identify_reference() { + let _settings = snapshot_settings().bind_to_scope(); let directory = tempfile::tempdir().unwrap(); let export_path = directory.path().join("results.md"); let output = hyperfine_debug() @@ -727,15 +728,12 @@ fn command_sorted_comparison_and_markup_identify_reference() { assert!(output.status.success(), "{:?}", output); let stdout = String::from_utf8(output.stdout).unwrap(); let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; - for (name, factor) in [ - ("baseline (reference)", "1.00"), - ("sleep 1 (faster)", "2.00"), - ("sleep 2 (same speed)", "1.00"), - ("sleep 3 (slower)", "1.50"), - ] { - let row = comparison.lines().find(|line| line.contains(name)).unwrap(); - assert!(row.trim_start().starts_with(factor), "{}", row); - } + insta::assert_snapshot!(comparison, @r" + 1.00 baseline (reference) + 2.00 sleep 1 (faster) + 1.00 sleep 2 (same speed) + 1.50 sleep 3 (slower) + "); let markdown = std::fs::read_to_string(export_path).unwrap(); for (command, relative) in [ ("baseline", "1.00 (reference)"), @@ -1011,7 +1009,8 @@ fn intermediate_markdown_waits_for_selected_reference() { #[test] fn markdown_export_uses_selected_parameterized_reference() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .args([ "--style=none", "--runs=1", @@ -1025,19 +1024,20 @@ fn markdown_export_uses_selected_parameterized_reference() { "--reference", "case-2", "sleep {x}", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - for (command, relative) in [ - ("case-1", "2.00 (faster)"), - ("case-2", "1.00 (reference)"), - ("case-3", "1.50 (slower)"), - ] { - let row = stdout.lines().find(|line| line.contains(command)).unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } + ]), @r" + success: true + exit_code: 0 + ----- stdout ----- + + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `case-1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) | + | `case-2` | 2.000 | 2.000 | 2.000 | 1.00 (reference) | + | `case-3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) | + + + ----- stderr ----- + "); } #[test] From 2e6abb70d14a9054cc4812dc2d631a5c8d40d21f Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 15:38:46 +0200 Subject: [PATCH 4/4] Snapshot reference Markdown output --- tests/integration_tests.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index fdc2124..42c23ab 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -735,18 +735,14 @@ fn command_sorted_comparison_and_markup_identify_reference() { 1.50 sleep 3 (slower) "); let markdown = std::fs::read_to_string(export_path).unwrap(); - for (command, relative) in [ - ("baseline", "1.00 (reference)"), - ("sleep 1", "2.00 (faster)"), - ("sleep 2", "1.00 (same speed)"), - ("sleep 3", "1.50 (slower)"), - ] { - let row = markdown - .lines() - .find(|line| line.contains(command)) - .unwrap(); - assert!(row.ends_with(&format!("| {relative} |")), "{}", row); - } + insta::assert_snapshot!(markdown, @r" + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `baseline` | 2.000 | 2.000 | 2.000 | 1.00 (reference) | + | `sleep 1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) | + | `sleep 2` | 2.000 | 2.000 | 2.000 | 1.00 (same speed) | + | `sleep 3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) | + "); // Equal zero times have no meaningful relative factor. let output = hyperfine_debug()