diff --git a/README.md b/README.md index ca4f06c..f92d4bc 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,16 @@ hyperfine \ 'python main.py' ``` +To compare a parameterized benchmark against a particular run, pass its exact printed name to +`--reference`. For example: +```sh +hyperfine -L delay 0.2,0.4,0.6 --prepare 'sleep {delay}' --reference 'echo a (delay = 0.4)' 'echo a' +``` +The name must identify exactly one benchmark, including any `--command-name` and displayed unused +parameters. `--reference-name` cannot be used with parameterized benchmarks; use `--command-name` +to name them instead. Without `--parameter-scan` or `--parameter-list`, `--reference` runs a +separate reference command. + If you need a unique value for each individual run of a benchmark command, hyperfine also exposes the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself: ```sh diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index d565dd3..0136ff8 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -116,13 +116,17 @@ not every time as would happen with the \fB\-\-prepare\fR option. .HP \fB\-\-reference\fR \fICMD\fP .IP -Benchmark \fICMD\fP as the reference for the relative comparison of results. -If this option is not specified, the fastest command is used as the reference. +Use \fICMD\fP as the reference for the relative comparison of results. Without +parameters, benchmark \fICMD\fP as a separate reference command. If this option +is not specified, the fastest command is used as the reference. +With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing +benchmark by its exact printed name. The name must match exactly one benchmark. .HP \fB\-\-reference\-name\fR \fINAME\fP .IP Give a meaningful \fINAME\fP to the reference command. Requires \fB\-\-reference\fR. +Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead. .HP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP .IP diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 711ca2f..4b03c3e 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -29,7 +29,7 @@ pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { .expect("at least one benchmark result") } -fn compute_relative_speeds<'a>( +pub fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, @@ -37,10 +37,13 @@ fn compute_relative_speeds<'a>( let mut results: Vec<_> = results .iter() .map(|result| { - let is_reference = result == reference; + // Separate benchmarks can have identical names and measurements. + let is_reference = std::ptr::eq(result, reference); let relative_ordering = compare_mean_time(result, reference); - if result.mean_wall_clock_time() == Time::zero() { + if result.mean_wall_clock_time() == Time::zero() + || reference.mean_wall_clock_time() == Time::zero() + { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, @@ -64,11 +67,11 @@ fn compute_relative_speeds<'a>( result.measurements.stddev(), reference.measurements.stddev(), ) { - (Some(result_stddev), Some(fastest_stddev)) => Some( + (Some(result_stddev), Some(reference_stddev)) => Some( ratio * ((result_stddev / result.mean_wall_clock_time()) .powi(uom::typenum::P2::new()) - + (fastest_stddev / reference.mean_wall_clock_time()) + + (reference_stddev / reference.mean_wall_clock_time()) .powi(uom::typenum::P2::new())) .sqrt(), ), @@ -192,3 +195,72 @@ fn test_compute_relative_speed_for_zero_times() { assert!(annotated_results.is_none()); } + +#[test] +fn reference_identity_distinguishes_equal_results() { + let results = vec![create_result("same", 1.0), create_result("same", 1.0)]; + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + + assert!(entries[0].is_reference); + assert!(!entries[1].is_reference); +} + +#[test] +fn reference_ratios_indicate_faster_and_slower_runs() { + use approx::assert_relative_eq; + + let results = vec![ + create_result("faster", 1.0), + create_result("reference", 2.0), + create_result("slower", 3.0), + ]; + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + + assert_relative_eq!(entries[0].relative_speed, 2.0); + assert_relative_eq!(entries[1].relative_speed, 1.0); + assert_relative_eq!(entries[2].relative_speed, 1.5); + assert!(!entries[0].is_reference); + assert!(entries[1].is_reference); + assert!(!entries[2].is_reference); +} + +#[test] +fn reference_ratios_handle_zero_times() { + let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; + for result in &mut results { + let measurement = result.measurements.measurements[0]; + result.measurements.measurements.push(measurement); + } + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + assert_eq!(entries[1].relative_speed, f64::INFINITY); + + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + assert_eq!(entries[0].relative_speed, f64::INFINITY); + assert_eq!(entries[0].relative_speed_stddev, None); + assert_eq!(entries[1].relative_speed, 1.0); +} + +#[test] +fn reference_ratio_propagates_uncertainty() { + use crate::benchmark::measurement::Measurement; + use crate::quantity::second; + use approx::assert_relative_eq; + + let mut results = vec![ + create_result("faster", 1.0), + create_result("reference", 3.0), + ]; + for (result, times) in results.iter_mut().zip([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0]]) { + result.measurements.measurements = times + .iter() + .copied() + .map(|seconds| Measurement { + time_wall_clock: Time::new::(seconds), + ..Default::default() + }) + .collect(); + } + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + assert_relative_eq!(entries[0].relative_speed, 2.0); + assert_relative_eq!(entries[0].relative_speed_stddev.unwrap(), 1.118033988749895); +} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index ce5fd81..76cba13 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -44,6 +44,7 @@ impl<'a> Scheduler<'a> { .options .reference_command .as_ref() + .filter(|_| !self.options.parameterized_mode) .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd)); executor.calibrate()?; @@ -54,7 +55,8 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. - self.export_manager.write_results(&self.results, true)?; + self.export_manager + .write_results(&self.results, true, self.reference_index())?; } Ok(()) @@ -70,10 +72,8 @@ impl<'a> Scheduler<'a> { } let reference = self - .options - .reference_command - .as_ref() - .map(|_| &self.results[0]) + .reference_index() + .map(|index| &self.results[index]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( @@ -130,9 +130,20 @@ impl<'a> Scheduler<'a> { console_writeln!(stdout, "{}", "Relative speed comparison".bold())?; for item in annotated_results { + let relationship = if self.reference_index().is_none() { + "" + } else if item.is_reference { + " (reference)" + } else { + match item.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Equal => " (same speed)", + Ordering::Greater => " (slower)", + } + }; console_writeln!( stdout, - " {}{} {}", + " {}{} {}{}", format!("{:10.2}", item.relative_speed).bold().green(), if item.is_reference { " ".into() @@ -142,6 +153,7 @@ impl<'a> Scheduler<'a> { " ".into() }, item.result.command_with_unused_parameters, + relationship, )?; } } @@ -162,8 +174,16 @@ impl<'a> Scheduler<'a> { Ok(()) } + fn reference_index(&self) -> Option { + self.options + .reference_command + .as_ref() + .map(|_| self.options.parameterized_reference_index.unwrap_or(0)) + } + pub fn final_export(&self) -> Result<()> { - self.export_manager.write_results(&self.results, false) + self.export_manager + .write_results(&self.results, false, self.reference_index()) } } diff --git a/src/cli.rs b/src/cli.rs index 39a2238..0a38675 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -105,8 +105,11 @@ fn build_command() -> Command { .action(ArgAction::Set) .value_name("CMD") .help( - "The reference command for the relative comparison of results. \ - If this is unset, results are compared with the fastest command as reference." + "The reference for the relative comparison of results. Without parameters, \ + CMD is run as a separate reference command. With --parameter-scan or \ + --parameter-list, CMD must exactly match one unique benchmark name as \ + shown in the output. If this is unset, results are compared with the \ + fastest command as reference." ) ) .arg( @@ -114,7 +117,8 @@ fn build_command() -> Command { .long("reference-name") .action(ArgAction::Set) .value_name("CMD") - .help("Give a meaningful name to the reference command.") + .help("Give a meaningful name to the reference command. This cannot be used \ + with parameterized benchmarks; use --command-name instead.") .requires("reference") ) .arg( diff --git a/src/export/csv.rs b/src/export/csv.rs index 3932f92..15cc3a5 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -18,6 +18,7 @@ impl Exporter for CsvExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { const CSV_UNIT: TimeUnit = TimeUnit::Second; const CSV_PRECISION: usize = 6; @@ -172,7 +173,7 @@ fn test_csv() { let actual = String::from_utf8( exporter - .serialize(&results, Some(TimeUnit::Second), SortOrder::Command) + .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None) .unwrap(), ) .unwrap(); diff --git a/src/export/json.rs b/src/export/json.rs index 6c9b21d..c5b6a84 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -22,6 +22,7 @@ impl Exporter for JsonExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); if let Ok(ref mut content) = output { diff --git a/src/export/markup.rs b/src/export/markup.rs index f3f8711..d5b2daa 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -1,10 +1,11 @@ use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed; use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed}; use crate::options::SortOrder; -use crate::quantity::{FormatQuantity, IsUnit, TimeUnit}; +use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero}; use super::Exporter; use anyhow::Result; +use std::cmp::Ordering; pub enum Alignment { Left, @@ -16,6 +17,8 @@ pub trait MarkupExporter { &self, entries: &[BenchmarkResultWithRelativeSpeed], unit: TimeUnit, + reference_pending: bool, + explicit_reference: bool, ) -> String { // prepare table header strings let notation = format!("[{}]", unit.short_name()); @@ -58,14 +61,35 @@ pub trait MarkupExporter { }; let min_str = measurement.measurements.min().format_value(unit); let max_str = measurement.measurements.max().format_value(unit); - let rel_str = format!("{:.2}", entry.relative_speed); - let rel_stddev_str = if entry.is_reference { + // The ratio of two zero times is undefined, even if they compare equal. + let relative_unavailable = reference_pending + || (explicit_reference + && !entry.is_reference + && entry.relative_ordering == Ordering::Equal + && measurement.mean_wall_clock_time() == Time::zero()); + let rel_str = if relative_unavailable { + "N/A".to_string() + } else { + format!("{:.2}", entry.relative_speed) + }; + let rel_stddev_str = if relative_unavailable || entry.is_reference { "".into() } else if let Some(stddev) = entry.relative_speed_stddev { format!(" ± {stddev:.2}") } else { "".into() }; + let rel_direction = if relative_unavailable || !explicit_reference { + "" + } else if entry.is_reference { + " (reference)" + } else { + match entry.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Greater => " (slower)", + Ordering::Equal => " (same speed)", + } + }; // prepare table row entries table.push_str(&self.table_row(&[ @@ -73,7 +97,7 @@ pub trait MarkupExporter { &format!("{mean_str}{stddev_str}"), &min_str, &max_str, - &format!("{rel_str}{rel_stddev_str}"), + &format!("{rel_str}{rel_stddev_str}{rel_direction}"), ])) } @@ -114,11 +138,20 @@ impl Exporter for T { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result> { let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); - let entries = relative_speed::compute(results, sort_order); + // Do not report ratios against another benchmark while the selected + // reference is still pending in an intermediate export. + let reference_pending = reference_index.is_some_and(|i| i >= results.len()); + let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) { + relative_speed::compute_relative_speeds(results, reference, sort_order) + } else { + relative_speed::compute(results, sort_order) + }; - let table = self.table_results(&entries, unit); + let table = + self.table_results(&entries, unit, reference_pending, reference_index.is_some()); Ok(table.as_bytes().to_vec()) } } diff --git a/src/export/mod.rs b/src/export/mod.rs index c8d51d7..c8dd9a0 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -51,6 +51,7 @@ trait Exporter { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result>; } @@ -130,11 +131,18 @@ impl ExportManager { /// results are written to all file targets (to always have them up to date, even /// if a benchmark fails). In the latter case, we only print to stdout targets (in /// order not to clutter the output of hyperfine with intermediate results). - pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { + /// `reference_index` refers to the complete benchmark sequence, so it can + /// point past `results` while that benchmark is still pending. + pub fn write_results( + &self, + results: &[BenchmarkResult], + intermediate: bool, + reference_index: Option, + ) -> Result<()> { for e in &self.exporters { let content = || { e.exporter - .serialize(results, self.time_unit, self.sort_order) + .serialize(results, self.time_unit, self.sort_order, reference_index) }; match e.target { diff --git a/src/export/tests.rs b/src/export/tests.rs index 9566f9d..098226f 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -12,9 +12,174 @@ fn get_output( results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, +) -> String { + get_output_with_reference::(results, unit, sort_order, None) +} + +fn get_output_with_reference( + results: &[BenchmarkResult], + unit: Option, + sort_order: SortOrder, + reference_index: Option, ) -> String { let exporter = E::default(); - String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap() + String::from_utf8( + exporter + .serialize(results, unit, sort_order, reference_index) + .unwrap(), + ) + .unwrap() +} + +#[test] +fn test_markup_exports_identify_explicit_reference_and_direction() { + let benchmark = |name: &str, seconds| { + let measurement = Measurement { + time_wall_clock: Time::new::(seconds), + ..Measurement::default() + }; + BenchmarkResult { + command: name.to_owned(), + command_with_unused_parameters: name.to_owned(), + measurements: Measurements::new(vec![measurement, measurement]), + ..BenchmarkResult::default() + } + }; + let results = [ + benchmark("faster", 1.0), + benchmark("reference", 2.0), + benchmark("equal", 2.0), + benchmark("slower", 3.0), + ]; + + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + ]; + for output in outputs { + for expected in [ + "2.00 ± 0.00 (faster)", + "1.00 (reference)", + "1.00 ± 0.00 (same speed)", + "1.50 ± 0.00 (slower)", + ] { + assert!(output.contains(expected), "{}", output); + } + } + } +} + +#[test] +fn test_markup_exports_leave_pending_reference_ratios_unavailable() { + let results = [BenchmarkResult { + command: "sleep 1".into(), + command_with_unused_parameters: "sleep 1".into(), + measurements: Measurements::new(vec![Measurement { + time_wall_clock: Time::new::(1.0), + ..Measurement::default() + }]), + ..BenchmarkResult::default() + }]; + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + Some(1), + ), + ]; + for output in outputs { + assert!(output.contains("N/A"), "{}", output); + assert!(!output.contains("(reference)"), "{}", output); + } + } +} + +#[test] +fn test_markup_exports_with_zero_reference() { + let benchmark = |name: &str, seconds| BenchmarkResult { + command: name.to_owned(), + command_with_unused_parameters: name.to_owned(), + measurements: Measurements::new(vec![Measurement { + time_wall_clock: Time::new::(seconds), + ..Measurement::default() + }]), + ..BenchmarkResult::default() + }; + let results = [ + benchmark("other zero", 0.0), + benchmark("reference", 0.0), + benchmark("slower", 1.0), + ]; + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + for reference in [Some(1), Some(3), None] { + let outputs = [ + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + get_output_with_reference::( + &results, + Some(TimeUnit::Second), + sort_order, + reference, + ), + ]; + for output in outputs { + match reference { + Some(1) => { + assert_eq!(output.matches("N/A").count(), 1, "{}", output); + assert!(output.contains("1.00 (reference)"), "{}", output); + assert!(output.contains("inf (slower)"), "{}", output); + } + Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output), + None => { + assert!(!output.contains("N/A"), "{}", output); + assert!(output.contains("inf"), "{}", output); + assert!(!output.contains("(reference)"), "{}", output); + } + } + } + } + } } /// Ensure the makrup output includes the table header and the multiple diff --git a/src/options.rs b/src/options.rs index 0b902d5..659b976 100644 --- a/src/options.rs +++ b/src/options.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::{cmp, env, fmt, io}; -use anyhow::ensure; +use anyhow::{bail, ensure}; use clap::ArgMatches; use crate::command::Commands; @@ -214,6 +214,12 @@ pub struct Options { // Name of the reference command pub reference_name: Option, + // Whether commands are expanded from --parameter-scan or --parameter-list. + pub parameterized_mode: bool, + + // Index of the selected command in a parameterized run, if any. + pub parameterized_reference_index: Option, + /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -257,6 +263,8 @@ impl Default for Options { command_failure_action: CmdFailureAction::RaiseError, reference_command: None, reference_name: None, + parameterized_mode: false, + parameterized_reference_index: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -309,6 +317,8 @@ impl Options { options.setup_command = matches.get_one::("setup").map(String::from); + options.parameterized_mode = matches.get_many::("parameter-scan").is_some() + || matches.get_many::("parameter-list").is_some(); options.reference_command = matches.get_one::("reference").map(String::from); options.reference_name = matches .get_one::("reference-name") @@ -464,7 +474,29 @@ impl Options { } pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { - let has_reference_command = self.reference_command.is_some(); + if self.parameterized_mode { + if let Some(reference) = &self.reference_command { + ensure!( + self.reference_name.is_none(), + "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark" + ); + let mut matches = commands + .iter() + .enumerate() + .filter(|(_, command)| command.get_name_with_unused_parameters() == *reference); + let Some((index, _)) = matches.next() else { + bail!( + "Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name." + ); + }; + ensure!( + matches.next().is_none(), + "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names." + ); + self.parameterized_reference_index = Some(index); + } + } + let has_reference_command = self.reference_command.is_some() && !self.parameterized_mode; let num_commands = commands.num_commands(has_reference_command); if let Some(preparation_command) = &self.preparation_command { diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5..b097e98 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -384,16 +384,14 @@ fn reference_is_executed_first() { } #[test] -fn reference_is_executed_first_parameter_value() { +fn parameterized_reference_keeps_parameter_order() { ExecutionOrderTest::new() .arg("--runs=2") - .reference("reference") + .reference("command 2") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("reference") - .expect_output("reference") .expect_output("command 1") .expect_output("command 1") .expect_output("command 2") diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e6832b2..2256585 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -705,6 +705,40 @@ fn shows_benchmark_comparison_relative_to_reference() { "); } +#[test] +fn command_sorted_comparison_labels_explicit_reference() { + for (args, expected) in [ + ( + vec!["--reference", "sleep 2", "sleep 1", "sleep 3"], + vec![ + "1.00 sleep 2 (reference)", + "2.00 ± 0.00 sleep 1 (faster)", + "1.50 ± 0.00 sleep 3 (slower)", + ], + ), + ( + vec!["--reference", "sleep 2", "sleep 2"], + vec![ + "1.00 sleep 2 (reference)", + "1.00 ± 0.00 sleep 2 (same speed)", + ], + ), + ] { + let output = hyperfine_debug() + .args(["--style=basic", "--runs=2", "--sort=command"]) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; + assert_eq!( + comparison.lines().map(str::trim).collect::>(), + expected + ); + } +} + #[test] fn shows_reference_name() { let _settings = snapshot_settings().bind_to_scope(); @@ -796,50 +830,453 @@ fn rejects_negative_parameter_steps() { } #[test] -fn performs_reference_and_all_benchmarks_in_parameter_scan() { - let _settings = snapshot_settings().bind_to_scope(); - assert_cmd_snapshot!(hyperfine_debug() - .arg("--style=basic") - .arg("--reference=sleep 25") - .arg("--parameter-scan") - .arg("time") - .arg("30") - .arg("45") - .arg("--parameter-step-size") - .arg("5") - .arg("sleep {time}"), @r" - success: true - exit_code: 0 - ----- stdout ----- - Benchmark 1: sleep 25 - Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 25.000 s … 25.000 s 10 runs +fn selects_reference_from_parameter_scan() { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--reference=sleep 35", + "--parameter-scan", + "time", + "30", + "45", + "--parameter-step-size", + "5", + "sleep {time}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let benchmarks: Vec<_> = stdout + .lines() + .filter(|line| line.starts_with("Benchmark ")) + .collect(); + assert_eq!( + benchmarks, + [ + "Benchmark 1: sleep 30", + "Benchmark 2: sleep 35", + "Benchmark 3: sleep 40", + "Benchmark 4: sleep 45", + ] + ); + assert!(stdout.contains("Summary\n sleep 35 ran\n"), "{}", stdout); +} - Benchmark 2: sleep 30 - Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 30.000 s … 30.000 s 10 runs +#[cfg(unix)] +#[test] +fn selects_reference_with_parameterized_prepare() { + let directory = tempfile::tempdir().unwrap(); + let csv_path = directory.path().join("results.csv"); + let json_path = directory.path().join("results.json"); + let output = hyperfine_raw_command() + .args([ + "--style=basic", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.2,0.4,0.6", + "--prepare", + "sleep {delay}", + "--reference", + "echo a (delay = 0.4)", + "--export-csv", + ]) + .arg(&csv_path) + .arg("--export-json") + .arg(&json_path) + .arg("echo a") + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 3); + assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout); - Benchmark 3: sleep 35 - Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 35.000 s … 35.000 s 10 runs + let mut csv = csv::Reader::from_path(csv_path).unwrap(); + assert!(csv + .headers() + .unwrap() + .iter() + .any(|header| header == "parameter_delay")); + let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect(); + assert_eq!(records.len(), 3); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap(); + let results = json["results"].as_array().unwrap(); + assert_eq!(results.len(), 3); + assert_eq!( + results + .iter() + .map(|result| result["parameters"]["delay"].as_str().unwrap()) + .collect::>(), + ["0.2", "0.4", "0.6"] + ); +} - Benchmark 4: sleep 40 - Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 40.000 s … 40.000 s 10 runs +#[test] +fn selects_reference_by_custom_benchmark_name() { + for (command, name, reference) in [ + ("sleep {x}", "case-{x}", "case-2"), + ("sleep 1", "case-{x}", "case-2 (x = 2)"), + ] { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "-L", + "x", + "1,2", + "--command-name", + name, + "--reference", + reference, + command, + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 2); + assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout); + } +} - Benchmark 5: sleep 45 - Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 45.000 s … 45.000 s 10 runs +#[test] +fn selects_reference_from_multiple_commands_and_parameters() { + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "-L", + "x", + "1,2", + "-L", + "y", + "3,4", + "--reference", + "sleep 4 (x = 2)", + "sleep {x}", + "sleep {y}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 8); + assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout); +} - Summary - sleep 25 ran - 1.20 ± 0.00 times faster than sleep 30 - 1.40 ± 0.00 times faster than sleep 35 - 1.60 ± 0.00 times faster than sleep 40 - 1.80 ± 0.00 times faster than sleep 45 +#[test] +fn rejects_unmatched_parameterized_reference_before_creating_export() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.csv"); + let marker_path = directory.path().join("setup-ran"); + for (args, benchmark) in [ + (vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"), + ( + vec!["--reference", "sleep 1", "-P", "secs", "2", "3"], + "sleep {secs}", + ), + ] { + std::fs::write(&export_path, "previous contents").unwrap(); + hyperfine() + .args(args) + .arg("--setup") + .arg(format!("echo touched > \"{}\"", marker_path.display())) + .arg("--export-csv") + .arg(&export_path) + .arg(benchmark) + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "does not match any parameterized benchmark", + )); - ----- stderr ----- - "); + assert_eq!( + std::fs::read_to_string(&export_path).unwrap(), + "previous contents" + ); + assert!(!marker_path.exists()); + } +} + +#[test] +fn rejects_ambiguous_parameterized_reference() { + for args in [ + vec![ + "-L", + "x", + "1,2", + "--command-name", + "case", + "--reference", + "case", + "sleep {x}", + ], + vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"], + ] { + hyperfine_debug() + .args(args) + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "matches multiple parameterized benchmarks", + )); + } +} + +#[test] +fn rejects_reference_name_with_parameterized_reference() { + hyperfine_debug() + .args([ + "-L", + "x", + "1,2", + "--reference", + "sleep 1", + "--reference-name", + "baseline", + "sleep {x}", + ]) + .assert() + .failure() + .stderr(predicate::str::contains( + "--reference-name cannot be used with a parameterized reference", + )); +} + +#[test] +fn parameterized_reference_preserves_per_benchmark_hook_order() { + let directory = tempfile::tempdir().unwrap(); + let log_path = directory.path().join("hooks.log"); + let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display()); + let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display()); + + hyperfine() + .args(["--runs=1", "-L", "x", "1,2,3"]) + .arg("--command-name") + .arg("case-{x}") + .arg("--reference") + .arg("case-2") + .arg("--prepare") + .arg(prepare("prepare-1")) + .arg("--prepare") + .arg(prepare("prepare-2")) + .arg("--prepare") + .arg(prepare("prepare-3")) + .arg(command) + .assert() + .success(); + + let contents = std::fs::read_to_string(log_path).unwrap(); + assert_eq!( + contents.lines().map(str::trim).collect::>(), + [ + "prepare-1", + "benchmark-1", + "prepare-2", + "benchmark-2", + "prepare-3", + "benchmark-3" + ] + ); +} + +#[test] +fn parameterized_reference_preserves_per_benchmark_output_order() { + let directory = tempfile::tempdir().unwrap(); + let output_paths = [ + directory.path().join("one.txt"), + directory.path().join("two.txt"), + directory.path().join("three.txt"), + ]; + let mut command = hyperfine(); + command.args([ + "--runs=1", + "-L", + "x", + "1,2,3", + "--command-name", + "case-{x}", + "--reference", + "case-2", + ]); + for path in &output_paths { + command.arg("--output").arg(path); + } + command.arg("echo benchmark-{x}").assert().success(); + + for (path, expected) in output_paths + .iter() + .zip(["benchmark-1", "benchmark-2", "benchmark-3"]) + { + assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected); + } +} + +#[cfg(unix)] +#[test] +fn intermediate_markdown_waits_for_selected_reference() { + let directory = tempfile::tempdir().unwrap(); + for (reference, preparations, reference_available) in [ + ("sleep 0.03", ["true", "false", "true"], false), + ("sleep 0.02", ["true", "true", "false"], true), + ] { + let export_path = directory.path().join("results.md"); + let mut command = hyperfine(); + command.args([ + "--style=none", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.01,0.02,0.03", + "--reference", + reference, + ]); + for preparation in preparations { + command.arg("--prepare").arg(preparation); + } + command + .arg("--export-markdown") + .arg(&export_path) + .arg("sleep {delay}") + .assert() + .failure() + .stderr(predicate::str::contains( + "The preparation command terminated", + )); + let markdown = std::fs::read_to_string(&export_path).unwrap(); + if reference_available { + assert!(!markdown.contains("N/A"), "{}", markdown); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.02")) + .unwrap(); + assert!(row.ends_with("| 1.00 (reference) |"), "{}", row); + } else { + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.01")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); + } + } +} + +#[test] +fn markdown_export_uses_selected_parameterized_reference() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "-L", + "x", + "3,2,1", + "--reference", + "sleep 2", + "sleep {x}", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let rows: Vec<_> = stdout + .lines() + .filter(|line| line.contains("sleep ")) + .collect(); + if sort_order == "command" { + assert!(rows[0].contains("sleep 3")); + assert!(rows[1].contains("sleep 2")); + assert!(rows[2].contains("sleep 1")); + } else { + assert!(rows[0].contains("sleep 1")); + assert!(rows[1].contains("sleep 2")); + assert!(rows[2].contains("sleep 3")); + } + for (command, relative) in [ + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (reference)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = rows.iter().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } +} + +#[test] +fn markdown_export_uses_standalone_reference() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "--reference", + "sleep 2", + "--reference-name", + "baseline", + "sleep 1", + "sleep 2", + "sleep 3", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("baseline", "1.00 (reference)"), + ("sleep 1", "2.00 (faster)"), + ("sleep 2", "1.00 (same speed)"), + ("sleep 3", "1.50 (slower)"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } +} + +#[test] +fn markdown_export_without_explicit_reference_is_unchanged() { + for sort_order in ["command", "mean-time"] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--time-unit=second", + "--export-markdown=-", + "--sort", + sort_order, + "sleep 3", + "sleep 1", + "sleep 2", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + for (command, relative) in [ + ("sleep 1", "1.00"), + ("sleep 2", "2.00"), + ("sleep 3", "3.00"), + ] { + let row = stdout.lines().find(|line| line.contains(command)).unwrap(); + assert!(row.ends_with(&format!("| {relative} |")), "{}", row); + } + } } #[test]