From 7167df6951d19b24609e4817bbd578b9b1affb4c Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:20:04 +0200 Subject: [PATCH] Fix relative runtime ratios with an explicit reference --- doc/hyperfine.1 | 2 + src/benchmark/relative_speed.rs | 127 ++++++++++++++++++++++++-------- src/benchmark/scheduler.rs | 19 ++++- src/cli.rs | 4 +- src/export/csv.rs | 3 +- src/export/json.rs | 1 + src/export/markup.rs | 3 +- src/export/mod.rs | 11 ++- src/export/tests.rs | 2 +- tests/integration_tests.rs | 99 +++++++++++++++++++++++++ 10 files changed, 232 insertions(+), 39 deletions(-) diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index d565dd3..0d61755 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -118,6 +118,8 @@ not every time as would happen with the \fB\-\-prepare\fR option. .IP Benchmark \fICMD\fP as the reference for the relative comparison of results. If this option is not specified, the fastest command is used as the reference. +In the command-sorted comparison and exported markup tables, values below 1 +indicate commands faster than the reference. .HP \fB\-\-reference\-name\fR \fINAME\fP .IP diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 711ca2f..bd84883 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -3,12 +3,13 @@ use std::cmp::Ordering; use super::benchmark_result::BenchmarkResult; use crate::{ options::SortOrder, - quantity::{self, Ratio, Time, Zero}, + quantity::{self, Time, Zero}, }; #[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, + /// Runtime divided by the reference runtime; values below one are faster. pub relative_speed: f64, pub relative_speed_stddev: Option, pub is_reference: bool, @@ -40,7 +41,8 @@ fn compute_relative_speeds<'a>( let is_reference = result == reference; let relative_ordering = compare_mean_time(result, reference); - if result.mean_wall_clock_time() == Time::zero() { + let reference_mean = reference.mean_wall_clock_time(); + if reference_mean == Time::zero() { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, @@ -50,13 +52,7 @@ fn compute_relative_speeds<'a>( }; } - let ratio = match relative_ordering { - Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(), - Ordering::Equal => Ratio::new::(1.0), - Ordering::Greater => { - result.mean_wall_clock_time() / reference.mean_wall_clock_time() - } - }; + let ratio = result.mean_wall_clock_time() / reference_mean; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance assumed to be 0, i.e. variables are assumed to be independent @@ -64,13 +60,11 @@ fn compute_relative_speeds<'a>( result.measurements.stddev(), reference.measurements.stddev(), ) { - (Some(result_stddev), Some(fastest_stddev)) => Some( - ratio - * ((result_stddev / result.mean_wall_clock_time()) - .powi(uom::typenum::P2::new()) - + (fastest_stddev / reference.mean_wall_clock_time()) - .powi(uom::typenum::P2::new())) - .sqrt(), + (Some(result_stddev), Some(reference_stddev)) => Some( + ((result_stddev / reference_mean).powi(uom::typenum::P2::new()) + + (ratio * (reference_stddev / reference_mean)) + .powi(uom::typenum::P2::new())) + .sqrt(), ), _ => None, }; @@ -122,18 +116,25 @@ pub fn compute_with_check( Some(compute_relative_speeds(results, fastest, sort_order)) } -/// Same as compute_with_check, potentially resulting in relative speeds of infinity -pub fn compute( - results: &[BenchmarkResult], +/// Compute ratios relative to the given reference, or the fastest result if omitted. +/// A zero reference runtime can produce infinite ratios. +pub fn compute<'a>( + results: &'a [BenchmarkResult], sort_order: SortOrder, -) -> Vec> { - let fastest = fastest_of(results); + reference: Option<&'a BenchmarkResult>, +) -> Vec> { + let reference = reference.unwrap_or_else(|| fastest_of(results)); - compute_relative_speeds(results, fastest, sort_order) + compute_relative_speeds(results, reference, sort_order) } #[cfg(test)] fn create_result(name: &str, mean: f64) -> BenchmarkResult { + create_result_from_times(name, &[mean]) +} + +#[cfg(test)] +fn create_result_from_times(name: &str, times: &[f64]) -> BenchmarkResult { use std::collections::BTreeMap; use crate::benchmark::measurement::{Measurement, Measurements}; @@ -143,11 +144,14 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { command: name.into(), command_with_unused_parameters: name.into(), measurements: Measurements { - measurements: vec![Measurement { - time_wall_clock: Time::new::(mean), - time_user: Time::new::(mean), - ..Default::default() - }], + measurements: times + .iter() + .map(|&time| Measurement { + time_wall_clock: Time::new::(time), + time_user: Time::new::(time), + ..Default::default() + }) + .collect(), }, parameters: BTreeMap::new(), } @@ -174,13 +178,20 @@ fn test_compute_relative_speed() { fn test_compute_relative_speed_with_reference() { use approx::assert_relative_eq; - let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)]; - let reference = create_result("cmd2", 4.0); + let results = vec![ + create_result_from_times("cmd2", &[1.0, 2.0, 3.0]), + create_result_from_times("cmd3", &[4.0, 5.0, 6.0]), + ]; + let reference = create_result_from_times("reference", &[3.0, 4.0, 5.0]); let annotated_results = compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap(); - assert_relative_eq!(2.0, annotated_results[0].relative_speed); + assert_relative_eq!(0.5, annotated_results[0].relative_speed); + assert_relative_eq!( + 0.2795084971874737, + annotated_results[0].relative_speed_stddev.unwrap() + ); assert_relative_eq!(1.25, annotated_results[1].relative_speed); } @@ -192,3 +203,59 @@ fn test_compute_relative_speed_for_zero_times() { assert!(annotated_results.is_none()); } + +#[test] +fn test_reference_ratios_do_not_depend_on_sort_order() { + use approx::assert_relative_eq; + + let results = vec![ + create_result("reference", 2.0), + create_result("slower", 3.0), + create_result("faster", 1.0), + create_result("equal", 2.0), + ]; + + for sort_order in [SortOrder::Command, SortOrder::MeanTime] { + let entries = compute(&results, sort_order, Some(&results[0])); + for entry in entries { + assert_relative_eq!( + entry + .result + .mean_wall_clock_time() + .get::() + / 2.0, + entry.relative_speed + ); + assert_eq!(entry.is_reference, entry.result.command == "reference"); + } + } +} + +#[test] +fn test_reference_ratios_without_stddev() { + let results = vec![ + create_result("reference", 2.0), + create_result("faster", 1.0), + ]; + let entries = compute(&results, SortOrder::Command, Some(&results[0])); + assert_eq!(entries[1].relative_speed, 0.5); + assert_eq!(entries[1].relative_speed_stddev, None); +} + +#[test] +fn test_reference_ratios_with_zero_times() { + let results = vec![ + create_result_from_times("reference", &[1.0, 2.0, 3.0]), + create_result_from_times("zero", &[0.0, 0.0, 0.0]), + ]; + let entries = compute(&results, SortOrder::Command, Some(&results[0])); + assert_eq!(entries[1].relative_speed, 0.0); + assert_eq!(entries[1].relative_speed_stddev, Some(0.0)); + + let entries = compute(&results, SortOrder::Command, Some(&results[1])); + assert_eq!(entries[0].relative_speed, f64::INFINITY); + assert_eq!(entries[0].relative_speed_stddev, None); + assert_eq!(entries[1].relative_speed, 1.0); + + assert!(compute_with_check_from_reference(&results, &results[0], SortOrder::Command).is_none()); +} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index ce5fd81..24ed726 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -96,7 +96,18 @@ impl<'a> Scheduler<'a> { )?; for item in others { - let stddev = if let Some(stddev) = item.relative_speed_stddev { + // The prose summary expresses both directions as factors above one. + let (relative_speed, relative_speed_stddev) = + if item.relative_ordering == Ordering::Less { + ( + 1.0 / item.relative_speed, + item.relative_speed_stddev + .map(|stddev| stddev / item.relative_speed.powi(2)), + ) + } else { + (item.relative_speed, item.relative_speed_stddev) + }; + let stddev = if let Some(stddev) = relative_speed_stddev { format!(" ± {}", format!("{:.2}", stddev).green()) } else { "".into() @@ -104,17 +115,17 @@ impl<'a> Scheduler<'a> { let comparator = match item.relative_ordering { Ordering::Less => format!( "{}{} times slower than", - format!("{:8.2}", item.relative_speed).bold().green(), + format!("{relative_speed:8.2}").bold().green(), stddev ), Ordering::Greater => format!( "{}{} times faster than", - format!("{:8.2}", item.relative_speed).bold().green(), + format!("{relative_speed:8.2}").bold().green(), stddev ), Ordering::Equal => format!( " As fast ({}{}) as", - format!("{:.2}", item.relative_speed).bold().green(), + format!("{relative_speed:.2}").bold().green(), stddev ), }; diff --git a/src/cli.rs b/src/cli.rs index 39a2238..b4234ea 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -106,7 +106,9 @@ fn build_command() -> Command { .value_name("CMD") .help( "The reference command for the relative comparison of results. \ - If this is unset, results are compared with the fastest command as reference." + If this is unset, results are compared with the fastest command as reference. \ + In the command-sorted comparison and exported markup tables, values below 1 \ + indicate commands faster than the reference." ) ) .arg( diff --git a/src/export/csv.rs b/src/export/csv.rs index 3932f92..3727494 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -18,6 +18,7 @@ impl Exporter for CsvExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference: Option<&BenchmarkResult>, ) -> Result> { const CSV_UNIT: TimeUnit = TimeUnit::Second; const CSV_PRECISION: usize = 6; @@ -172,7 +173,7 @@ fn test_csv() { let actual = String::from_utf8( exporter - .serialize(&results, Some(TimeUnit::Second), SortOrder::Command) + .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None) .unwrap(), ) .unwrap(); diff --git a/src/export/json.rs b/src/export/json.rs index 6c9b21d..3bd05b7 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -22,6 +22,7 @@ impl Exporter for JsonExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference: Option<&BenchmarkResult>, ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); if let Ok(ref mut content) = output { diff --git a/src/export/markup.rs b/src/export/markup.rs index f3f8711..c8b7f11 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -114,9 +114,10 @@ impl Exporter for T { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference: Option<&BenchmarkResult>, ) -> Result> { let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); - let entries = relative_speed::compute(results, sort_order); + let entries = relative_speed::compute(results, sort_order, reference); let table = self.table_results(&entries, unit); Ok(table.as_bytes().to_vec()) diff --git a/src/export/mod.rs b/src/export/mod.rs index c8d51d7..13bc5a3 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -51,6 +51,7 @@ trait Exporter { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference: Option<&BenchmarkResult>, ) -> Result>; } @@ -69,6 +70,7 @@ pub struct ExportManager { exporters: Vec, time_unit: Option, sort_order: SortOrder, + has_reference: bool, } impl ExportManager { @@ -83,6 +85,7 @@ impl ExportManager { exporters: vec![], time_unit, sort_order, + has_reference: matches.contains_id("reference"), }; { let mut add_exporter = |flag, exporttype| -> Result<()> { @@ -131,10 +134,16 @@ impl ExportManager { /// if a benchmark fails). In the latter case, we only print to stdout targets (in /// order not to clutter the output of hyperfine with intermediate results). pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { + // The scheduler benchmarks an explicit reference before the other commands. + let reference = if self.has_reference { + results.first() + } else { + None + }; for e in &self.exporters { let content = || { e.exporter - .serialize(results, self.time_unit, self.sort_order) + .serialize(results, self.time_unit, self.sort_order, reference) }; match e.target { diff --git a/src/export/tests.rs b/src/export/tests.rs index 9566f9d..abe340a 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -14,7 +14,7 @@ fn get_output( sort_order: SortOrder, ) -> String { let exporter = E::default(); - String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap() + String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap() } /// Ensure the makrup output includes the table header and the multiple diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e6832b2..67987aa 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -705,6 +705,105 @@ fn shows_benchmark_comparison_relative_to_reference() { "); } +#[test] +fn command_sorted_comparison_relative_to_reference() { + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() + .args([ + "--style=basic", + "--sort=command", + "--reference=sleep 2", + "--reference-name=baseline", + "sleep 1", + "sleep 3", + "sleep 2", + ]), @r" + success: true + exit_code: 0 + ----- stdout ----- + Benchmark 1: baseline + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 3 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Benchmark 4: sleep 2 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Relative speed comparison + 1.00 baseline + 0.50 ± 0.00 sleep 1 + 1.50 ± 0.00 sleep 3 + 1.00 ± 0.00 sleep 2 + + ----- stderr ----- + "); +} + +#[test] +fn markdown_export_relative_to_reference() { + let tempdir = tempfile::tempdir().unwrap(); + let export_path = tempdir.path().join("results.md"); + + insta::allow_duplicates! { + for sort_order in ["auto", "command", "mean-time"] { + for to_stdout in [false, true] { + let output = hyperfine_debug() + .args([ + "--style=none", + "--time-unit=second", + "--reference=sleep 2", + "--reference-name=baseline", + "sleep 1", + "sleep 3", + ]) + .arg(format!("--sort={sort_order}")) + .arg("--export-markdown") + .arg(if to_stdout { + std::path::Path::new("-") + } else { + &export_path + }) + .assert() + .success() + .get_output() + .stdout + .clone(); + let table = if to_stdout { + String::from_utf8(output).unwrap() + } else { + std::fs::read_to_string(&export_path).unwrap() + }; + + if sort_order == "mean-time" { + insta::assert_snapshot!(table.trim(), @r" + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 | + | `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 | + | `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 | + "); + } else { + insta::assert_snapshot!(table.trim(), @r" + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 | + | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 | + | `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 | + "); + } + } + } + } +} + #[test] fn shows_reference_name() { let _settings = snapshot_settings().bind_to_scope();