Fix relative runtime ratios with an explicit reference

doc/hyperfine.1

@@ -118,6 +118,8 @@ not every time as would happen with the \fB\-\-prepare\fR option.
 .IP
 Benchmark \fICMD\fP as the reference for the relative comparison of results.
 If this option is not specified, the fastest command is used as the reference.
+In the command-sorted comparison and exported markup tables, values below 1
+indicate commands faster than the reference.
 .HP
 \fB\-\-reference\-name\fR \fINAME\fP
 .IP

src/benchmark/relative_speed.rs

@@ -3,12 +3,13 @@ use std::cmp::Ordering;
 use super::benchmark_result::BenchmarkResult;
 use crate::{
     options::SortOrder,
-    quantity::{self, Ratio, Time, Zero},
+    quantity::{self, Time, Zero},
 };
 
 #[derive(Debug)]
 pub struct BenchmarkResultWithRelativeSpeed<'a> {
     pub result: &'a BenchmarkResult,
+    /// Runtime divided by the reference runtime; values below one are faster.
     pub relative_speed: f64,
     pub relative_speed_stddev: Option<f64>,
     pub is_reference: bool,
@@ -40,7 +41,8 @@ fn compute_relative_speeds<'a>(
             let is_reference = result == reference;
             let relative_ordering = compare_mean_time(result, reference);
 
-            if result.mean_wall_clock_time() == Time::zero() {
+            let reference_mean = reference.mean_wall_clock_time();
+            if reference_mean == Time::zero() {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
                     relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -50,13 +52,7 @@ fn compute_relative_speeds<'a>(
                 };
             }
 
-            let ratio = match relative_ordering {
-                Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
-                Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
-                Ordering::Greater => {
-                    result.mean_wall_clock_time() / reference.mean_wall_clock_time()
-                }
-            };
+            let ratio = result.mean_wall_clock_time() / reference_mean;
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
             // Covariance assumed to be 0, i.e. variables are assumed to be independent
@@ -64,13 +60,11 @@ fn compute_relative_speeds<'a>(
                 result.measurements.stddev(),
                 reference.measurements.stddev(),
             ) {
-                (Some(result_stddev), Some(fastest_stddev)) => Some(
-                    ratio
-                        * ((result_stddev / result.mean_wall_clock_time())
-                            .powi(uom::typenum::P2::new())
-                            + (fastest_stddev / reference.mean_wall_clock_time())
-                                .powi(uom::typenum::P2::new()))
-                        .sqrt(),
+                (Some(result_stddev), Some(reference_stddev)) => Some(
+                    ((result_stddev / reference_mean).powi(uom::typenum::P2::new())
+                        + (ratio * (reference_stddev / reference_mean))
+                            .powi(uom::typenum::P2::new()))
+                    .sqrt(),
                 ),
                 _ => None,
             };
@@ -122,18 +116,25 @@ pub fn compute_with_check(
     Some(compute_relative_speeds(results, fastest, sort_order))
 }
 
-/// Same as compute_with_check, potentially resulting in relative speeds of infinity
-pub fn compute(
-    results: &[BenchmarkResult],
+/// Compute ratios relative to the given reference, or the fastest result if omitted.
+/// A zero reference runtime can produce infinite ratios.
+pub fn compute<'a>(
+    results: &'a [BenchmarkResult],
     sort_order: SortOrder,
-) -> Vec<BenchmarkResultWithRelativeSpeed<'_>> {
-    let fastest = fastest_of(results);
+    reference: Option<&'a BenchmarkResult>,
+) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
+    let reference = reference.unwrap_or_else(|| fastest_of(results));
 
-    compute_relative_speeds(results, fastest, sort_order)
+    compute_relative_speeds(results, reference, sort_order)
 }
 
 #[cfg(test)]
 fn create_result(name: &str, mean: f64) -> BenchmarkResult {
+    create_result_from_times(name, &[mean])
+}
+
+#[cfg(test)]
+fn create_result_from_times(name: &str, times: &[f64]) -> BenchmarkResult {
     use std::collections::BTreeMap;
 
     use crate::benchmark::measurement::{Measurement, Measurements};
@@ -143,11 +144,14 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult {
         command: name.into(),
         command_with_unused_parameters: name.into(),
         measurements: Measurements {
-            measurements: vec![Measurement {
-                time_wall_clock: Time::new::<second>(mean),
-                time_user: Time::new::<second>(mean),
-                ..Default::default()
-            }],
+            measurements: times
+                .iter()
+                .map(|&time| Measurement {
+                    time_wall_clock: Time::new::<second>(time),
+                    time_user: Time::new::<second>(time),
+                    ..Default::default()
+                })
+                .collect(),
         },
         parameters: BTreeMap::new(),
     }
@@ -174,13 +178,20 @@ fn test_compute_relative_speed() {
 fn test_compute_relative_speed_with_reference() {
     use approx::assert_relative_eq;
 
-    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
-    let reference = create_result("cmd2", 4.0);
+    let results = vec![
+        create_result_from_times("cmd2", &[1.0, 2.0, 3.0]),
+        create_result_from_times("cmd3", &[4.0, 5.0, 6.0]),
+    ];
+    let reference = create_result_from_times("reference", &[3.0, 4.0, 5.0]);
 
     let annotated_results =
         compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
 
-    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
+    assert_relative_eq!(0.5, annotated_results[0].relative_speed);
+    assert_relative_eq!(
+        0.2795084971874737,
+        annotated_results[0].relative_speed_stddev.unwrap()
+    );
     assert_relative_eq!(1.25, annotated_results[1].relative_speed);
 }
 
@@ -192,3 +203,59 @@ fn test_compute_relative_speed_for_zero_times() {
 
     assert!(annotated_results.is_none());
 }
+
+#[test]
+fn test_reference_ratios_do_not_depend_on_sort_order() {
+    use approx::assert_relative_eq;
+
+    let results = vec![
+        create_result("reference", 2.0),
+        create_result("slower", 3.0),
+        create_result("faster", 1.0),
+        create_result("equal", 2.0),
+    ];
+
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        let entries = compute(&results, sort_order, Some(&results[0]));
+        for entry in entries {
+            assert_relative_eq!(
+                entry
+                    .result
+                    .mean_wall_clock_time()
+                    .get::<quantity::second>()
+                    / 2.0,
+                entry.relative_speed
+            );
+            assert_eq!(entry.is_reference, entry.result.command == "reference");
+        }
+    }
+}
+
+#[test]
+fn test_reference_ratios_without_stddev() {
+    let results = vec![
+        create_result("reference", 2.0),
+        create_result("faster", 1.0),
+    ];
+    let entries = compute(&results, SortOrder::Command, Some(&results[0]));
+    assert_eq!(entries[1].relative_speed, 0.5);
+    assert_eq!(entries[1].relative_speed_stddev, None);
+}
+
+#[test]
+fn test_reference_ratios_with_zero_times() {
+    let results = vec![
+        create_result_from_times("reference", &[1.0, 2.0, 3.0]),
+        create_result_from_times("zero", &[0.0, 0.0, 0.0]),
+    ];
+    let entries = compute(&results, SortOrder::Command, Some(&results[0]));
+    assert_eq!(entries[1].relative_speed, 0.0);
+    assert_eq!(entries[1].relative_speed_stddev, Some(0.0));
+
+    let entries = compute(&results, SortOrder::Command, Some(&results[1]));
+    assert_eq!(entries[0].relative_speed, f64::INFINITY);
+    assert_eq!(entries[0].relative_speed_stddev, None);
+    assert_eq!(entries[1].relative_speed, 1.0);
+
+    assert!(compute_with_check_from_reference(&results, &results[0], SortOrder::Command).is_none());
+}

src/benchmark/scheduler.rs

@@ -96,7 +96,18 @@ impl<'a> Scheduler<'a> {
                     )?;
 
                     for item in others {
-                        let stddev = if let Some(stddev) = item.relative_speed_stddev {
+                        // The prose summary expresses both directions as factors above one.
+                        let (relative_speed, relative_speed_stddev) =
+                            if item.relative_ordering == Ordering::Less {
+                                (
+                                    1.0 / item.relative_speed,
+                                    item.relative_speed_stddev
+                                        .map(|stddev| stddev / item.relative_speed.powi(2)),
+                                )
+                            } else {
+                                (item.relative_speed, item.relative_speed_stddev)
+                            };
+                        let stddev = if let Some(stddev) = relative_speed_stddev {
                             format!(" ± {}", format!("{:.2}", stddev).green())
                         } else {
                             "".into()
@@ -104,17 +115,17 @@ impl<'a> Scheduler<'a> {
                         let comparator = match item.relative_ordering {
                             Ordering::Less => format!(
                                 "{}{} times slower than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                format!("{relative_speed:8.2}").bold().green(),
                                 stddev
                             ),
                             Ordering::Greater => format!(
                                 "{}{} times faster than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
+                                format!("{relative_speed:8.2}").bold().green(),
                                 stddev
                             ),
                             Ordering::Equal => format!(
                                 "    As fast ({}{}) as",
-                                format!("{:.2}", item.relative_speed).bold().green(),
+                                format!("{relative_speed:.2}").bold().green(),
                                 stddev
                             ),
                         };

src/cli.rs

@@ -106,7 +106,9 @@ fn build_command() -> Command {
                 .value_name("CMD")
                 .help(
                     "The reference command for the relative comparison of results. \
-                    If this is unset, results are compared with the fastest command as reference."
+                    If this is unset, results are compared with the fastest command as reference. \
+                    In the command-sorted comparison and exported markup tables, values below 1 \
+                    indicate commands faster than the reference."
                 )
         )
         .arg(

src/export/csv.rs

@@ -18,6 +18,7 @@ impl Exporter for CsvExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference: Option<&BenchmarkResult>,
     ) -> Result<Vec<u8>> {
         const CSV_UNIT: TimeUnit = TimeUnit::Second;
         const CSV_PRECISION: usize = 6;
@@ -172,7 +173,7 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command)
+            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None)
             .unwrap(),
     )
     .unwrap();

src/export/json.rs

@@ -22,6 +22,7 @@ impl Exporter for JsonExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference: Option<&BenchmarkResult>,
     ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary { results });
         if let Ok(ref mut content) = output {

src/export/markup.rs

@@ -114,9 +114,10 @@ impl<T: MarkupExporter> Exporter for T {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference: Option<&BenchmarkResult>,
     ) -> Result<Vec<u8>> {
         let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
-        let entries = relative_speed::compute(results, sort_order);
+        let entries = relative_speed::compute(results, sort_order, reference);
 
         let table = self.table_results(&entries, unit);
         Ok(table.as_bytes().to_vec())

src/export/mod.rs

@@ -51,6 +51,7 @@ trait Exporter {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference: Option<&BenchmarkResult>,
     ) -> Result<Vec<u8>>;
 }
 
@@ -69,6 +70,7 @@ pub struct ExportManager {
     exporters: Vec<ExporterWithTarget>,
     time_unit: Option<TimeUnit>,
     sort_order: SortOrder,
+    has_reference: bool,
 }
 
 impl ExportManager {
@@ -83,6 +85,7 @@ impl ExportManager {
             exporters: vec![],
             time_unit,
             sort_order,
+            has_reference: matches.contains_id("reference"),
         };
         {
             let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -131,10 +134,16 @@ impl ExportManager {
     /// if a benchmark fails). In the latter case, we only print to stdout targets (in
     /// order not to clutter the output of hyperfine with intermediate results).
     pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
+        // The scheduler benchmarks an explicit reference before the other commands.
+        let reference = if self.has_reference {
+            results.first()
+        } else {
+            None
+        };
         for e in &self.exporters {
             let content = || {
                 e.exporter
-                    .serialize(results, self.time_unit, self.sort_order)
+                    .serialize(results, self.time_unit, self.sort_order, reference)
             };
 
             match e.target {

src/export/tests.rs

@@ -14,7 +14,7 @@ fn get_output<E: Exporter + Default>(
     sort_order: SortOrder,
 ) -> String {
     let exporter = E::default();
-    String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap()
+    String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap()
 }
 
 /// Ensure the makrup output includes the table header and the multiple

tests/integration_tests.rs

@@ -705,6 +705,105 @@ fn shows_benchmark_comparison_relative_to_reference() {
     ");
 }
 
+#[test]
+fn command_sorted_comparison_relative_to_reference() {
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
+        .args([
+            "--style=basic",
+            "--sort=command",
+            "--reference=sleep 2",
+            "--reference-name=baseline",
+            "sleep 1",
+            "sleep 3",
+            "sleep 2",
+        ]), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    Benchmark 1: baseline
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Benchmark 2: sleep 1
+      Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    1.000 s …  1.000 s    10 runs
+
+    Benchmark 3: sleep 3
+      Time (mean ± σ):      3.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    3.000 s …  3.000 s    10 runs
+
+    Benchmark 4: sleep 2
+      Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+      Range (min … max):    2.000 s …  2.000 s    10 runs
+
+    Relative speed comparison
+            1.00          baseline
+            0.50 ±  0.00  sleep 1
+            1.50 ±  0.00  sleep 3
+            1.00 ±  0.00  sleep 2
+
+    ----- stderr -----
+    ");
+}
+
+#[test]
+fn markdown_export_relative_to_reference() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let export_path = tempdir.path().join("results.md");
+
+    insta::allow_duplicates! {
+        for sort_order in ["auto", "command", "mean-time"] {
+            for to_stdout in [false, true] {
+                let output = hyperfine_debug()
+                    .args([
+                        "--style=none",
+                        "--time-unit=second",
+                        "--reference=sleep 2",
+                        "--reference-name=baseline",
+                        "sleep 1",
+                        "sleep 3",
+                    ])
+                    .arg(format!("--sort={sort_order}"))
+                    .arg("--export-markdown")
+                    .arg(if to_stdout {
+                        std::path::Path::new("-")
+                    } else {
+                        &export_path
+                    })
+                    .assert()
+                    .success()
+                    .get_output()
+                    .stdout
+                    .clone();
+                let table = if to_stdout {
+                    String::from_utf8(output).unwrap()
+                } else {
+                    std::fs::read_to_string(&export_path).unwrap()
+                };
+
+                if sort_order == "mean-time" {
+                    insta::assert_snapshot!(table.trim(), @r"
+                    | Command | Mean [s] | Min [s] | Max [s] | Relative |
+                    |:---|---:|---:|---:|---:|
+                    | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
+                    | `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
+                    | `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
+                    ");
+                } else {
+                    insta::assert_snapshot!(table.trim(), @r"
+                    | Command | Mean [s] | Min [s] | Max [s] | Relative |
+                    |:---|---:|---:|---:|---:|
+                    | `baseline` | 2.000 ± 0.000 | 2.000 | 2.000 | 1.00 |
+                    | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 0.50 ± 0.00 |
+                    | `sleep 3` | 3.000 ± 0.000 | 3.000 | 3.000 | 1.50 ± 0.00 |
+                    ");
+                }
+            }
+        }
+    }
+}
+
 #[test]
 fn shows_reference_name() {
     let _settings = snapshot_settings().bind_to_scope();