Use parameterized benchmarks as references

src/export/csv.rs

@@ -18,6 +18,7 @@ impl Exporter for CsvExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         const CSV_UNIT: TimeUnit = TimeUnit::Second;
         const CSV_PRECISION: usize = 6;
@@ -172,7 +173,7 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command)
+            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None)
             .unwrap(),
     )
     .unwrap();

src/export/json.rs

@@ -22,6 +22,7 @@ impl Exporter for JsonExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary { results });
         if let Ok(ref mut content) = output {

src/export/markup.rs

@@ -1,10 +1,11 @@
 use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
 use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
 use crate::options::SortOrder;
-use crate::quantity::{FormatQuantity, IsUnit, TimeUnit};
+use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero};
 
 use super::Exporter;
 use anyhow::Result;
+use std::cmp::Ordering;
 
 pub enum Alignment {
     Left,
@@ -16,6 +17,8 @@ pub trait MarkupExporter {
         &self,
         entries: &[BenchmarkResultWithRelativeSpeed],
         unit: TimeUnit,
+        reference_pending: bool,
+        explicit_reference: bool,
     ) -> String {
         // prepare table header strings
         let notation = format!("[{}]", unit.short_name());
@@ -58,14 +61,35 @@ pub trait MarkupExporter {
             };
             let min_str = measurement.measurements.min().format_value(unit);
             let max_str = measurement.measurements.max().format_value(unit);
-            let rel_str = format!("{:.2}", entry.relative_speed);
-            let rel_stddev_str = if entry.is_reference {
+            // The ratio of two zero times is undefined, even if they compare equal.
+            let relative_unavailable = reference_pending
+                || (explicit_reference
+                    && !entry.is_reference
+                    && entry.relative_ordering == Ordering::Equal
+                    && measurement.mean_wall_clock_time() == Time::zero());
+            let rel_str = if relative_unavailable {
+                "N/A".to_string()
+            } else {
+                format!("{:.2}", entry.relative_speed)
+            };
+            let rel_stddev_str = if relative_unavailable || entry.is_reference {
                 "".into()
             } else if let Some(stddev) = entry.relative_speed_stddev {
                 format!(" ± {stddev:.2}")
             } else {
                 "".into()
             };
+            let rel_direction = if relative_unavailable || !explicit_reference {
+                ""
+            } else if entry.is_reference {
+                " (reference)"
+            } else {
+                match entry.relative_ordering {
+                    Ordering::Less => " (faster)",
+                    Ordering::Greater => " (slower)",
+                    Ordering::Equal => " (same speed)",
+                }
+            };
 
             // prepare table row entries
             table.push_str(&self.table_row(&[
@@ -73,7 +97,7 @@ pub trait MarkupExporter {
                 &format!("{mean_str}{stddev_str}"),
                 &min_str,
                 &max_str,
-                &format!("{rel_str}{rel_stddev_str}"),
+                &format!("{rel_str}{rel_stddev_str}{rel_direction}"),
             ]))
         }
 
@@ -114,11 +138,20 @@ impl<T: MarkupExporter> Exporter for T {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
-        let entries = relative_speed::compute(results, sort_order);
+        // Do not report ratios against another benchmark while the selected
+        // reference is still pending in an intermediate export.
+        let reference_pending = reference_index.is_some_and(|i| i >= results.len());
+        let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) {
+            relative_speed::compute_relative_speeds(results, reference, sort_order)
+        } else {
+            relative_speed::compute(results, sort_order)
+        };
 
-        let table = self.table_results(&entries, unit);
+        let table =
+            self.table_results(&entries, unit, reference_pending, reference_index.is_some());
         Ok(table.as_bytes().to_vec())
     }
 }

src/export/mod.rs

@@ -51,6 +51,7 @@ trait Exporter {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference_index: Option<usize>,
     ) -> Result<Vec<u8>>;
 }
 
@@ -130,11 +131,18 @@ impl ExportManager {
     /// results are written to all file targets (to always have them up to date, even
     /// if a benchmark fails). In the latter case, we only print to stdout targets (in
     /// order not to clutter the output of hyperfine with intermediate results).
-    pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
+    /// `reference_index` refers to the complete benchmark sequence, so it can
+    /// point past `results` while that benchmark is still pending.
+    pub fn write_results(
+        &self,
+        results: &[BenchmarkResult],
+        intermediate: bool,
+        reference_index: Option<usize>,
+    ) -> Result<()> {
         for e in &self.exporters {
             let content = || {
                 e.exporter
-                    .serialize(results, self.time_unit, self.sort_order)
+                    .serialize(results, self.time_unit, self.sort_order, reference_index)
             };
 
             match e.target {

src/export/tests.rs

@@ -12,9 +12,174 @@ fn get_output<E: Exporter + Default>(
     results: &[BenchmarkResult],
     unit: Option<TimeUnit>,
     sort_order: SortOrder,
+) -> String {
+    get_output_with_reference::<E>(results, unit, sort_order, None)
+}
+
+fn get_output_with_reference<E: Exporter + Default>(
+    results: &[BenchmarkResult],
+    unit: Option<TimeUnit>,
+    sort_order: SortOrder,
+    reference_index: Option<usize>,
 ) -> String {
     let exporter = E::default();
-    String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap()
+    String::from_utf8(
+        exporter
+            .serialize(results, unit, sort_order, reference_index)
+            .unwrap(),
+    )
+    .unwrap()
+}
+
+#[test]
+fn test_markup_exports_identify_explicit_reference_and_direction() {
+    let benchmark = |name: &str, seconds| {
+        let measurement = Measurement {
+            time_wall_clock: Time::new::<second>(seconds),
+            ..Measurement::default()
+        };
+        BenchmarkResult {
+            command: name.to_owned(),
+            command_with_unused_parameters: name.to_owned(),
+            measurements: Measurements::new(vec![measurement, measurement]),
+            ..BenchmarkResult::default()
+        }
+    };
+    let results = [
+        benchmark("faster", 1.0),
+        benchmark("reference", 2.0),
+        benchmark("equal", 2.0),
+        benchmark("slower", 3.0),
+    ];
+
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        let outputs = [
+            get_output_with_reference::<MarkdownExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<AsciidocExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<OrgmodeExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+        ];
+        for output in outputs {
+            for expected in [
+                "2.00 ± 0.00 (faster)",
+                "1.00 (reference)",
+                "1.00 ± 0.00 (same speed)",
+                "1.50 ± 0.00 (slower)",
+            ] {
+                assert!(output.contains(expected), "{}", output);
+            }
+        }
+    }
+}
+
+#[test]
+fn test_markup_exports_leave_pending_reference_ratios_unavailable() {
+    let results = [BenchmarkResult {
+        command: "sleep 1".into(),
+        command_with_unused_parameters: "sleep 1".into(),
+        measurements: Measurements::new(vec![Measurement {
+            time_wall_clock: Time::new::<second>(1.0),
+            ..Measurement::default()
+        }]),
+        ..BenchmarkResult::default()
+    }];
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        let outputs = [
+            get_output_with_reference::<MarkdownExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<AsciidocExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<OrgmodeExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+        ];
+        for output in outputs {
+            assert!(output.contains("N/A"), "{}", output);
+            assert!(!output.contains("(reference)"), "{}", output);
+        }
+    }
+}
+
+#[test]
+fn test_markup_exports_with_zero_reference() {
+    let benchmark = |name: &str, seconds| BenchmarkResult {
+        command: name.to_owned(),
+        command_with_unused_parameters: name.to_owned(),
+        measurements: Measurements::new(vec![Measurement {
+            time_wall_clock: Time::new::<second>(seconds),
+            ..Measurement::default()
+        }]),
+        ..BenchmarkResult::default()
+    };
+    let results = [
+        benchmark("other zero", 0.0),
+        benchmark("reference", 0.0),
+        benchmark("slower", 1.0),
+    ];
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        for reference in [Some(1), Some(3), None] {
+            let outputs = [
+                get_output_with_reference::<MarkdownExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+                get_output_with_reference::<AsciidocExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+                get_output_with_reference::<OrgmodeExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+            ];
+            for output in outputs {
+                match reference {
+                    Some(1) => {
+                        assert_eq!(output.matches("N/A").count(), 1, "{}", output);
+                        assert!(output.contains("1.00 (reference)"), "{}", output);
+                        assert!(output.contains("inf (slower)"), "{}", output);
+                    }
+                    Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output),
+                    None => {
+                        assert!(!output.contains("N/A"), "{}", output);
+                        assert!(output.contains("inf"), "{}", output);
+                        assert!(!output.contains("(reference)"), "{}", output);
+                    }
+                }
+            }
+        }
+    }
 }
 
 /// Ensure the makrup output includes the table header and the multiple