Use parameterized benchmarks as references

README.md

@@ -98,6 +98,16 @@ hyperfine \
     'python main.py'
 ```
 
+To compare a parameterized benchmark against a particular run, pass its exact printed name to
+`--reference`. For example:
+```sh
+hyperfine -L delay 0.2,0.4,0.6 --prepare 'sleep {delay}' --reference 'echo a (delay = 0.4)' 'echo a'
+```
+The name must identify exactly one benchmark, including any `--command-name` and displayed unused
+parameters. `--reference-name` cannot be used with parameterized benchmarks; use `--command-name`
+to name them instead. Without `--parameter-scan` or `--parameter-list`, `--reference` runs a
+separate reference command.
+
 If you need a unique value for each individual run of a benchmark command, hyperfine also exposes
 the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself:
 ```sh

doc/hyperfine.1

@@ -116,13 +116,17 @@ not every time as would happen with the \fB\-\-prepare\fR option.
 .HP
 \fB\-\-reference\fR \fICMD\fP
 .IP
-Benchmark \fICMD\fP as the reference for the relative comparison of results.
+Use \fICMD\fP as the reference for the relative comparison of results. Without
-If this option is not specified, the fastest command is used as the reference.
+parameters, benchmark \fICMD\fP as a separate reference command. If this option
+is not specified, the fastest command is used as the reference.
+With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing
+benchmark by its exact printed name. The name must match exactly one benchmark.
 .HP
 \fB\-\-reference\-name\fR \fINAME\fP
 .IP
 Give a meaningful \fINAME\fP to the reference command.
 Requires \fB\-\-reference\fR.
+Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead.
 .HP
 \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP
 .IP

src/benchmark/relative_speed.rs

@@ -29,7 +29,7 @@ pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
         .expect("at least one benchmark result")
 }
 
-fn compute_relative_speeds<'a>(
+pub fn compute_relative_speeds<'a>(
     results: &'a [BenchmarkResult],
     reference: &'a BenchmarkResult,
     sort_order: SortOrder,
@@ -37,10 +37,13 @@ fn compute_relative_speeds<'a>(
     let mut results: Vec<_> = results
         .iter()
         .map(|result| {
-            let is_reference = result == reference;
+            // Separate benchmarks can have identical names and measurements.
+            let is_reference = std::ptr::eq(result, reference);
             let relative_ordering = compare_mean_time(result, reference);
 
-            if result.mean_wall_clock_time() == Time::zero() {
+            if result.mean_wall_clock_time() == Time::zero()
+                || reference.mean_wall_clock_time() == Time::zero()
+            {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
                     relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -64,11 +67,11 @@ fn compute_relative_speeds<'a>(
                 result.measurements.stddev(),
                 reference.measurements.stddev(),
             ) {
-                (Some(result_stddev), Some(fastest_stddev)) => Some(
+                (Some(result_stddev), Some(reference_stddev)) => Some(
                     ratio
                         * ((result_stddev / result.mean_wall_clock_time())
                             .powi(uom::typenum::P2::new())
-                            + (fastest_stddev / reference.mean_wall_clock_time())
+                            + (reference_stddev / reference.mean_wall_clock_time())
                                 .powi(uom::typenum::P2::new()))
                         .sqrt(),
                 ),
@@ -192,3 +195,72 @@ fn test_compute_relative_speed_for_zero_times() {
 
     assert!(annotated_results.is_none());
 }
+
+#[test]
+fn reference_identity_distinguishes_equal_results() {
+    let results = vec![create_result("same", 1.0), create_result("same", 1.0)];
+    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
+
+    assert!(entries[0].is_reference);
+    assert!(!entries[1].is_reference);
+}
+
+#[test]
+fn reference_ratios_indicate_faster_and_slower_runs() {
+    use approx::assert_relative_eq;
+
+    let results = vec![
+        create_result("faster", 1.0),
+        create_result("reference", 2.0),
+        create_result("slower", 3.0),
+    ];
+    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
+
+    assert_relative_eq!(entries[0].relative_speed, 2.0);
+    assert_relative_eq!(entries[1].relative_speed, 1.0);
+    assert_relative_eq!(entries[2].relative_speed, 1.5);
+    assert!(!entries[0].is_reference);
+    assert!(entries[1].is_reference);
+    assert!(!entries[2].is_reference);
+}
+
+#[test]
+fn reference_ratios_handle_zero_times() {
+    let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
+    for result in &mut results {
+        let measurement = result.measurements.measurements[0];
+        result.measurements.measurements.push(measurement);
+    }
+    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
+    assert_eq!(entries[1].relative_speed, f64::INFINITY);
+
+    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
+    assert_eq!(entries[0].relative_speed, f64::INFINITY);
+    assert_eq!(entries[0].relative_speed_stddev, None);
+    assert_eq!(entries[1].relative_speed, 1.0);
+}
+
+#[test]
+fn reference_ratio_propagates_uncertainty() {
+    use crate::benchmark::measurement::Measurement;
+    use crate::quantity::second;
+    use approx::assert_relative_eq;
+
+    let mut results = vec![
+        create_result("faster", 1.0),
+        create_result("reference", 3.0),
+    ];
+    for (result, times) in results.iter_mut().zip([[1.0, 2.0, 3.0], [3.0, 4.0, 5.0]]) {
+        result.measurements.measurements = times
+            .iter()
+            .copied()
+            .map(|seconds| Measurement {
+                time_wall_clock: Time::new::<second>(seconds),
+                ..Default::default()
+            })
+            .collect();
+    }
+    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
+    assert_relative_eq!(entries[0].relative_speed, 2.0);
+    assert_relative_eq!(entries[0].relative_speed_stddev.unwrap(), 1.118033988749895);
+}

src/benchmark/scheduler.rs

@@ -44,6 +44,7 @@ impl<'a> Scheduler<'a> {
             .options
             .reference_command
             .as_ref()
+            .filter(|_| !self.options.parameterized_mode)
             .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
 
         executor.calibrate()?;
@@ -54,7 +55,8 @@ impl<'a> Scheduler<'a> {
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.
-            self.export_manager.write_results(&self.results, true)?;
+            self.export_manager
+                .write_results(&self.results, true, self.reference_index())?;
         }
 
         Ok(())
@@ -70,10 +72,8 @@ impl<'a> Scheduler<'a> {
         }
 
         let reference = self
-            .options
+            .reference_index()
-            .reference_command
+            .map(|index| &self.results[index])
-            .as_ref()
-            .map(|_| &self.results[0])
             .unwrap_or_else(|| relative_speed::fastest_of(&self.results));
 
         if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
@@ -130,9 +130,20 @@ impl<'a> Scheduler<'a> {
                     console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
 
                     for item in annotated_results {
+                        let relationship = if self.reference_index().is_none() {
+                            ""
+                        } else if item.is_reference {
+                            " (reference)"
+                        } else {
+                            match item.relative_ordering {
+                                Ordering::Less => " (faster)",
+                                Ordering::Equal => " (same speed)",
+                                Ordering::Greater => " (slower)",
+                            }
+                        };
                         console_writeln!(
                             stdout,
-                            "  {}{}  {}",
+                            "  {}{}  {}{}",
                             format!("{:10.2}", item.relative_speed).bold().green(),
                             if item.is_reference {
                                 "        ".into()
@@ -142,6 +153,7 @@ impl<'a> Scheduler<'a> {
                                 "        ".into()
                             },
                             item.result.command_with_unused_parameters,
+                            relationship,
                         )?;
                     }
                 }
@@ -162,8 +174,16 @@ impl<'a> Scheduler<'a> {
         Ok(())
     }
 
+    fn reference_index(&self) -> Option<usize> {
+        self.options
+            .reference_command
+            .as_ref()
+            .map(|_| self.options.parameterized_reference_index.unwrap_or(0))
+    }
+
     pub fn final_export(&self) -> Result<()> {
-        self.export_manager.write_results(&self.results, false)
+        self.export_manager
+            .write_results(&self.results, false, self.reference_index())
     }
 }
 

src/cli.rs

@@ -105,8 +105,11 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("CMD")
                 .help(
-                    "The reference command for the relative comparison of results. \
+                    "The reference for the relative comparison of results. Without parameters, \
-                    If this is unset, results are compared with the fastest command as reference."
+                    CMD is run as a separate reference command. With --parameter-scan or \
+                    --parameter-list, CMD must exactly match one unique benchmark name as \
+                    shown in the output. If this is unset, results are compared with the \
+                    fastest command as reference."
                 )
         )
         .arg(
@@ -114,7 +117,8 @@ fn build_command() -> Command {
                 .long("reference-name")
                 .action(ArgAction::Set)
                 .value_name("CMD")
-                .help("Give a meaningful name to the reference command.")
+                .help("Give a meaningful name to the reference command. This cannot be used \
+                       with parameterized benchmarks; use --command-name instead.")
                 .requires("reference")
         )
         .arg(

src/export/csv.rs

@@ -18,6 +18,7 @@ impl Exporter for CsvExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         const CSV_UNIT: TimeUnit = TimeUnit::Second;
         const CSV_PRECISION: usize = 6;
@@ -172,7 +173,7 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command)
+            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None)
             .unwrap(),
     )
     .unwrap();

src/export/json.rs

@@ -22,6 +22,7 @@ impl Exporter for JsonExporter {
         results: &[BenchmarkResult],
         _unit: Option<TimeUnit>,
         _sort_order: SortOrder,
+        _reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary { results });
         if let Ok(ref mut content) = output {

src/export/markup.rs

@@ -1,10 +1,11 @@
 use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
 use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
 use crate::options::SortOrder;
-use crate::quantity::{FormatQuantity, IsUnit, TimeUnit};
+use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero};
 
 use super::Exporter;
 use anyhow::Result;
+use std::cmp::Ordering;
 
 pub enum Alignment {
     Left,
@@ -16,6 +17,8 @@ pub trait MarkupExporter {
         &self,
         entries: &[BenchmarkResultWithRelativeSpeed],
         unit: TimeUnit,
+        reference_pending: bool,
+        explicit_reference: bool,
     ) -> String {
         // prepare table header strings
         let notation = format!("[{}]", unit.short_name());
@@ -58,14 +61,35 @@ pub trait MarkupExporter {
             };
             let min_str = measurement.measurements.min().format_value(unit);
             let max_str = measurement.measurements.max().format_value(unit);
-            let rel_str = format!("{:.2}", entry.relative_speed);
+            // The ratio of two zero times is undefined, even if they compare equal.
-            let rel_stddev_str = if entry.is_reference {
+            let relative_unavailable = reference_pending
+                || (explicit_reference
+                    && !entry.is_reference
+                    && entry.relative_ordering == Ordering::Equal
+                    && measurement.mean_wall_clock_time() == Time::zero());
+            let rel_str = if relative_unavailable {
+                "N/A".to_string()
+            } else {
+                format!("{:.2}", entry.relative_speed)
+            };
+            let rel_stddev_str = if relative_unavailable || entry.is_reference {
                 "".into()
             } else if let Some(stddev) = entry.relative_speed_stddev {
                 format!(" ± {stddev:.2}")
             } else {
                 "".into()
             };
+            let rel_direction = if relative_unavailable || !explicit_reference {
+                ""
+            } else if entry.is_reference {
+                " (reference)"
+            } else {
+                match entry.relative_ordering {
+                    Ordering::Less => " (faster)",
+                    Ordering::Greater => " (slower)",
+                    Ordering::Equal => " (same speed)",
+                }
+            };
 
             // prepare table row entries
             table.push_str(&self.table_row(&[
@@ -73,7 +97,7 @@ pub trait MarkupExporter {
                 &format!("{mean_str}{stddev_str}"),
                 &min_str,
                 &max_str,
-                &format!("{rel_str}{rel_stddev_str}"),
+                &format!("{rel_str}{rel_stddev_str}{rel_direction}"),
             ]))
         }
 
@@ -114,11 +138,20 @@ impl<T: MarkupExporter> Exporter for T {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference_index: Option<usize>,
     ) -> Result<Vec<u8>> {
         let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
-        let entries = relative_speed::compute(results, sort_order);
+        // Do not report ratios against another benchmark while the selected
+        // reference is still pending in an intermediate export.
+        let reference_pending = reference_index.is_some_and(|i| i >= results.len());
+        let entries = if let Some(reference) = reference_index.and_then(|i| results.get(i)) {
+            relative_speed::compute_relative_speeds(results, reference, sort_order)
+        } else {
+            relative_speed::compute(results, sort_order)
+        };
 
-        let table = self.table_results(&entries, unit);
+        let table =
+            self.table_results(&entries, unit, reference_pending, reference_index.is_some());
         Ok(table.as_bytes().to_vec())
     }
 }

src/export/mod.rs

@@ -51,6 +51,7 @@ trait Exporter {
         results: &[BenchmarkResult],
         unit: Option<TimeUnit>,
         sort_order: SortOrder,
+        reference_index: Option<usize>,
     ) -> Result<Vec<u8>>;
 }
 
@@ -130,11 +131,18 @@ impl ExportManager {
     /// results are written to all file targets (to always have them up to date, even
     /// if a benchmark fails). In the latter case, we only print to stdout targets (in
     /// order not to clutter the output of hyperfine with intermediate results).
-    pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
+    /// `reference_index` refers to the complete benchmark sequence, so it can
+    /// point past `results` while that benchmark is still pending.
+    pub fn write_results(
+        &self,
+        results: &[BenchmarkResult],
+        intermediate: bool,
+        reference_index: Option<usize>,
+    ) -> Result<()> {
         for e in &self.exporters {
             let content = || {
                 e.exporter
-                    .serialize(results, self.time_unit, self.sort_order)
+                    .serialize(results, self.time_unit, self.sort_order, reference_index)
             };
 
             match e.target {

src/export/tests.rs

@@ -12,9 +12,174 @@ fn get_output<E: Exporter + Default>(
     results: &[BenchmarkResult],
     unit: Option<TimeUnit>,
     sort_order: SortOrder,
+) -> String {
+    get_output_with_reference::<E>(results, unit, sort_order, None)
+}
+
+fn get_output_with_reference<E: Exporter + Default>(
+    results: &[BenchmarkResult],
+    unit: Option<TimeUnit>,
+    sort_order: SortOrder,
+    reference_index: Option<usize>,
 ) -> String {
     let exporter = E::default();
-    String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap()
+    String::from_utf8(
+        exporter
+            .serialize(results, unit, sort_order, reference_index)
+            .unwrap(),
+    )
+    .unwrap()
+}
+
+#[test]
+fn test_markup_exports_identify_explicit_reference_and_direction() {
+    let benchmark = |name: &str, seconds| {
+        let measurement = Measurement {
+            time_wall_clock: Time::new::<second>(seconds),
+            ..Measurement::default()
+        };
+        BenchmarkResult {
+            command: name.to_owned(),
+            command_with_unused_parameters: name.to_owned(),
+            measurements: Measurements::new(vec![measurement, measurement]),
+            ..BenchmarkResult::default()
+        }
+    };
+    let results = [
+        benchmark("faster", 1.0),
+        benchmark("reference", 2.0),
+        benchmark("equal", 2.0),
+        benchmark("slower", 3.0),
+    ];
+
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        let outputs = [
+            get_output_with_reference::<MarkdownExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<AsciidocExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<OrgmodeExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+        ];
+        for output in outputs {
+            for expected in [
+                "2.00 ± 0.00 (faster)",
+                "1.00 (reference)",
+                "1.00 ± 0.00 (same speed)",
+                "1.50 ± 0.00 (slower)",
+            ] {
+                assert!(output.contains(expected), "{}", output);
+            }
+        }
+    }
+}
+
+#[test]
+fn test_markup_exports_leave_pending_reference_ratios_unavailable() {
+    let results = [BenchmarkResult {
+        command: "sleep 1".into(),
+        command_with_unused_parameters: "sleep 1".into(),
+        measurements: Measurements::new(vec![Measurement {
+            time_wall_clock: Time::new::<second>(1.0),
+            ..Measurement::default()
+        }]),
+        ..BenchmarkResult::default()
+    }];
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        let outputs = [
+            get_output_with_reference::<MarkdownExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<AsciidocExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+            get_output_with_reference::<OrgmodeExporter>(
+                &results,
+                Some(TimeUnit::Second),
+                sort_order,
+                Some(1),
+            ),
+        ];
+        for output in outputs {
+            assert!(output.contains("N/A"), "{}", output);
+            assert!(!output.contains("(reference)"), "{}", output);
+        }
+    }
+}
+
+#[test]
+fn test_markup_exports_with_zero_reference() {
+    let benchmark = |name: &str, seconds| BenchmarkResult {
+        command: name.to_owned(),
+        command_with_unused_parameters: name.to_owned(),
+        measurements: Measurements::new(vec![Measurement {
+            time_wall_clock: Time::new::<second>(seconds),
+            ..Measurement::default()
+        }]),
+        ..BenchmarkResult::default()
+    };
+    let results = [
+        benchmark("other zero", 0.0),
+        benchmark("reference", 0.0),
+        benchmark("slower", 1.0),
+    ];
+    for sort_order in [SortOrder::Command, SortOrder::MeanTime] {
+        for reference in [Some(1), Some(3), None] {
+            let outputs = [
+                get_output_with_reference::<MarkdownExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+                get_output_with_reference::<AsciidocExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+                get_output_with_reference::<OrgmodeExporter>(
+                    &results,
+                    Some(TimeUnit::Second),
+                    sort_order,
+                    reference,
+                ),
+            ];
+            for output in outputs {
+                match reference {
+                    Some(1) => {
+                        assert_eq!(output.matches("N/A").count(), 1, "{}", output);
+                        assert!(output.contains("1.00 (reference)"), "{}", output);
+                        assert!(output.contains("inf (slower)"), "{}", output);
+                    }
+                    Some(_) => assert_eq!(output.matches("N/A").count(), 3, "{}", output),
+                    None => {
+                        assert!(!output.contains("N/A"), "{}", output);
+                        assert!(output.contains("inf"), "{}", output);
+                        assert!(!output.contains("(reference)"), "{}", output);
+                    }
+                }
+            }
+        }
+    }
 }
 
 /// Ensure the makrup output includes the table header and the multiple

src/options.rs

@@ -4,7 +4,7 @@ use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::{cmp, env, fmt, io};
 
-use anyhow::ensure;
+use anyhow::{bail, ensure};
 use clap::ArgMatches;
 
 use crate::command::Commands;
@@ -214,6 +214,12 @@ pub struct Options {
     // Name of the reference command
     pub reference_name: Option<String>,
 
+    // Whether commands are expanded from --parameter-scan or --parameter-list.
+    pub parameterized_mode: bool,
+
+    // Index of the selected command in a parameterized run, if any.
+    pub parameterized_reference_index: Option<usize>,
+
     /// Command(s) to run before each timing run
     pub preparation_command: Option<Vec<String>>,
 
@@ -257,6 +263,8 @@ impl Default for Options {
             command_failure_action: CmdFailureAction::RaiseError,
             reference_command: None,
             reference_name: None,
+            parameterized_mode: false,
+            parameterized_reference_index: None,
             preparation_command: None,
             conclusion_command: None,
             setup_command: None,
@@ -309,6 +317,8 @@ impl Options {
 
         options.setup_command = matches.get_one::<String>("setup").map(String::from);
 
+        options.parameterized_mode = matches.get_many::<String>("parameter-scan").is_some()
+            || matches.get_many::<String>("parameter-list").is_some();
         options.reference_command = matches.get_one::<String>("reference").map(String::from);
         options.reference_name = matches
             .get_one::<String>("reference-name")
@@ -464,7 +474,29 @@ impl Options {
     }
 
     pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> {
-        let has_reference_command = self.reference_command.is_some();
+        if self.parameterized_mode {
+            if let Some(reference) = &self.reference_command {
+                ensure!(
+                    self.reference_name.is_none(),
+                    "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark"
+                );
+                let mut matches = commands
+                    .iter()
+                    .enumerate()
+                    .filter(|(_, command)| command.get_name_with_unused_parameters() == *reference);
+                let Some((index, _)) = matches.next() else {
+                    bail!(
+                        "Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name."
+                    );
+                };
+                ensure!(
+                    matches.next().is_none(),
+                    "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names."
+                );
+                self.parameterized_reference_index = Some(index);
+            }
+        }
+        let has_reference_command = self.reference_command.is_some() && !self.parameterized_mode;
         let num_commands = commands.num_commands(has_reference_command);
 
         if let Some(preparation_command) = &self.preparation_command {

tests/execution_order_tests.rs

@@ -384,16 +384,14 @@ fn reference_is_executed_first() {
 }
 
 #[test]
-fn reference_is_executed_first_parameter_value() {
+fn parameterized_reference_keeps_parameter_order() {
     ExecutionOrderTest::new()
         .arg("--runs=2")
-        .reference("reference")
+        .reference("command 2")
         .arg("--parameter-list")
         .arg("number")
         .arg("1,2,3")
         .command("command {number}")
-        .expect_output("reference")
-        .expect_output("reference")
         .expect_output("command 1")
         .expect_output("command 1")
         .expect_output("command 2")

tests/integration_tests.rs

@@ -705,6 +705,40 @@ fn shows_benchmark_comparison_relative_to_reference() {
     ");
 }
 
+#[test]
+fn command_sorted_comparison_labels_explicit_reference() {
+    for (args, expected) in [
+        (
+            vec!["--reference", "sleep 2", "sleep 1", "sleep 3"],
+            vec![
+                "1.00          sleep 2 (reference)",
+                "2.00 ±  0.00  sleep 1 (faster)",
+                "1.50 ±  0.00  sleep 3 (slower)",
+            ],
+        ),
+        (
+            vec!["--reference", "sleep 2", "sleep 2"],
+            vec![
+                "1.00          sleep 2 (reference)",
+                "1.00 ±  0.00  sleep 2 (same speed)",
+            ],
+        ),
+    ] {
+        let output = hyperfine_debug()
+            .args(["--style=basic", "--runs=2", "--sort=command"])
+            .args(args)
+            .output()
+            .unwrap();
+        assert!(output.status.success(), "{:?}", output);
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1;
+        assert_eq!(
+            comparison.lines().map(str::trim).collect::<Vec<_>>(),
+            expected
+        );
+    }
+}
+
 #[test]
 fn shows_reference_name() {
     let _settings = snapshot_settings().bind_to_scope();
@@ -796,50 +830,453 @@ fn rejects_negative_parameter_steps() {
 }
 
 #[test]
-fn performs_reference_and_all_benchmarks_in_parameter_scan() {
+fn selects_reference_from_parameter_scan() {
-    let _settings = snapshot_settings().bind_to_scope();
+    let output = hyperfine_debug()
-    assert_cmd_snapshot!(hyperfine_debug()
+        .args([
-        .arg("--style=basic")
+            "--style=basic",
-        .arg("--reference=sleep 25")
+            "--reference=sleep 35",
-        .arg("--parameter-scan")
+            "--parameter-scan",
-        .arg("time")
+            "time",
-        .arg("30")
+            "30",
-        .arg("45")
+            "45",
-        .arg("--parameter-step-size")
+            "--parameter-step-size",
-        .arg("5")
+            "5",
-        .arg("sleep {time}"), @r"
+            "sleep {time}",
-    success: true
+        ])
-    exit_code: 0
+        .output()
-    ----- stdout -----
+        .unwrap();
-    Benchmark 1: sleep 25
+    assert!(output.status.success(), "{:?}", output);
-      Time (mean ± σ):     25.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+    let stdout = String::from_utf8(output.stdout).unwrap();
-      Range (min … max):   25.000 s … 25.000 s    10 runs
+    let benchmarks: Vec<_> = stdout
+        .lines()
+        .filter(|line| line.starts_with("Benchmark "))
+        .collect();
+    assert_eq!(
+        benchmarks,
+        [
+            "Benchmark 1: sleep 30",
+            "Benchmark 2: sleep 35",
+            "Benchmark 3: sleep 40",
+            "Benchmark 4: sleep 45",
+        ]
+    );
+    assert!(stdout.contains("Summary\n  sleep 35 ran\n"), "{}", stdout);
+}
 
-    Benchmark 2: sleep 30
+#[cfg(unix)]
-      Time (mean ± σ):     30.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+#[test]
-      Range (min … max):   30.000 s … 30.000 s    10 runs
+fn selects_reference_with_parameterized_prepare() {
+    let directory = tempfile::tempdir().unwrap();
+    let csv_path = directory.path().join("results.csv");
+    let json_path = directory.path().join("results.json");
+    let output = hyperfine_raw_command()
+        .args([
+            "--style=basic",
+            "--shell=none",
+            "--runs=1",
+            "-L",
+            "delay",
+            "0.2,0.4,0.6",
+            "--prepare",
+            "sleep {delay}",
+            "--reference",
+            "echo a (delay = 0.4)",
+            "--export-csv",
+        ])
+        .arg(&csv_path)
+        .arg("--export-json")
+        .arg(&json_path)
+        .arg("echo a")
+        .output()
+        .unwrap();
+    assert!(output.status.success(), "{:?}", output);
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    assert_eq!(stdout.matches("Benchmark ").count(), 3);
+    assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout);
 
-    Benchmark 3: sleep 35
+    let mut csv = csv::Reader::from_path(csv_path).unwrap();
-      Time (mean ± σ):     35.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+    assert!(csv
-      Range (min … max):   35.000 s … 35.000 s    10 runs
+        .headers()
+        .unwrap()
+        .iter()
+        .any(|header| header == "parameter_delay"));
+    let records: Vec<_> = csv.records().map(|record| record.unwrap()).collect();
+    assert_eq!(records.len(), 3);
+    let json: serde_json::Value =
+        serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap();
+    let results = json["results"].as_array().unwrap();
+    assert_eq!(results.len(), 3);
+    assert_eq!(
+        results
+            .iter()
+            .map(|result| result["parameters"]["delay"].as_str().unwrap())
+            .collect::<Vec<_>>(),
+        ["0.2", "0.4", "0.6"]
+    );
+}
 
-    Benchmark 4: sleep 40
+#[test]
-      Time (mean ± σ):     40.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+fn selects_reference_by_custom_benchmark_name() {
-      Range (min … max):   40.000 s … 40.000 s    10 runs
+    for (command, name, reference) in [
+        ("sleep {x}", "case-{x}", "case-2"),
+        ("sleep 1", "case-{x}", "case-2 (x = 2)"),
+    ] {
+        let output = hyperfine_debug()
+            .args([
+                "--style=basic",
+                "--runs=1",
+                "-L",
+                "x",
+                "1,2",
+                "--command-name",
+                name,
+                "--reference",
+                reference,
+                command,
+            ])
+            .output()
+            .unwrap();
+        assert!(output.status.success(), "{:?}", output);
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        assert_eq!(stdout.matches("Benchmark ").count(), 2);
+        assert!(stdout.contains(&format!("{reference} ran")), "{}", stdout);
+    }
+}
 
-    Benchmark 5: sleep 45
+#[test]
-      Time (mean ± σ):     45.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
+fn selects_reference_from_multiple_commands_and_parameters() {
-      Range (min … max):   45.000 s … 45.000 s    10 runs
+    let output = hyperfine_debug()
+        .args([
+            "--style=basic",
+            "--runs=1",
+            "-L",
+            "x",
+            "1,2",
+            "-L",
+            "y",
+            "3,4",
+            "--reference",
+            "sleep 4 (x = 2)",
+            "sleep {x}",
+            "sleep {y}",
+        ])
+        .output()
+        .unwrap();
+    assert!(output.status.success(), "{:?}", output);
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    assert_eq!(stdout.matches("Benchmark ").count(), 8);
+    assert!(stdout.contains("sleep 4 (x = 2) ran"), "{}", stdout);
+}
 
-    Summary
+#[test]
-      sleep 25 ran
+fn rejects_unmatched_parameterized_reference_before_creating_export() {
-        1.20 ± 0.00 times faster than sleep 30
+    let directory = tempfile::tempdir().unwrap();
-        1.40 ± 0.00 times faster than sleep 35
+    let export_path = directory.path().join("results.csv");
-        1.60 ± 0.00 times faster than sleep 40
+    let marker_path = directory.path().join("setup-ran");
-        1.80 ± 0.00 times faster than sleep 45
+    for (args, benchmark) in [
+        (vec!["-L", "x", "1,2", "--reference", "echo 3"], "echo {x}"),
+        (
+            vec!["--reference", "sleep 1", "-P", "secs", "2", "3"],
+            "sleep {secs}",
+        ),
+    ] {
+        std::fs::write(&export_path, "previous contents").unwrap();
+        hyperfine()
+            .args(args)
+            .arg("--setup")
+            .arg(format!("echo touched > \"{}\"", marker_path.display()))
+            .arg("--export-csv")
+            .arg(&export_path)
+            .arg(benchmark)
+            .assert()
+            .failure()
+            .stdout(predicate::str::is_empty())
+            .stderr(predicate::str::contains(
+                "does not match any parameterized benchmark",
+            ));
 
-    ----- stderr -----
+        assert_eq!(
-    ");
+            std::fs::read_to_string(&export_path).unwrap(),
+            "previous contents"
+        );
+        assert!(!marker_path.exists());
+    }
+}
+
+#[test]
+fn rejects_ambiguous_parameterized_reference() {
+    for args in [
+        vec![
+            "-L",
+            "x",
+            "1,2",
+            "--command-name",
+            "case",
+            "--reference",
+            "case",
+            "sleep {x}",
+        ],
+        vec!["-L", "x", "1,1", "--reference", "sleep 1", "sleep {x}"],
+    ] {
+        hyperfine_debug()
+            .args(args)
+            .assert()
+            .failure()
+            .stdout(predicate::str::is_empty())
+            .stderr(predicate::str::contains(
+                "matches multiple parameterized benchmarks",
+            ));
+    }
+}
+
+#[test]
+fn rejects_reference_name_with_parameterized_reference() {
+    hyperfine_debug()
+        .args([
+            "-L",
+            "x",
+            "1,2",
+            "--reference",
+            "sleep 1",
+            "--reference-name",
+            "baseline",
+            "sleep {x}",
+        ])
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "--reference-name cannot be used with a parameterized reference",
+        ));
+}
+
+#[test]
+fn parameterized_reference_preserves_per_benchmark_hook_order() {
+    let directory = tempfile::tempdir().unwrap();
+    let log_path = directory.path().join("hooks.log");
+    let prepare = |name: &str| format!("echo {name} >> \"{}\"", log_path.display());
+    let command = format!("echo benchmark-{{x}} >> \"{}\"", log_path.display());
+
+    hyperfine()
+        .args(["--runs=1", "-L", "x", "1,2,3"])
+        .arg("--command-name")
+        .arg("case-{x}")
+        .arg("--reference")
+        .arg("case-2")
+        .arg("--prepare")
+        .arg(prepare("prepare-1"))
+        .arg("--prepare")
+        .arg(prepare("prepare-2"))
+        .arg("--prepare")
+        .arg(prepare("prepare-3"))
+        .arg(command)
+        .assert()
+        .success();
+
+    let contents = std::fs::read_to_string(log_path).unwrap();
+    assert_eq!(
+        contents.lines().map(str::trim).collect::<Vec<_>>(),
+        [
+            "prepare-1",
+            "benchmark-1",
+            "prepare-2",
+            "benchmark-2",
+            "prepare-3",
+            "benchmark-3"
+        ]
+    );
+}
+
+#[test]
+fn parameterized_reference_preserves_per_benchmark_output_order() {
+    let directory = tempfile::tempdir().unwrap();
+    let output_paths = [
+        directory.path().join("one.txt"),
+        directory.path().join("two.txt"),
+        directory.path().join("three.txt"),
+    ];
+    let mut command = hyperfine();
+    command.args([
+        "--runs=1",
+        "-L",
+        "x",
+        "1,2,3",
+        "--command-name",
+        "case-{x}",
+        "--reference",
+        "case-2",
+    ]);
+    for path in &output_paths {
+        command.arg("--output").arg(path);
+    }
+    command.arg("echo benchmark-{x}").assert().success();
+
+    for (path, expected) in output_paths
+        .iter()
+        .zip(["benchmark-1", "benchmark-2", "benchmark-3"])
+    {
+        assert_eq!(std::fs::read_to_string(path).unwrap().trim(), expected);
+    }
+}
+
+#[cfg(unix)]
+#[test]
+fn intermediate_markdown_waits_for_selected_reference() {
+    let directory = tempfile::tempdir().unwrap();
+    for (reference, preparations, reference_available) in [
+        ("sleep 0.03", ["true", "false", "true"], false),
+        ("sleep 0.02", ["true", "true", "false"], true),
+    ] {
+        let export_path = directory.path().join("results.md");
+        let mut command = hyperfine();
+        command.args([
+            "--style=none",
+            "--shell=none",
+            "--runs=1",
+            "-L",
+            "delay",
+            "0.01,0.02,0.03",
+            "--reference",
+            reference,
+        ]);
+        for preparation in preparations {
+            command.arg("--prepare").arg(preparation);
+        }
+        command
+            .arg("--export-markdown")
+            .arg(&export_path)
+            .arg("sleep {delay}")
+            .assert()
+            .failure()
+            .stderr(predicate::str::contains(
+                "The preparation command terminated",
+            ));
+        let markdown = std::fs::read_to_string(&export_path).unwrap();
+        if reference_available {
+            assert!(!markdown.contains("N/A"), "{}", markdown);
+            let row = markdown
+                .lines()
+                .find(|line| line.contains("sleep 0.02"))
+                .unwrap();
+            assert!(row.ends_with("| 1.00 (reference) |"), "{}", row);
+        } else {
+            let row = markdown
+                .lines()
+                .find(|line| line.contains("sleep 0.01"))
+                .unwrap();
+            assert!(row.ends_with("| N/A |"), "{}", row);
+        }
+    }
+}
+
+#[test]
+fn markdown_export_uses_selected_parameterized_reference() {
+    for sort_order in ["command", "mean-time"] {
+        let output = hyperfine_debug()
+            .args([
+                "--style=none",
+                "--runs=1",
+                "--time-unit=second",
+                "--export-markdown=-",
+                "--sort",
+                sort_order,
+                "-L",
+                "x",
+                "3,2,1",
+                "--reference",
+                "sleep 2",
+                "sleep {x}",
+            ])
+            .output()
+            .unwrap();
+        assert!(output.status.success(), "{:?}", output);
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        let rows: Vec<_> = stdout
+            .lines()
+            .filter(|line| line.contains("sleep "))
+            .collect();
+        if sort_order == "command" {
+            assert!(rows[0].contains("sleep 3"));
+            assert!(rows[1].contains("sleep 2"));
+            assert!(rows[2].contains("sleep 1"));
+        } else {
+            assert!(rows[0].contains("sleep 1"));
+            assert!(rows[1].contains("sleep 2"));
+            assert!(rows[2].contains("sleep 3"));
+        }
+        for (command, relative) in [
+            ("sleep 1", "2.00 (faster)"),
+            ("sleep 2", "1.00 (reference)"),
+            ("sleep 3", "1.50 (slower)"),
+        ] {
+            let row = rows.iter().find(|line| line.contains(command)).unwrap();
+            assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
+        }
+    }
+}
+
+#[test]
+fn markdown_export_uses_standalone_reference() {
+    for sort_order in ["command", "mean-time"] {
+        let output = hyperfine_debug()
+            .args([
+                "--style=none",
+                "--runs=1",
+                "--time-unit=second",
+                "--export-markdown=-",
+                "--sort",
+                sort_order,
+                "--reference",
+                "sleep 2",
+                "--reference-name",
+                "baseline",
+                "sleep 1",
+                "sleep 2",
+                "sleep 3",
+            ])
+            .output()
+            .unwrap();
+        assert!(output.status.success(), "{:?}", output);
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        for (command, relative) in [
+            ("baseline", "1.00 (reference)"),
+            ("sleep 1", "2.00 (faster)"),
+            ("sleep 2", "1.00 (same speed)"),
+            ("sleep 3", "1.50 (slower)"),
+        ] {
+            let row = stdout.lines().find(|line| line.contains(command)).unwrap();
+            assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
+        }
+    }
+}
+
+#[test]
+fn markdown_export_without_explicit_reference_is_unchanged() {
+    for sort_order in ["command", "mean-time"] {
+        let output = hyperfine_debug()
+            .args([
+                "--style=none",
+                "--runs=1",
+                "--time-unit=second",
+                "--export-markdown=-",
+                "--sort",
+                sort_order,
+                "sleep 3",
+                "sleep 1",
+                "sleep 2",
+            ])
+            .output()
+            .unwrap();
+        assert!(output.status.success(), "{:?}", output);
+        let stdout = String::from_utf8(output.stdout).unwrap();
+        for (command, relative) in [
+            ("sleep 1", "1.00"),
+            ("sleep 2", "2.00"),
+            ("sleep 3", "3.00"),
+        ] {
+            let row = stdout.lines().find(|line| line.contains(command)).unwrap();
+            assert!(row.ends_with(&format!("| {relative} |")), "{}", row);
+        }
+    }
 }
 
 #[test]