Alternative performance metrics in the terminal (#986)

src/benchmark/benchmark_result.rs

@@ -3,7 +3,6 @@ use std::collections::BTreeMap;
 use serde::Serialize;
 
 use crate::benchmark::measurement::Measurements;
-use crate::quantity::Time;
 
 /// Parameter value
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
@@ -47,9 +46,4 @@ impl BenchmarkResult {
     pub fn get_name(&self) -> &str {
         self.name.as_deref().unwrap_or(&self.command)
     }
-
-    /// The average wall clock time
-    pub fn mean_wall_clock_time(&self) -> Time {
-        self.measurements.time_wall_clock_mean()
-    }
 }

src/benchmark/executor.rs

@@ -220,6 +220,7 @@ impl Executor for ShellExecutor<'_> {
                 ensure_non_negative(measurement.time_user - spawning_time.time_user);
             measurement.time_system =
                 ensure_non_negative(measurement.time_system - spawning_time.time_system);
+            measurement.time_cpu = measurement.time_user + measurement.time_system;
         }
 
         Ok(measurement)
@@ -278,6 +279,7 @@ impl Executor for ShellExecutor<'_> {
 
         self.shell_spawning_time = Some(Measurement {
             time_wall_clock: measurements.time_wall_clock_mean(),
+            time_cpu: measurements.time_user_mean() + measurements.time_system_mean(),
             time_user: measurements.time_user_mean(),
             time_system: measurements.time_system_mean(),
             memory_peak_resident: None,
@@ -337,6 +339,7 @@ impl Executor for MockExecutor {
 
         Ok(Measurement {
             time_wall_clock: Self::extract_time(command.get_command_line()),
+            time_cpu: Time::zero(),
             time_user: Time::zero(),
             time_system: Time::zero(),
             memory_peak_resident: Some(Information::zero()),

src/benchmark/measurement.rs

@@ -2,7 +2,8 @@ use std::process::ExitStatus;
 
 use serde::Serialize;
 
-use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation};
+use crate::metric::Metric;
+use crate::quantity::statistics::{mean, modified_zscores_f64};
 use crate::quantity::{serialize_information, serialize_time, Information, Time};
 use crate::util::exit_code::extract_exit_code;
 
@@ -77,6 +78,10 @@ pub struct Measurement {
     #[serde(serialize_with = "serialize_time")]
     pub time_wall_clock: Time,
 
+    /// Total CPU time (user and kernel mode)
+    #[serde(serialize_with = "serialize_time")]
+    pub time_cpu: Time,
+
     /// Time spent in user mode
     #[serde(serialize_with = "serialize_time")]
     pub time_user: Time,
@@ -131,35 +136,14 @@ impl Measurements {
         mean(self.wall_clock_times())
     }
 
-    /// The standard deviation of all wall clock times. Not available if only one run has been performed
-    pub fn stddev(&self) -> Option<Time> {
-        let times: Vec<_> = self.wall_clock_times().collect(); // TODO: Avoid collecting
-
-        if times.len() < 2 {
-            None
-        } else {
-            Some(standard_deviation(times))
-        }
-    }
-
-    /// The median wall clock time
-    pub fn median(&self) -> Time {
-        median(self.wall_clock_times())
-    }
-
-    /// The minimum wall clock time
-    pub fn min(&self) -> Time {
-        min(self.wall_clock_times())
-    }
-
-    /// The maximum wall clock time
-    pub fn max(&self) -> Time {
-        max(self.wall_clock_times())
-    }
-
-    /// Compute modified Z-scores for the wall clock times
-    pub fn modified_zscores(&self) -> Vec<f64> {
-        modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
+    /// Compute modified Z-scores for a metric validated to be available in every run.
+    pub fn modified_zscores(&self, metric: Metric) -> Vec<f64> {
+        let values: Vec<_> = self
+            .measurements
+            .iter()
+            .map(|m| metric.value(m).expect("Validated metric is available"))
+            .collect();
+        modified_zscores_f64(&values)
     }
 
     /// The average user time
@@ -172,3 +156,28 @@ impl Measurements {
         mean(self.measurements.iter().map(|m| m.time_system))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::outlier_detection::OUTLIER_THRESHOLD;
+    use crate::quantity::{byte, second};
+
+    #[test]
+    fn outliers_follow_the_selected_metric() {
+        let measurements = Measurements::new(
+            [(1.0, 100.0), (1.0, 1.0), (1.0, 1.0), (100.0, 1.0)]
+                .iter()
+                .copied()
+                .map(|(time, memory)| Measurement {
+                    time_wall_clock: Time::new::<second>(time),
+                    memory_peak_resident: Some(Information::new::<byte>(memory)),
+                    ..Measurement::default()
+                })
+                .collect(),
+        );
+
+        assert!(measurements.modified_zscores(Metric::TimeWallClock)[0] < OUTLIER_THRESHOLD);
+        assert!(measurements.modified_zscores(Metric::MemoryPeakResident)[0] > OUTLIER_THRESHOLD);
+    }
+}

src/benchmark/mod.rs

@@ -1,7 +1,6 @@
 pub mod benchmark_result;
 pub mod executor;
 pub mod measurement;
-pub mod relative_speed;
 pub mod scheduler;
 
 use std::cmp;
@@ -18,14 +17,15 @@ use crate::options::{
 use crate::outlier_detection::OUTLIER_THRESHOLD;
 use crate::output::console_writeln;
 use crate::output::progress_bar::{
-    finish_initial_measurement, get_progress_bar, start_initial_measurement,
+    finish_initial_measurement, get_progress_bar, set_benchmark_header, start_initial_measurement,
 };
+use crate::output::report;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
-use crate::quantity::{self, const_time_from_seconds, FormatQuantity, Time, Zero};
+use crate::quantity::{self, const_time_from_seconds, Time, Zero};
 use benchmark_result::BenchmarkResult;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, ensure, Result};
 use colored::*;
 
 use self::executor::Executor;
@@ -155,20 +155,21 @@ impl<'a> Benchmark<'a> {
         self.run_intermediate_command(command, error_output, output_policy, iteration)
     }
 
-    /// Run the benchmark for a single command
-    pub fn run(&self) -> Result<BenchmarkResult> {
-        if self.options.output_style != OutputStyleOption::Disabled {
-            console_writeln!(
-                io::stdout(),
-                "{}{}: {}",
-                "Benchmark ".bold(),
-                (self.number + 1).to_string().bold(),
-                self.command.get_name_with_unused_parameters(),
-            )?;
+    fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {
+        if self.options.skip_unavailable_metrics {
+            return Ok(());
         }
+        for selection in &self.options.metrics {
+            ensure!(selection.metric.value(measurement).is_some(),
+                "Metric '{}' is unavailable for '{}'. Check platform support and hardware-counter permissions.",
+                selection.metric.name(), self.command.get_command_line());
+        }
+        Ok(())
+    }
 
+    /// Run the benchmark for a single command
+    pub fn run(&self, reference: Option<&BenchmarkResult>) -> Result<BenchmarkResult> {
         let mut measurements = Measurements::default();
-        let mut all_succeeded = true;
 
         let output_policy = &self.options.command_output_policies[self.number];
 
@@ -213,6 +214,15 @@ impl<'a> Benchmark<'a> {
 
         self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
+        let progress_header = format!(
+            "{}: {}",
+            format!("Benchmark {}", self.number + 1).bold(),
+            self.command
+                .get_name_with_unused_parameters()
+                .white()
+                .bold()
+        );
+
         // Warmup phase
         if self.options.warmup_count > 0 {
             let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
@@ -225,6 +235,10 @@ impl<'a> Benchmark<'a> {
                 None
             };
 
+            if let Some(bar) = &progress_bar {
+                set_benchmark_header(bar, progress_header.clone());
+            }
+
             for i in 0..self.options.warmup_count {
                 let warmup_iteration = BenchmarkIteration::Warmup(i);
                 let _ = run_preparation_command(warmup_iteration)?;
@@ -259,6 +273,10 @@ impl<'a> Benchmark<'a> {
             None
         };
 
+        if let Some(bar) = &progress_bar {
+            set_benchmark_header(bar, progress_header);
+        }
+
         let benchmark_iteration = BenchmarkIteration::Benchmark(0);
         let preparation_result = run_preparation_command(benchmark_iteration)?;
         let preparation_overhead = preparation_result.map_or(Time::zero(), |res| {
@@ -275,18 +293,14 @@ impl<'a> Benchmark<'a> {
             None,
             output_policy,
         )?;
-        let success = measurement.exit_status.success();
+        self.validate_measurement(&measurement)?;
+        let mut all_succeeded = measurement.exit_status.success();
 
         if let Some(bar) = progress_bar.as_ref() {
-            let time_unit = self
-                .options
-                .time_unit
-                .unwrap_or(measurement.time_wall_clock.suitable_unit());
-            let estimate = measurement.time_wall_clock.format(time_unit);
-            finish_initial_measurement(
-                bar,
-                format!("Current estimate: {}", estimate.to_string().green()),
-            );
+            let primary = self.options.metrics[0];
+            let value = primary.metric.value(&measurement).unwrap();
+            let estimate = primary.display_unit(value).format(value);
+            finish_initial_measurement(bar, format!("Current estimate: {}", estimate.green()));
         }
 
         let conclusion_result = run_conclusion_command(benchmark_iteration)?;
@@ -316,10 +330,10 @@ impl<'a> Benchmark<'a> {
         let count_remaining = count - 1;
 
         // Save the first result
+        let primary = self.options.metrics[0];
+        let mut primary_total = primary.metric.value(&measurement).unwrap();
         measurements.push(measurement);
 
-        all_succeeded = all_succeeded && success;
-
         // Re-configure the progress bar
         if let Some(bar) = progress_bar.as_ref() {
             bar.set_length(count);
@@ -330,18 +344,12 @@ impl<'a> Benchmark<'a> {
         for i in 0..count_remaining {
             let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
 
-            let msg = {
-                let t_wall_clock_mean = measurements.time_wall_clock_mean();
-                let time_unit = self
-                    .options
-                    .time_unit
-                    .unwrap_or(t_wall_clock_mean.suitable_unit());
-                let mean = t_wall_clock_mean.format(time_unit);
-                format!("Current estimate: {}", mean.to_string().green())
-            };
-
             if let Some(bar) = progress_bar.as_ref() {
-                bar.set_message(msg.to_owned())
+                let mean = primary_total / measurements.len() as f64;
+                bar.set_message(format!(
+                    "Current estimate: {}",
+                    primary.display_unit(mean).format(mean).green()
+                ));
             }
 
             run_preparation_command(benchmark_iteration)?;
@@ -352,7 +360,9 @@ impl<'a> Benchmark<'a> {
                 None,
                 output_policy,
             )?;
+            self.validate_measurement(&measurement)?;
             let success = measurement.exit_status.success();
+            primary_total += primary.metric.value(&measurement).unwrap();
             measurements.push(measurement);
 
             all_succeeded = all_succeeded && success;
@@ -368,63 +378,27 @@ impl<'a> Benchmark<'a> {
             bar.finish_and_clear()
         }
 
-        // Formatting and console output
-        let t_wall_clock_mean = measurements.time_wall_clock_mean();
-        let time_unit = self
-            .options
-            .time_unit
-            .unwrap_or(t_wall_clock_mean.suitable_unit());
-        let mean_str = t_wall_clock_mean.format(time_unit);
-        let min_str = measurements.min().format(time_unit);
-        let max_str = measurements.max().format(time_unit);
-        let num_str = format!("{num_runs} runs", num_runs = measurements.len());
-
-        let user_str = measurements.time_user_mean().format(time_unit);
-        let system_str = measurements.time_system_mean().format(time_unit);
-
         if self.options.output_style != OutputStyleOption::Disabled {
-            let mut stdout = io::stdout().lock();
-            if measurements.len() == 1 {
-                console_writeln!(
-                    stdout,
-                    "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
-                    "abs".green().bold(),
-                    mean_str.green().bold(),
-                    "        ", // alignment
-                    user_str.blue(),
-                    system_str.blue()
-                )?;
-            } else {
-                let stddev_str = measurements.stddev().unwrap().format(time_unit);
-
-                console_writeln!(
-                    stdout,
-                    "  Time ({} ± {}):     {:>8} ± {:>8}    [User: {}, System: {}]",
-                    "mean".green().bold(),
-                    "σ".green(),
-                    mean_str.green().bold(),
-                    stddev_str.green(),
-                    user_str.blue(),
-                    system_str.blue()
-                )?;
-
-                console_writeln!(
-                    stdout,
-                    "  Range ({} … {}):   {:>8} … {:>8}    {}",
-                    "min".cyan(),
-                    "max".purple(),
-                    min_str.cyan(),
-                    max_str.purple(),
-                    num_str.dimmed()
-                )?;
-            }
+            report::print(
+                self.number,
+                &self.command.get_name_with_unused_parameters(),
+                &measurements,
+                &self.options.metrics,
+                self.options.skip_unavailable_metrics,
+                reference,
+            )?;
         }
 
         // Warnings
         let mut warnings = vec![];
 
         // Check execution time
-        if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
+        if self
+            .options
+            .metrics
+            .iter()
+            .any(|m| m.metric == crate::metric::Metric::TimeWallClock)
+            && matches!(self.options.executor_kind, ExecutorKind::Shell(_))
             && measurements
                 .wall_clock_times()
                 .any(|t| t < MIN_EXECUTION_TIME)
@@ -438,7 +412,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = measurements.modified_zscores();
+        let scores = measurements.modified_zscores(primary.metric);
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -452,12 +426,11 @@ impl<'a> Benchmark<'a> {
         };
 
         if scores[0] > OUTLIER_THRESHOLD {
-            warnings.push(Warnings::SlowInitialRun(
-                measurements.wall_clock_times().next().unwrap(),
+            warnings.push(Warnings::InitialRunOutlier(
+                primary,
+                primary.metric.value(&measurements.measurements[0]).unwrap(),
                 outlier_warning_options,
             ));
-        } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
-            warnings.push(Warnings::OutliersDetected(outlier_warning_options));
         }
 
         if !warnings.is_empty() {

src/benchmark/relative_speed.rs

@@ -1,237 +0,0 @@
-use std::cmp::Ordering;
-
-use super::benchmark_result::BenchmarkResult;
-use crate::{
-    options::SortOrder,
-    quantity::{self, Ratio, Time, Zero},
-};
-
-#[derive(Debug)]
-pub struct BenchmarkResultWithRelativeSpeed<'a> {
-    pub result: &'a BenchmarkResult,
-    pub relative_speed: f64,
-    pub relative_speed_stddev: Option<f64>,
-    pub is_reference: bool,
-    // Less means faster
-    pub relative_ordering: Ordering,
-}
-
-impl BenchmarkResultWithRelativeSpeed<'_> {
-    pub fn reference_label(&self) -> &'static str {
-        if self.is_reference {
-            " (reference)"
-        } else {
-            match self.relative_ordering {
-                Ordering::Less => " (faster)",
-                Ordering::Equal => " (same speed)",
-                Ordering::Greater => " (slower)",
-            }
-        }
-    }
-}
-
-pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
-    l.mean_wall_clock_time()
-        .partial_cmp(&r.mean_wall_clock_time())
-        .unwrap_or(Ordering::Equal)
-}
-
-pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
-    results
-        .iter()
-        .min_by(|&l, &r| compare_mean_time(l, r))
-        .expect("at least one benchmark result")
-}
-
-fn compute_relative_speeds<'a>(
-    results: &'a [BenchmarkResult],
-    reference: &'a BenchmarkResult,
-    sort_order: SortOrder,
-) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    let mut results: Vec<_> = results
-        .iter()
-        .map(|result| {
-            // Separate benchmarks can have identical names and measurements.
-            let is_reference = std::ptr::eq(result, reference);
-            let relative_ordering = compare_mean_time(result, reference);
-
-            if result.mean_wall_clock_time() == Time::zero()
-                || reference.mean_wall_clock_time() == Time::zero()
-            {
-                return BenchmarkResultWithRelativeSpeed {
-                    result,
-                    relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
-                    relative_speed_stddev: None,
-                    is_reference,
-                    relative_ordering,
-                };
-            }
-
-            let ratio = match relative_ordering {
-                Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
-                Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
-                Ordering::Greater => {
-                    result.mean_wall_clock_time() / reference.mean_wall_clock_time()
-                }
-            };
-
-            // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
-            // Covariance assumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (
-                result.measurements.stddev(),
-                reference.measurements.stddev(),
-            ) {
-                (Some(result_stddev), Some(reference_stddev)) => Some(
-                    ratio
-                        * ((result_stddev / result.mean_wall_clock_time())
-                            .powi(uom::typenum::P2::new())
-                            + (reference_stddev / reference.mean_wall_clock_time())
-                                .powi(uom::typenum::P2::new()))
-                        .sqrt(),
-                ),
-                _ => None,
-            };
-
-            BenchmarkResultWithRelativeSpeed {
-                result,
-                relative_speed: ratio.get::<quantity::ratio>(),
-                relative_speed_stddev: ratio_stddev.map(|r| r.get::<quantity::ratio>()),
-                is_reference,
-                relative_ordering,
-            }
-        })
-        .collect();
-
-    match sort_order {
-        SortOrder::Command => {}
-        SortOrder::MeanTime => {
-            results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));
-        }
-    }
-
-    results
-}
-
-pub fn compute_with_check_from_reference<'a>(
-    results: &'a [BenchmarkResult],
-    reference: &'a BenchmarkResult,
-    sort_order: SortOrder,
-) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
-    if fastest_of(results).mean_wall_clock_time() == Time::zero()
-        || reference.mean_wall_clock_time() == Time::zero()
-    {
-        return None;
-    }
-
-    Some(compute_relative_speeds(results, reference, sort_order))
-}
-
-pub fn compute_with_check(
-    results: &[BenchmarkResult],
-    sort_order: SortOrder,
-) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
-    let fastest = fastest_of(results);
-
-    if fastest.mean_wall_clock_time() == Time::zero() {
-        return None;
-    }
-
-    Some(compute_relative_speeds(results, fastest, sort_order))
-}
-
-/// Compute relative speeds against the given reference, or the fastest result.
-pub fn compute<'a>(
-    results: &'a [BenchmarkResult],
-    sort_order: SortOrder,
-    reference: Option<&'a BenchmarkResult>,
-) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    let reference = reference.unwrap_or_else(|| fastest_of(results));
-    compute_relative_speeds(results, reference, sort_order)
-}
-
-#[cfg(test)]
-fn create_result(name: &str, mean: f64) -> BenchmarkResult {
-    use std::collections::BTreeMap;
-
-    use crate::benchmark::measurement::{Measurement, Measurements};
-    use crate::quantity::second;
-
-    BenchmarkResult {
-        command: name.into(),
-        display_name: name.into(),
-        name: None,
-        measurements: Measurements {
-            measurements: vec![Measurement {
-                time_wall_clock: Time::new::<second>(mean),
-                time_user: Time::new::<second>(mean),
-                ..Default::default()
-            }],
-        },
-        parameters: BTreeMap::new(),
-    }
-}
-
-#[test]
-fn test_compute_relative_speed() {
-    use approx::assert_relative_eq;
-
-    let results = vec![
-        create_result("cmd1", 3.0),
-        create_result("cmd2", 2.0),
-        create_result("cmd3", 5.0),
-    ];
-
-    let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();
-
-    assert_relative_eq!(1.5, annotated_results[0].relative_speed);
-    assert_relative_eq!(1.0, annotated_results[1].relative_speed);
-    assert_relative_eq!(2.5, annotated_results[2].relative_speed);
-}
-
-#[test]
-fn test_compute_relative_speed_with_reference() {
-    use approx::assert_relative_eq;
-
-    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
-    let reference = create_result("cmd2", 4.0);
-
-    let annotated_results =
-        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
-
-    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
-    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
-}
-
-#[test]
-fn test_compute_relative_speed_for_zero_times() {
-    let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
-
-    let annotated_results = compute_with_check(&results, SortOrder::Command);
-
-    assert!(annotated_results.is_none());
-}
-
-#[test]
-fn reference_identity_distinguishes_equal_results() {
-    let results = vec![create_result("same", 1.0), create_result("same", 1.0)];
-    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
-
-    assert!(entries[0].is_reference);
-    assert!(!entries[1].is_reference);
-}
-
-#[test]
-fn reference_ratios_handle_zero_times() {
-    let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
-    for result in &mut results {
-        let measurement = result.measurements.measurements[0].clone();
-        result.measurements.measurements.push(measurement);
-    }
-    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
-    assert_eq!(entries[1].relative_speed, f64::INFINITY);
-
-    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
-    assert_eq!(entries[0].relative_speed, f64::INFINITY);
-    assert_eq!(entries[0].relative_speed_stddev, None);
-    assert_eq!(entries[1].relative_speed, 1.0);
-}

src/benchmark/scheduler.rs

@@ -1,14 +1,10 @@
 use super::benchmark_result::BenchmarkResult;
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
-use super::{relative_speed, Benchmark};
-use colored::*;
-use std::cmp::Ordering;
-use std::io::{self, Write};
+use super::Benchmark;
 
-use crate::command::{Command, Commands};
+use crate::command::Commands;
 use crate::export::ExportManager;
-use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
-use crate::output::console_writeln;
+use crate::options::{ExecutorKind, Options};
 
 use anyhow::Result;
 
@@ -40,133 +36,23 @@ impl<'a> Scheduler<'a> {
             ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
         };
 
-        let reference = self
-            .options
-            .reference_command
-            .as_ref()
-            .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
-
         executor.calibrate()?;
 
-        for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
-            self.results
-                .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
+        for (number, cmd) in self.commands.iter().enumerate() {
+            self.results.push(
+                Benchmark::new(number, cmd, self.options, &*executor).run(self.results.first())?,
+            );
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.
-            self.export_manager
-                .write_results(&self.results, true, self.options.reference_index)?;
-        }
-
-        Ok(())
-    }
-
-    pub fn print_relative_speed_comparison(&self) -> Result<()> {
-        if self.options.output_style == OutputStyleOption::Disabled {
-            return Ok(());
-        }
-
-        if self.results.len() < 2 {
-            return Ok(());
-        }
-
-        let reference = self
-            .options
-            .reference_index
-            .map(|index| &self.results[index])
-            .unwrap_or_else(|| relative_speed::fastest_of(&self.results));
-
-        if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
-            &self.results,
-            reference,
-            self.options.sort_order_speed_comparison,
-        ) {
-            let mut stdout = io::stdout().lock();
-            match self.options.sort_order_speed_comparison {
-                SortOrder::MeanTime => {
-                    console_writeln!(stdout, "{}", "Summary".bold())?;
-
-                    let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
-                    let others = annotated_results.iter().filter(|r| !r.is_reference);
-
-                    console_writeln!(stdout, "  {} ran", reference.result.display_name.cyan())?;
-
-                    for item in others {
-                        let stddev = if let Some(stddev) = item.relative_speed_stddev {
-                            format!(" ± {}", format!("{:.2}", stddev).green())
-                        } else {
-                            "".into()
-                        };
-                        let comparator = match item.relative_ordering {
-                            Ordering::Less => format!(
-                                "{}{} times slower than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                            Ordering::Greater => format!(
-                                "{}{} times faster than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                            Ordering::Equal => format!(
-                                "    As fast ({}{}) as",
-                                format!("{:.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                        };
-                        console_writeln!(
-                            stdout,
-                            "{} {}",
-                            comparator,
-                            &item.result.display_name.magenta()
-                        )?;
-                    }
-                }
-                SortOrder::Command => {
-                    console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
-
-                    for item in annotated_results {
-                        let relationship = if self.options.reference_index.is_some() {
-                            item.reference_label()
-                        } else {
-                            ""
-                        };
-                        console_writeln!(
-                            stdout,
-                            "  {}{}  {}{}",
-                            format!("{:10.2}", item.relative_speed).bold().green(),
-                            if item.is_reference {
-                                "        ".into()
-                            } else if let Some(stddev) = item.relative_speed_stddev {
-                                format!(" ± {}", format!("{stddev:5.2}").green())
-                            } else {
-                                "        ".into()
-                            },
-                            item.result.display_name,
-                            relationship,
-                        )?;
-                    }
-                }
-            }
-        } else {
-            console_writeln!(
-                io::stderr(),
-                "{}: The benchmark comparison could not be computed as some benchmark times are zero. \
-                 This could be caused by background interference during the initial calibration phase \
-                 of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
-                 Try to re-run the benchmark on a quiet system. If you did not do so already, try the \
-                 --shell=none/-N option. If it does not help either, you command is most likely too fast \
-                 to be accurately benchmarked by hyperfine.",
-                 "Note".bold().red()
-            )?;
+            self.export_manager.write_results(&self.results, true)?;
         }
 
         Ok(())
     }
 
     pub fn final_export(&self) -> Result<()> {
-        self.export_manager
-            .write_results(&self.results, false, self.options.reference_index)
+        self.export_manager.write_results(&self.results, false)
     }
 }
 
@@ -183,11 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
     assert_eq!(options.executor_kind, ExecutorKind::Mock(None));
 
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
-    let export_manager = ExportManager::from_cli_arguments(
-        &cli_arguments,
-        options.time_unit,
-        options.sort_order_exports,
-    )?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
 
     options.validate_against_command_list(&commands)?;
 
@@ -199,12 +81,15 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
 
 #[test]
 fn scheduler_basic() -> Result<()> {
-    insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#"
+    insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @"
     - command: sleep 0.123
       measurements:
         - time_wall_clock:
             value: 0.123
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -218,6 +103,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.123
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -233,6 +121,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.456
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -246,6 +137,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.456
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -256,7 +150,7 @@ fn scheduler_basic() -> Result<()> {
             value: 0
             unit: byte
           exit_code: 0
-    "#);
+    ");
 
     Ok(())
 }

src/cli.rs

@@ -7,6 +7,12 @@ use clap::{
     crate_version, Arg, ArgAction, ArgMatches, Command, ValueHint,
 };
 
+pub const DEFAULT_METRICS: &str = if cfg!(windows) {
+    "time_wall_clock"
+} else {
+    "time_wall_clock,memory_peak_resident"
+};
+
 pub fn get_cli_arguments<'a, I, T>(args: I) -> ArgMatches
 where
     I: IntoIterator<Item = T>,
@@ -38,12 +44,40 @@ fn build_command() -> Command {
                        line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \
                        The latter requires a shell, which can be enabled via '-S' or '--shell=...'. \
                        If multiple commands are given, hyperfine will show a \
-                       comparison of the respective runtimes.")
+                       comparison with the first command as the reference.")
                 .required(true)
                 .action(ArgAction::Append)
                 .value_hint(ValueHint::CommandString)
                 .value_parser(NonEmptyStringValueParser::new()),
         )
+        .arg(
+            Arg::new("metrics")
+                .long("metrics")
+                .value_name("PRESET|METRIC[:UNIT],…")
+                .default_value("default")
+                .hide_default_value(true)
+                .help("Performance metrics to measure, in order.\n\n\
+                       This is either a preset name, or a comma-separated list of metrics with optional units. \
+                       The first metric is used for non-JSON exports, \
+                       JSON always includes all collected metrics.\n\n\
+                       Presets (use one on its own):\n  \
+                         default  Wall-clock time and peak RSS\n  \
+                         time     Wall-clock, CPU, user, and system time\n  \
+                         all      All available metrics\n\n\
+                       Presets use automatic units.\n\n\
+                       Metrics and optional units:\n  \
+                         Time: time_wall_clock, time_cpu, time_user, time_system\n    \
+                           Units: ns, us, ms, s, min, h\n  \
+                         Memory: memory_peak_resident\n    \
+                           Units: B, kB, MB, GB, TB, KiB, MiB, GiB, TiB\n  \
+                         Counters: cpu_cycles, instructions, cache_references,\n    \
+                           cache_misses, branch_misses\n    \
+                           Units: count, k (thousand), M (million), B (billion)\n\n\
+                       Examples:\n  \
+                       --metrics time_cpu,instructions\n  \
+                       --metrics memory_peak_resident:MiB,time_wall_clock:ms\n  \
+                       --metrics time"),
+        )
         .arg(
             Arg::new("warmup")
                 .long("warmup")
@@ -93,34 +127,12 @@ fn build_command() -> Command {
                 .value_name("CMD")
                 .value_hint(ValueHint::CommandString)
                 .help(
-                    "Execute CMD before each set of timing runs. This is useful for \
+                    "Execute CMD before each set of benchmark runs. This is useful for \
                      compiling your software with the provided parameters, or to do any \
                      other work that should happen once before a series of benchmark runs, \
                      not every time as would happen with the --prepare option."
                 ),
         )
-        .arg(
-            Arg::new("reference")
-                .long("reference")
-                .action(ArgAction::Set)
-                .value_name("CMD")
-                .help(
-                    "The reference for the relative comparison of results. Without parameters, \
-                    CMD is run as a separate reference command. With --parameter-scan or \
-                    --parameter-list, CMD must exactly match one unique benchmark name as \
-                    shown in the output. If this is unset, results are compared with the \
-                    fastest command as reference."
-                )
-        )
-        .arg(
-            Arg::new("reference-name")
-                .long("reference-name")
-                .action(ArgAction::Set)
-                .value_name("CMD")
-                .help("Give a meaningful name to the reference command. This cannot be used \
-                       with parameterized benchmarks; use --command-name instead.")
-                .requires("reference")
-        )
         .arg(
             Arg::new("prepare")
                 .long("prepare")
@@ -130,7 +142,7 @@ fn build_command() -> Command {
                 .value_name("CMD")
                 .value_hint(ValueHint::CommandString)
                 .help(
-                    "Execute CMD before each timing run. This is useful for \
+                    "Execute CMD before each benchmark run. This is useful for \
                      clearing disk caches, for example.\nThe --prepare option can \
                      be specified once for all commands or multiple times, once for \
                      each command. In the latter case, each preparation command will \
@@ -146,7 +158,7 @@ fn build_command() -> Command {
                 .value_name("CMD")
                 .value_hint(ValueHint::CommandString)
                 .help(
-                    "Execute CMD after each timing run. This is useful for killing \
+                    "Execute CMD after each benchmark run. This is useful for killing \
                      long-running processes started (e.g. a web server started in --prepare), \
                      for example.\nThe --conclude option can be specified once for all \
                      commands or multiple times, once for each command. In the latter case, \
@@ -275,47 +287,14 @@ fn build_command() -> Command {
                      the output of the tool.",
                 ),
         )
-        .arg(
-            Arg::new("sort")
-            .long("sort")
-            .action(ArgAction::Set)
-            .value_name("METHOD")
-            .value_parser(["auto", "command", "mean-time"])
-            .default_value("auto")
-            .hide_default_value(true)
-            .help(
-                "Specify the sort order of the speed comparison summary and the exported tables for \
-                 markup formats (Markdown, AsciiDoc, org-mode):\n  \
-                   * 'auto' (default): the speed comparison will be ordered by time and\n    \
-                     the markup tables will be ordered by command (input order).\n  \
-                   * 'command': order benchmarks in the way they were specified\n  \
-                   * 'mean-time': order benchmarks by mean runtime\n"
-            ),
-        )
-        .arg(
-            Arg::new("time-unit")
-                .long("time-unit")
-                .short('u')
-                .action(ArgAction::Set)
-                .value_name("UNIT")
-                .value_parser(["µs", "us", "microsecond", "microseconds", "ms", "millisecond", "milliseconds", "s", "second", "seconds", "min", "minute", "minutes", "h", "hour", "hours"])
-                .help("Set the time unit to be used. If the option is not given, the time unit is determined automatically. \
-                       This option affects the standard output as well as all export formats except for CSV and JSON.\n\
-                       Possible values:\n  \
-                         * 'µs', 'us', 'microsecond', 'microseconds'\n  \
-                         * 'ms', 'millisecond', 'milliseconds'\n  \
-                         * 's', 'second', 'seconds'\n  \
-                         * 'min', 'minute', 'minutes'\n  \
-                         * 'h', 'hour', 'hours'"),
-        )
         .arg(
             Arg::new("export-asciidoc")
                 .long("export-asciidoc")
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the timing summary statistics as an AsciiDoc table to the given FILE. \
-                       The output time unit can be changed using the --time-unit option."),
+                .help("Export the primary metric summary statistics as an AsciiDoc table to the given FILE. \
+                       The unit can be set with --metrics METRIC:UNIT."),
         )
         .arg(
             Arg::new("export-csv")
@@ -323,9 +302,9 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the timing summary statistics as CSV to the given FILE. If you need \
-                       the timing results for each individual run, use the JSON export format. \
-                       The output time unit is always seconds."),
+                .help("Export the primary metric summary statistics as CSV to the given FILE. If you need \
+                       all metrics for each individual run, use the JSON export format. \
+                       Use the explicitly selected unit, or the base unit when unspecified."),
         )
         .arg(
             Arg::new("export-json")
@@ -333,8 +312,8 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the timing summary statistics and timings of individual runs as JSON to the given FILE. \
-                       The output time unit is always seconds"),
+                .help("Export all collected metrics and individual runs as JSON to the given FILE. \
+                       Values always use base units (seconds, bytes, counts)."),
         )
         .arg(
             Arg::new("export-markdown")
@@ -342,8 +321,8 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the timing summary statistics as a Markdown table to the given FILE. \
-                       The output time unit can be changed using the --time-unit option."),
+                .help("Export the primary metric summary statistics as a Markdown table to the given FILE. \
+                       The unit can be set with --metrics METRIC:UNIT."),
         )
         .arg(
             Arg::new("export-orgmode")
@@ -351,8 +330,8 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the timing summary statistics as an Emacs org-mode table to the given FILE. \
-                       The output time unit can be changed using the --time-unit option."),
+                .help("Export the primary metric summary statistics as an Emacs org-mode table to the given FILE. \
+                       The unit can be set with --metrics METRIC:UNIT."),
         )
         .arg(
             Arg::new("show-output")

src/command.rs

@@ -257,8 +257,8 @@ impl<'a> Commands<'a> {
         self.0.iter()
     }
 
-    pub fn num_commands(&self, has_reference_command: bool) -> usize {
-        self.0.len() + if has_reference_command { 1 } else { 0 }
+    pub fn num_commands(&self) -> usize {
+        self.0.len()
     }
 
     /// Finds all the strings that appear multiple times in the input iterator, returning them in

src/error.rs

@@ -41,6 +41,8 @@ impl From<DecimalError> for ParameterScanError {
 
 #[derive(Debug, Error)]
 pub enum OptionsError<'a> {
+    #[error("{0}")]
+    InvalidMetrics(String),
     #[error(
         "Conflicting requirements for the number of runs (empty range, min is larger than max)"
     )]

src/export/asciidoc.rs

@@ -24,7 +24,16 @@ impl MarkupExporter for AsciidocExporter {
     }
 
     fn table_row(&self, cells: &[&str]) -> String {
-        format!("\n| {} \n", cells.join(" \n| "))
+        let mut row = String::from("\n");
+        for cell in cells {
+            row.push('|');
+            if !cell.is_empty() {
+                row.push(' ');
+                row.push_str(cell);
+            }
+            row.push('\n');
+        }
+        row
     }
 
     fn table_divider(&self, _cell_aligmnents: &[Alignment]) -> String {
@@ -43,7 +52,7 @@ fn test_asciidoc_exporter_table_data() {
     let data = vec!["a", "b", "c"];
 
     let actual = exporter.table_row(&data);
-    let expect = "\n| a \n| b \n| c \n";
+    let expect = "\n| a\n| b\n| c\n";
 
     assert_eq!(expect, actual);
 }

src/export/csv.rs

@@ -1,65 +1,73 @@
-use std::borrow::Cow;
-
 use csv::WriterBuilder;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
-use crate::options::SortOrder;
-use crate::quantity::{FormatQuantity, TimeUnit};
+use crate::metric::MetricSelection;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 
 #[derive(Default)]
 pub struct CsvExporter {}
 
 impl Exporter for CsvExporter {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        _time_unit: Option<TimeUnit>,
-        _sort_order: SortOrder,
-        _reference_index: Option<usize>,
-    ) -> Result<Vec<u8>> {
-        const CSV_UNIT: TimeUnit = TimeUnit::Second;
-        const CSV_PRECISION: usize = 6;
-
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
+        // CSV never chooses units automatically: an omitted suffix means base units.
+        let unit = primary.csv_unit();
         let mut writer = WriterBuilder::new().from_writer(vec![]);
-
-        {
-            let mut headers: Vec<Cow<[u8]>> = [
-                // The list of times and exit codes cannot be exported to the CSV file - omit them.
-                "command", "mean", "stddev", "median", "user", "system", "min", "max",
-            ]
-            .iter()
-            .map(|x| Cow::Borrowed(x.as_bytes()))
-            .collect();
-            if let Some(res) = results.first() {
-                for param_name in res.parameters.keys() {
-                    headers.push(Cow::Owned(format!("parameter_{param_name}").into_bytes()));
-                }
-            }
-            writer.write_record(headers)?;
+        let mut headers: Vec<String> = [
+            "command", "metric", "unit", "mean", "stddev", "median", "min", "max",
+        ]
+        .iter()
+        .copied()
+        .map(str::to_owned)
+        .collect();
+        if let Some(result) = results.first() {
+            headers.extend(
+                result
+                    .parameters
+                    .keys()
+                    .map(|name| format!("parameter_{name}")),
+            );
         }
+        writer.write_record(headers)?;
 
-        for res in results {
-            let mut fields = vec![Cow::Borrowed(res.get_name().as_bytes())];
-            for f in &[
-                res.mean_wall_clock_time(),
-                res.measurements.stddev().unwrap_or_default(),
-                res.measurements.median(),
-                res.measurements.time_user_mean(),
-                res.measurements.time_system_mean(),
-                res.measurements.min(),
-                res.measurements.max(),
-            ] {
-                fields.push(Cow::Owned(
-                    f.format_with_precision(CSV_UNIT, CSV_PRECISION)
-                        .into_bytes(),
-                ))
-            }
-            for v in res.parameters.values() {
-                fields.push(Cow::Borrowed(v.value.as_bytes()))
-            }
+        for result in results {
+            let stats = primary
+                .metric
+                .summarize(&result.measurements)
+                .with_context(|| {
+                    format!(
+                        "Metric '{}' is unavailable for '{}'",
+                        primary.metric.name(),
+                        result.get_name()
+                    )
+                })?;
+            let mut fields = vec![
+                result.get_name().to_owned(),
+                primary.metric.name().to_owned(),
+                unit.symbol.to_owned(),
+            ];
+            // Preserve numeric precision rather than using rounded display values.
+            fields.extend(
+                [
+                    Some(stats.mean),
+                    stats.stddev,
+                    Some(stats.median),
+                    Some(stats.min),
+                    Some(stats.max),
+                ]
+                .map(|value| {
+                    value
+                        .map(|value| (value / unit.scale).to_string())
+                        .unwrap_or_default()
+                }),
+            );
+            fields.extend(
+                result
+                    .parameters
+                    .values()
+                    .map(|parameter| parameter.value.clone()),
+            );
             writer.write_record(fields)?;
         }
 
@@ -86,6 +94,7 @@ fn test_csv() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(7.0),
+                    time_cpu: Time::new::<second>(7.0),
                     time_user: Time::new::<second>(7.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -94,6 +103,7 @@ fn test_csv() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(8.0),
+                    time_cpu: Time::new::<second>(8.0),
                     time_user: Time::new::<second>(8.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -102,6 +112,7 @@ fn test_csv() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(12.0),
+                    time_cpu: Time::new::<second>(12.0),
                     time_user: Time::new::<second>(12.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -133,6 +144,7 @@ fn test_csv() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(17.0),
+                    time_cpu: Time::new::<second>(17.0),
                     time_user: Time::new::<second>(17.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -141,6 +153,7 @@ fn test_csv() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(18.0),
+                    time_cpu: Time::new::<second>(18.0),
                     time_user: Time::new::<second>(18.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -149,6 +162,7 @@ fn test_csv() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(19.0),
+                    time_cpu: Time::new::<second>(19.0),
                     time_user: Time::new::<second>(19.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -177,14 +191,28 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None)
+            .serialize(&results, MetricSelection::default())
             .unwrap(),
     )
     .unwrap();
 
     insta::assert_snapshot!(actual, @r#"
-    command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo
-    command_a,9.000000,2.645751,8.000000,9.000000,0.000000,7.000000,12.000000,two,one
-    command_b,18.000000,1.000000,18.000000,18.000000,0.000000,17.000000,19.000000,seven,one
+    command,metric,unit,mean,stddev,median,min,max,parameter_bar,parameter_foo
+    command_a,time_wall_clock,s,9,2.6457513110645907,8,7,12,two,one
+    command_b,time_wall_clock,s,18,1,18,17,19,seven,one
     "#);
+
+    for (selection, unit, mean) in [
+        ("memory_peak_resident", "B", "1024"),
+        ("memory_peak_resident:KiB", "KiB", "1"),
+    ] {
+        let output = exporter
+            .serialize(&results, MetricSelection::parse_list(selection).unwrap()[0])
+            .unwrap();
+        let mut reader = csv::Reader::from_reader(output.as_slice());
+        let row = reader.records().next().unwrap().unwrap();
+        assert_eq!(&row[1], "memory_peak_resident");
+        assert_eq!(&row[2], unit);
+        assert_eq!(&row[3], mean);
+    }
 }

src/export/json.rs

@@ -1,14 +1,11 @@
-use std::ops::{Add, AddAssign, Div};
+use std::collections::BTreeMap;
 
-use serde::*;
+use serde::Serialize;
 use serde_json::to_vec_pretty;
 
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
-use crate::benchmark::measurement::Measurements;
-use crate::options::SortOrder;
-use crate::quantity::statistics::{max, mean, median, min, standard_deviation, UnsafeRawValue};
-use crate::quantity::{byte, ratio, second, QuantityInUnit, Ratio, TimeUnit, Zero};
+use crate::metric::{Metric, MetricSelection, Stats};
 
 use anyhow::Result;
 
@@ -21,6 +18,7 @@ const JSON_SCHEMA_VERSION: u32 = 2;
 struct HyperfineSummary<'a> {
     schema_version: u32,
     metadata: &'a Metadata,
+    primary_metric: &'static str,
     results: Vec<JsonBenchmarkResult<'a>>,
 }
 
@@ -28,98 +26,7 @@ struct HyperfineSummary<'a> {
 struct JsonBenchmarkResult<'a> {
     #[serde(flatten)]
     result: &'a BenchmarkResult,
-    #[serde(skip_serializing_if = "std::ops::Not::not")]
-    is_reference: bool,
-    summary: BenchmarkSummary,
-}
-
-#[derive(Serialize)]
-struct BenchmarkSummary {
-    time_wall_clock: StatisticalSummary,
-    time_user: StatisticalSummary,
-    time_system: StatisticalSummary,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    memory_peak_resident: Option<StatisticalSummary>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    cpu_cycles: Option<StatisticalSummary>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    instructions: Option<StatisticalSummary>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    cache_references: Option<StatisticalSummary>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    cache_misses: Option<StatisticalSummary>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    branch_misses: Option<StatisticalSummary>,
-}
-
-impl BenchmarkSummary {
-    fn from_measurements(measurements: &Measurements) -> Self {
-        let values = &measurements.measurements;
-        Self {
-            time_wall_clock: StatisticalSummary::from_values(
-                values.iter().map(|m| Some(m.time_wall_clock)),
-                Some(second),
-            )
-            .expect("Benchmarks have at least one measurement"),
-            time_user: StatisticalSummary::from_values(
-                values.iter().map(|m| Some(m.time_user)),
-                Some(second),
-            )
-            .expect("Benchmarks have at least one measurement"),
-            time_system: StatisticalSummary::from_values(
-                values.iter().map(|m| Some(m.time_system)),
-                Some(second),
-            )
-            .expect("Benchmarks have at least one measurement"),
-            memory_peak_resident: StatisticalSummary::from_values(
-                values
-                    .iter()
-                    .filter_map(|m| m.memory_peak_resident)
-                    .map(Some),
-                Some(byte),
-            ),
-            cpu_cycles: StatisticalSummary::from_values(
-                values.iter().map(|m| {
-                    m.hardware_counters
-                        .cpu_cycles
-                        .map(|v| Ratio::new::<ratio>(v as f64))
-                }),
-                None::<ratio>,
-            ),
-            instructions: StatisticalSummary::from_values(
-                values.iter().map(|m| {
-                    m.hardware_counters
-                        .instructions
-                        .map(|v| Ratio::new::<ratio>(v as f64))
-                }),
-                None::<ratio>,
-            ),
-            cache_references: StatisticalSummary::from_values(
-                values.iter().map(|m| {
-                    m.hardware_counters
-                        .cache_references
-                        .map(|v| Ratio::new::<ratio>(v as f64))
-                }),
-                None::<ratio>,
-            ),
-            cache_misses: StatisticalSummary::from_values(
-                values.iter().map(|m| {
-                    m.hardware_counters
-                        .cache_misses
-                        .map(|v| Ratio::new::<ratio>(v as f64))
-                }),
-                None::<ratio>,
-            ),
-            branch_misses: StatisticalSummary::from_values(
-                values.iter().map(|m| {
-                    m.hardware_counters
-                        .branch_misses
-                        .map(|v| Ratio::new::<ratio>(v as f64))
-                }),
-                None::<ratio>,
-            ),
-        }
-    }
+    summary: BTreeMap<&'static str, StatisticalSummary>,
 }
 
 #[derive(Serialize)]
@@ -135,37 +42,20 @@ struct StatisticalSummary {
 }
 
 impl StatisticalSummary {
-    fn from_values<Q, P, U>(
-        values: impl IntoIterator<Item = Option<Q>>,
-        unit: Option<U>,
-    ) -> Option<Self>
-    where
-        Q: Copy
-            + PartialOrd
-            + Add<Output = Q>
-            + AddAssign
-            + Zero
-            + Div<Ratio, Output = P>
-            + UnsafeRawValue
-            + QuantityInUnit<U>,
-        P: Into<Q>,
-        U: uom::si::Unit,
-    {
-        // Only summarize a metric when it is available for every supplied measurement.
-        let values: Vec<_> = values.into_iter().collect::<Option<_>>()?;
-        if values.is_empty() {
-            return None;
+    fn new(metric: Metric, stats: Stats) -> Self {
+        Self {
+            unit: match metric.base_unit().symbol {
+                "s" => Some("second"),
+                "B" => Some("byte"),
+                _ => None,
+            },
+            count: stats.count,
+            mean: stats.mean,
+            stddev: stats.stddev,
+            median: stats.median,
+            min: stats.min,
+            max: stats.max,
         }
-        Some(Self {
-            unit: unit.map(|_| U::singular()),
-            count: values.len(),
-            mean: mean(values.iter().copied()).value_in_unit(),
-            stddev: (values.len() > 1)
-                .then(|| standard_deviation(values.iter().copied()).value_in_unit()),
-            median: median(values.iter().copied()).value_in_unit(),
-            min: min(values.iter().copied()).value_in_unit(),
-            max: max(values.iter().copied()).value_in_unit(),
-        })
     }
 }
 
@@ -175,63 +65,28 @@ pub struct JsonExporter {
 }
 
 impl Exporter for JsonExporter {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        _time_unit: Option<TimeUnit>,
-        _sort_order: SortOrder,
-        reference_index: Option<usize>,
-    ) -> Result<Vec<u8>> {
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
         let mut output = to_vec_pretty(&HyperfineSummary {
             schema_version: JSON_SCHEMA_VERSION,
             metadata: &self.metadata,
+            primary_metric: primary.metric.name(),
             results: results
                 .iter()
-                .enumerate()
-                .map(|(index, result)| JsonBenchmarkResult {
+                .map(|result| JsonBenchmarkResult {
                     result,
-                    is_reference: reference_index == Some(index),
-                    summary: BenchmarkSummary::from_measurements(&result.measurements),
+                    summary: Metric::ALL
+                        .iter()
+                        .copied()
+                        .filter_map(|metric| {
+                            metric.summarize(&result.measurements).map(|stats| {
+                                (metric.name(), StatisticalSummary::new(metric, stats))
+                            })
+                        })
+                        .collect(),
                 })
                 .collect(),
-        });
-        if let Ok(ref mut content) = output {
-            content.push(b'\n');
-        }
-
-        Ok(output?)
-    }
-}
-
-#[test]
-fn test_statistical_summary_unit_conversion() {
-    use crate::quantity::{kibibyte, millisecond, Information, Time};
-
-    let time = StatisticalSummary::from_values(
-        [Time::new::<second>(1.0), Time::new::<second>(3.0)].map(Some),
-        Some(millisecond),
-    )
-    .unwrap();
-    let information = StatisticalSummary::from_values(
-        [
-            Information::new::<byte>(1024.0),
-            Information::new::<byte>(3072.0),
-        ]
-        .map(Some),
-        Some(kibibyte),
-    )
-    .unwrap();
-
-    for (summary, unit, scale) in [
-        (time, "millisecond", 1000.0),
-        (information, "kibibyte", 1.0),
-    ] {
-        assert_eq!(summary.unit, Some(unit));
-        assert_eq!(summary.count, 2);
-        approx::assert_relative_eq!(summary.mean, 2.0 * scale);
-        approx::assert_relative_eq!(summary.median, 2.0 * scale);
-        approx::assert_relative_eq!(summary.min, scale);
-        approx::assert_relative_eq!(summary.max, 3.0 * scale);
-        approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale);
+        })?;
+        output.push(b'\n');
+        Ok(output)
     }
 }

src/export/markup.rs

@@ -1,11 +1,8 @@
-use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
-use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
-use crate::options::SortOrder;
-use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero};
+use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::metric::{MetricSelection, Stats, Unit};
 
 use super::Exporter;
-use anyhow::Result;
-use std::cmp::Ordering;
+use anyhow::{Context, Result};
 
 pub enum Alignment {
     Left,
@@ -15,132 +12,95 @@ pub enum Alignment {
 pub trait MarkupExporter {
     fn table_results(
         &self,
-        entries: &[BenchmarkResultWithRelativeSpeed],
-        time_unit: TimeUnit,
-        reference_pending: bool,
-        explicit_reference: bool,
+        results: &[BenchmarkResult],
+        summaries: &[Stats],
+        primary: MetricSelection,
+        unit: Unit,
     ) -> String {
-        // prepare table header strings
-        let notation = format!("[{}]", time_unit.short_name());
-
-        // prepare table cells alignment
+        let notation = format!("[{}]", unit.symbol);
         let cells_alignment = [
             Alignment::Left,
             Alignment::Right,
             Alignment::Right,
-            Alignment::Right,
-            Alignment::Right,
+            Alignment::Left,
         ];
-
-        // emit table header format
         let mut table = self.table_header(&cells_alignment);
-
-        // emit table header data
         table.push_str(&self.table_row(&[
             "Command",
-            &format!("Mean {notation}"),
-            &format!("Min {notation}"),
-            &format!("Max {notation}"),
-            "Relative",
+            &format!("Mean {} {notation}", primary.metric.label()),
+            "Change",
+            "Factor",
         ]));
-
-        // emit horizontal line
         table.push_str(&self.table_divider(&cells_alignment));
 
-        for entry in entries {
-            let result = &entry.result;
-            // prepare data row strings
-            let cmd_str = result.display_name.replace('|', "\\|");
-            let mean_str = result.mean_wall_clock_time().format_value(time_unit);
-            let stddev_str = if let Some(stddev) = result.measurements.stddev() {
-                format!(" ± {}", stddev.format_value(time_unit))
+        for (index, (result, stats)) in results.iter().zip(summaries).enumerate() {
+            let command = result.display_name.replace('|', "\\|");
+            let mean = unit.format_value(stats.mean);
+            let stddev = stats
+                .stddev
+                .map(|value| format!(" ± {}", unit.format_value(value)))
+                .unwrap_or_default();
+            let change = if index == 0 {
+                String::new()
             } else {
-                "".into()
+                stats.format_change_from(&summaries[0])
             };
-            let min_str = result.measurements.min().format_value(time_unit);
-            let max_str = result.measurements.max().format_value(time_unit);
-            // The ratio of two zero times is undefined, even if they compare equal.
-            let relative_unavailable = reference_pending
-                || (explicit_reference
-                    && !entry.is_reference
-                    && entry.relative_ordering == Ordering::Equal
-                    && result.mean_wall_clock_time() == Time::zero());
-            let relative = if relative_unavailable {
-                "N/A".to_string()
+            let factor = if index == 0 {
+                String::new()
             } else {
-                let stddev = match (entry.is_reference, entry.relative_speed_stddev) {
-                    (false, Some(stddev)) => format!(" ± {stddev:.2}"),
-                    _ => String::new(),
-                };
-                let label = if explicit_reference {
-                    entry.reference_label()
-                } else {
-                    ""
-                };
-                format!("{:.2}{stddev}{label}", entry.relative_speed)
+                stats.format_factor_from(&summaries[0], primary.metric)
             };
-
-            // prepare table row entries
             table.push_str(&self.table_row(&[
-                &self.command(&cmd_str),
-                &format!("{mean_str}{stddev_str}"),
-                &min_str,
-                &max_str,
-                &relative,
-            ]))
+                &self.command(&command),
+                &format!("{mean}{stddev}"),
+                &change,
+                &factor,
+            ]));
         }
-
-        // emit table footer format
         table.push_str(&self.table_footer(&cells_alignment));
-
         table
     }
 
     fn table_row(&self, cells: &[&str]) -> String;
 
-    fn table_divider(&self, cell_aligmnents: &[Alignment]) -> String;
+    fn table_divider(&self, cell_alignments: &[Alignment]) -> String;
 
-    fn table_header(&self, _cell_aligmnents: &[Alignment]) -> String {
-        "".to_string()
+    fn table_header(&self, _cell_alignments: &[Alignment]) -> String {
+        String::new()
     }
 
-    fn table_footer(&self, _cell_aligmnents: &[Alignment]) -> String {
-        "".to_string()
+    fn table_footer(&self, _cell_alignments: &[Alignment]) -> String {
+        String::new()
     }
 
-    fn command(&self, size: &str) -> String;
-}
-
-fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit {
-    if let Some(first_result) = results.first() {
-        // Use the first BenchmarkResult entry to determine the unit for all entries.
-        first_result.mean_wall_clock_time().suitable_unit()
-    } else {
-        // Default to `Second`.
-        TimeUnit::Second
-    }
+    fn command(&self, command: &str) -> String;
 }
 
 impl<T: MarkupExporter> Exporter for T {
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        time_unit: Option<TimeUnit>,
-        sort_order: SortOrder,
-        reference_index: Option<usize>,
-    ) -> Result<Vec<u8>> {
-        let unit = time_unit.unwrap_or_else(|| determine_unit_from_results(results));
-        // Do not report ratios against another benchmark while the selected
-        // reference is still pending in an intermediate export.
-        let reference_pending = reference_index.is_some_and(|i| i >= results.len());
-        let entries = relative_speed::compute(
-            results,
-            sort_order,
-            reference_index.and_then(|i| results.get(i)),
-        );
-
-        let table =
-            self.table_results(&entries, unit, reference_pending, reference_index.is_some());
-        Ok(table.as_bytes().to_vec())
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
+        let stats: Vec<_> = results
+            .iter()
+            .map(|result| {
+                primary
+                    .metric
+                    .summarize(&result.measurements)
+                    .with_context(|| {
+                        format!(
+                            "Metric '{}' is unavailable for '{}'",
+                            primary.metric.name(),
+                            result.get_name()
+                        )
+                    })
+            })
+            .collect::<Result<_>>()?;
+        // Use one common unit, avoiding an unscaled column if the reference is zero.
+        let mean = stats
+            .iter()
+            .find(|stats| stats.mean != 0.0)
+            .map_or(0.0, |stats| stats.mean);
+        let unit = primary.display_unit(mean);
+        Ok(self
+            .table_results(results, &stats, primary, unit)
+            .into_bytes())
     }
 }

src/export/mod.rs

@@ -17,9 +17,8 @@ use self::markdown::MarkdownExporter;
 use self::orgmode::OrgmodeExporter;
 
 use crate::benchmark::benchmark_result::BenchmarkResult;
-use crate::options::SortOrder;
+use crate::metric::MetricSelection;
 use crate::output::console_writeln;
-use crate::quantity::TimeUnit;
 
 use anyhow::{Context, Result};
 use clap::ArgMatches;
@@ -46,13 +45,7 @@ pub enum ExportType {
 /// Interface for different exporters.
 trait Exporter {
     /// Export the given entries in the serialized form.
-    fn serialize(
-        &self,
-        results: &[BenchmarkResult],
-        time_unit: Option<TimeUnit>,
-        sort_order: SortOrder,
-        reference_index: Option<usize>,
-    ) -> Result<Vec<u8>>;
+    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>>;
 }
 
 pub enum ExportTarget {
@@ -68,22 +61,16 @@ struct ExporterWithTarget {
 /// Handles the management of multiple file exporters.
 pub struct ExportManager {
     exporters: Vec<ExporterWithTarget>,
-    time_unit: Option<TimeUnit>,
-    sort_order: SortOrder,
+    primary: MetricSelection,
 }
 
 impl ExportManager {
     /// Build the ExportManager that will export the results specified
     /// in the given ArgMatches
-    pub fn from_cli_arguments(
-        matches: &ArgMatches,
-        time_unit: Option<TimeUnit>,
-        sort_order: SortOrder,
-    ) -> Result<Self> {
+    pub fn from_cli_arguments(matches: &ArgMatches, primary: MetricSelection) -> Result<Self> {
         let mut export_manager = Self {
             exporters: vec![],
-            time_unit,
-            sort_order,
+            primary,
         };
         {
             let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -131,19 +118,9 @@ impl ExportManager {
     /// results are written to all file targets (to always have them up to date, even
     /// if a benchmark fails). In the latter case, we only print to stdout targets (in
     /// order not to clutter the output of hyperfine with intermediate results).
-    /// `reference_index` refers to the complete benchmark sequence, so it can
-    /// point past `results` while that benchmark is still pending.
-    pub fn write_results(
-        &self,
-        results: &[BenchmarkResult],
-        intermediate: bool,
-        reference_index: Option<usize>,
-    ) -> Result<()> {
+    pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
         for e in &self.exporters {
-            let content = || {
-                e.exporter
-                    .serialize(results, self.time_unit, self.sort_order, reference_index)
-            };
+            let content = || e.exporter.serialize(results, self.primary);
 
             match e.target {
                 ExportTarget::File(ref filename) => {

src/export/tests.rs

@@ -2,19 +2,27 @@ use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::benchmark::measurement::{Measurement, Measurements};
 use crate::export::asciidoc::AsciidocExporter;
+use crate::export::markdown::MarkdownExporter;
 use crate::export::orgmode::OrgmodeExporter;
-use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero};
-use crate::{export::markdown::MarkdownExporter, options::SortOrder};
+use crate::metric::{MetricSelection, Unit};
+use crate::quantity::{byte, second, Information, Time, Zero};
 use std::collections::BTreeMap;
 use std::process::ExitStatus;
 
-fn get_output<E: Exporter + Default>(
-    results: &[BenchmarkResult],
-    unit: Option<TimeUnit>,
-    sort_order: SortOrder,
-) -> String {
+fn get_output<E: Exporter + Default>(results: &[BenchmarkResult], unit: Option<Unit>) -> String {
     let exporter = E::default();
-    String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap()
+    String::from_utf8(
+        exporter
+            .serialize(
+                results,
+                MetricSelection {
+                    unit,
+                    ..MetricSelection::default()
+                },
+            )
+            .unwrap(),
+    )
+    .unwrap()
 }
 
 /// Ensure the makrup output includes the table header and the multiple
@@ -33,6 +41,7 @@ fn test_markup_export_auto_ms() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.09),
+                    time_cpu: Time::new::<second>(0.09),
                     time_user: Time::new::<second>(0.09),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -41,6 +50,7 @@ fn test_markup_export_auto_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.10),
+                    time_cpu: Time::new::<second>(0.10),
                     time_user: Time::new::<second>(0.10),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -49,6 +59,7 @@ fn test_markup_export_auto_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.14),
+                    time_cpu: Time::new::<second>(0.14),
                     time_user: Time::new::<second>(0.14),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -65,6 +76,7 @@ fn test_markup_export_auto_ms() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.0),
+                    time_cpu: Time::new::<second>(2.0),
                     time_user: Time::new::<second>(2.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -73,6 +85,7 @@ fn test_markup_export_auto_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(3.0),
+                    time_cpu: Time::new::<second>(3.0),
                     time_user: Time::new::<second>(3.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -81,6 +94,7 @@ fn test_markup_export_auto_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(4.0),
+                    time_cpu: Time::new::<second>(4.0),
                     time_user: Time::new::<second>(4.0),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -92,42 +106,39 @@ fn test_markup_export_auto_ms() {
         },
     ];
 
-    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None, SortOrder::Command), @r#"
-    | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
-    |:---|---:|---:|---:|---:|
-    | `sleep 0.1` | 110.0 ± 26.5 | 90.0 | 140.0 | 1.00 |
-    | `sleep 2` | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | 27.27 ± 11.21 |
-    "#);
+    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None), @"
+    | Command | Mean Wall Time [ms] | Change | Factor |
+    |:---|---:|---:|:---|
+    | `sleep 0.1` | 110.0 ± 26.5 |  |  |
+    | `sleep 2` | 3000.0 ± 1000.0 | +2627.3% | (27.3x slower) |
+    ");
 
-    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None, SortOrder::Command), @r#"
-    [cols="<,>,>,>,>"]
+    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None), @r#"
+    [cols="<,>,>,<"]
     |===
-    | Command 
-    | Mean [ms] 
-    | Min [ms] 
-    | Max [ms] 
-    | Relative 
+    | Command
+    | Mean Wall Time [ms]
+    | Change
+    | Factor
 
-    | `sleep 0.1` 
-    | 110.0 ± 26.5 
-    | 90.0 
-    | 140.0 
-    | 1.00 
+    | `sleep 0.1`
+    | 110.0 ± 26.5
+    |
+    |
 
-    | `sleep 2` 
-    | 3000.0 ± 1000.0 
-    | 2000.0 
-    | 4000.0 
-    | 27.27 ± 11.21 
+    | `sleep 2`
+    | 3000.0 ± 1000.0
+    | +2627.3%
+    | (27.3x slower)
     |===
     "#);
 
-    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None, SortOrder::Command), @r#"
-    | Command  |  Mean [ms] |  Min [ms] |  Max [ms] |  Relative |
-    |--+--+--+--+--|
-    | =sleep 0.1=  |  110.0 ± 26.5 |  90.0 |  140.0 |  1.00 |
-    | =sleep 2=  |  3000.0 ± 1000.0 |  2000.0 |  4000.0 |  27.27 ± 11.21 |
-    "#);
+    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None), @"
+    | Command  |  Mean Wall Time [ms] |  Change |  Factor |
+    |--+--+--+--|
+    | =sleep 0.1=  |  110.0 ± 26.5 |   |   |
+    | =sleep 2=  |  3000.0 ± 1000.0 |  +2627.3% |  (27.3x slower) |
+    ");
 }
 
 /// This (again) demonstrates that the first entry's units (s) are used to set
@@ -142,6 +153,7 @@ fn test_markup_export_auto_s() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.1),
+                    time_cpu: Time::new::<second>(2.1),
                     time_user: Time::new::<second>(2.1),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -150,6 +162,7 @@ fn test_markup_export_auto_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.2),
+                    time_cpu: Time::new::<second>(2.2),
                     time_user: Time::new::<second>(2.2),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -158,6 +171,7 @@ fn test_markup_export_auto_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.3),
+                    time_cpu: Time::new::<second>(2.3),
                     time_user: Time::new::<second>(2.3),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -174,6 +188,7 @@ fn test_markup_export_auto_s() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.1),
+                    time_cpu: Time::new::<second>(0.1),
                     time_user: Time::new::<second>(0.1),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -182,6 +197,7 @@ fn test_markup_export_auto_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.2),
+                    time_cpu: Time::new::<second>(0.2),
                     time_user: Time::new::<second>(0.2),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -190,6 +206,7 @@ fn test_markup_export_auto_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.3),
+                    time_cpu: Time::new::<second>(0.3),
                     time_user: Time::new::<second>(0.3),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -201,42 +218,39 @@ fn test_markup_export_auto_s() {
         },
     ];
 
-    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None, SortOrder::Command), @r#"
-    | Command | Mean [s] | Min [s] | Max [s] | Relative |
-    |:---|---:|---:|---:|---:|
-    | `sleep 2` | 2.200 ± 0.100 | 2.100 | 2.300 | 11.00 ± 5.52 |
-    | `sleep 0.1` | 0.200 ± 0.100 | 0.100 | 0.300 | 1.00 |
-    "#);
+    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None), @"
+    | Command | Mean Wall Time [s] | Change | Factor |
+    |:---|---:|---:|:---|
+    | `sleep 2` | 2.200 ± 0.100 |  |  |
+    | `sleep 0.1` | 0.200 ± 0.100 | -90.9% | (11.0x faster) |
+    ");
 
-    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None, SortOrder::Command), @r#"
-    [cols="<,>,>,>,>"]
+    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None), @r#"
+    [cols="<,>,>,<"]
     |===
-    | Command 
-    | Mean [s] 
-    | Min [s] 
-    | Max [s] 
-    | Relative 
+    | Command
+    | Mean Wall Time [s]
+    | Change
+    | Factor
 
-    | `sleep 2` 
-    | 2.200 ± 0.100 
-    | 2.100 
-    | 2.300 
-    | 11.00 ± 5.52 
+    | `sleep 2`
+    | 2.200 ± 0.100
+    |
+    |
 
-    | `sleep 0.1` 
-    | 0.200 ± 0.100 
-    | 0.100 
-    | 0.300 
-    | 1.00 
+    | `sleep 0.1`
+    | 0.200 ± 0.100
+    | -90.9%
+    | (11.0x faster)
     |===
     "#);
 
-    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None, SortOrder::Command), @r#"
-    | Command  |  Mean [s] |  Min [s] |  Max [s] |  Relative |
-    |--+--+--+--+--|
-    | =sleep 2=  |  2.200 ± 0.100 |  2.100 |  2.300 |  11.00 ± 5.52 |
-    | =sleep 0.1=  |  0.200 ± 0.100 |  0.100 |  0.300 |  1.00 |
-    "#);
+    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None), @"
+    | Command  |  Mean Wall Time [s] |  Change |  Factor |
+    |--+--+--+--|
+    | =sleep 2=  |  2.200 ± 0.100 |   |   |
+    | =sleep 0.1=  |  0.200 ± 0.100 |  -90.9% |  (11.0x faster) |
+    ");
 }
 
 /// This (again) demonstrates that the given time unit (ms) is used to set
@@ -251,6 +265,7 @@ fn test_markup_export_manual_ms() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.1),
+                    time_cpu: Time::new::<second>(2.1),
                     time_user: Time::new::<second>(2.1),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -259,6 +274,7 @@ fn test_markup_export_manual_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.2),
+                    time_cpu: Time::new::<second>(2.2),
                     time_user: Time::new::<second>(2.2),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -267,6 +283,7 @@ fn test_markup_export_manual_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.3),
+                    time_cpu: Time::new::<second>(2.3),
                     time_user: Time::new::<second>(2.3),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -283,6 +300,7 @@ fn test_markup_export_manual_ms() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.1),
+                    time_cpu: Time::new::<second>(0.1),
                     time_user: Time::new::<second>(0.1),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -291,6 +309,7 @@ fn test_markup_export_manual_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.2),
+                    time_cpu: Time::new::<second>(0.2),
                     time_user: Time::new::<second>(0.2),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -299,6 +318,7 @@ fn test_markup_export_manual_ms() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.3),
+                    time_cpu: Time::new::<second>(0.3),
                     time_user: Time::new::<second>(0.3),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -310,42 +330,39 @@ fn test_markup_export_manual_ms() {
         },
     ];
 
-    insta::assert_snapshot!(get_output::<MarkdownExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
-    | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
-    |:---|---:|---:|---:|---:|
-    | `sleep 2` | 2200.0 ± 100.0 | 2100.0 | 2300.0 | 11.00 ± 5.52 |
-    | `sleep 0.1` | 200.0 ± 100.0 | 100.0 | 300.0 | 1.00 |
-    "#);
+    insta::assert_snapshot!(get_output::<MarkdownExporter>(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @"
+    | Command | Mean Wall Time [ms] | Change | Factor |
+    |:---|---:|---:|:---|
+    | `sleep 2` | 2200.0 ± 100.0 |  |  |
+    | `sleep 0.1` | 200.0 ± 100.0 | -90.9% | (11.0x faster) |
+    ");
 
-    insta::assert_snapshot!(get_output::<AsciidocExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
-    [cols="<,>,>,>,>"]
+    insta::assert_snapshot!(get_output::<AsciidocExporter>(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @r#"
+    [cols="<,>,>,<"]
     |===
-    | Command 
-    | Mean [ms] 
-    | Min [ms] 
-    | Max [ms] 
-    | Relative 
+    | Command
+    | Mean Wall Time [ms]
+    | Change
+    | Factor
 
-    | `sleep 2` 
-    | 2200.0 ± 100.0 
-    | 2100.0 
-    | 2300.0 
-    | 11.00 ± 5.52 
+    | `sleep 2`
+    | 2200.0 ± 100.0
+    |
+    |
 
-    | `sleep 0.1` 
-    | 200.0 ± 100.0 
-    | 100.0 
-    | 300.0 
-    | 1.00 
+    | `sleep 0.1`
+    | 200.0 ± 100.0
+    | -90.9%
+    | (11.0x faster)
     |===
     "#);
 
-    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
-    | Command  |  Mean [ms] |  Min [ms] |  Max [ms] |  Relative |
-    |--+--+--+--+--|
-    | =sleep 2=  |  2200.0 ± 100.0 |  2100.0 |  2300.0 |  11.00 ± 5.52 |
-    | =sleep 0.1=  |  200.0 ± 100.0 |  100.0 |  300.0 |  1.00 |
-    "#);
+    insta::assert_snapshot!(get_output::<OrgmodeExporter>(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @"
+    | Command  |  Mean Wall Time [ms] |  Change |  Factor |
+    |--+--+--+--|
+    | =sleep 2=  |  2200.0 ± 100.0 |   |   |
+    | =sleep 0.1=  |  200.0 ± 100.0 |  -90.9% |  (11.0x faster) |
+    ");
 }
 
 /// The given time unit (s) is used to set the units for all entries.
@@ -359,6 +376,7 @@ fn test_markup_export_manual_s() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.01),
+                    time_cpu: Time::new::<second>(2.01),
                     time_user: Time::new::<second>(2.01),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -367,6 +385,7 @@ fn test_markup_export_manual_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.02),
+                    time_cpu: Time::new::<second>(2.02),
                     time_user: Time::new::<second>(2.02),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -375,6 +394,7 @@ fn test_markup_export_manual_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(2.03),
+                    time_cpu: Time::new::<second>(2.03),
                     time_user: Time::new::<second>(2.03),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -391,6 +411,7 @@ fn test_markup_export_manual_s() {
             measurements: Measurements::new(vec![
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.11),
+                    time_cpu: Time::new::<second>(0.11),
                     time_user: Time::new::<second>(0.11),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -399,6 +420,7 @@ fn test_markup_export_manual_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.12),
+                    time_cpu: Time::new::<second>(0.12),
                     time_user: Time::new::<second>(0.12),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -407,6 +429,7 @@ fn test_markup_export_manual_s() {
                 },
                 Measurement {
                     time_wall_clock: Time::new::<second>(0.13),
+                    time_cpu: Time::new::<second>(0.13),
                     time_user: Time::new::<second>(0.13),
                     time_system: Time::zero(),
                     memory_peak_resident: Some(Information::new::<byte>(1024.)),
@@ -418,40 +441,30 @@ fn test_markup_export_manual_s() {
         },
     ];
 
-    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, Some(TimeUnit::Second), SortOrder::Command), @r#"
-    | Command | Mean [s] | Min [s] | Max [s] | Relative |
-    |:---|---:|---:|---:|---:|
-    | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 |
-    | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 |
-    "#);
+    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, Some(Unit { symbol: "s", scale: 1.0 })), @"
+    | Command | Mean Wall Time [s] | Change | Factor |
+    |:---|---:|---:|:---|
+    | `sleep 2` | 2.020 ± 0.010 |  |  |
+    | `sleep 0.1` | 0.120 ± 0.010 | -94.1% | (16.8x faster) |
+    ");
 
-    insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, Some(TimeUnit::Second), SortOrder::MeanTime), @r#"
-    | Command | Mean [s] | Min [s] | Max [s] | Relative |
-    |:---|---:|---:|---:|---:|
-    | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 |
-    | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 |
-    "#);
-
-    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, Some(TimeUnit::Second), SortOrder::Command), @r#"
-    [cols="<,>,>,>,>"]
+    insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, Some(Unit { symbol: "s", scale: 1.0 })), @r#"
+    [cols="<,>,>,<"]
     |===
-    | Command 
-    | Mean [s] 
-    | Min [s] 
-    | Max [s] 
-    | Relative 
+    | Command
+    | Mean Wall Time [s]
+    | Change
+    | Factor
 
-    | `sleep 2` 
-    | 2.020 ± 0.010 
-    | 2.010 
-    | 2.030 
-    | 16.83 ± 1.41 
+    | `sleep 2`
+    | 2.020 ± 0.010
+    |
+    |
 
-    | `sleep 0.1` 
-    | 0.120 ± 0.010 
-    | 0.110 
-    | 0.130 
-    | 1.00 
+    | `sleep 0.1`
+    | 0.120 ± 0.010
+    | -94.1%
+    | (16.8x faster)
     |===
     "#);
 }

src/main.rs

@@ -21,6 +21,7 @@ pub mod cli;
 pub mod command;
 pub mod error;
 pub mod export;
+pub mod metric;
 pub mod options;
 pub mod outlier_detection;
 pub mod output;
@@ -39,15 +40,10 @@ fn run() -> Result<()> {
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
     options.validate_against_command_list(&commands)?;
 
-    let export_manager = ExportManager::from_cli_arguments(
-        &cli_arguments,
-        options.time_unit,
-        options.sort_order_exports,
-    )?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
     scheduler.run_benchmarks()?;
-    scheduler.print_relative_speed_comparison()?;
     scheduler.final_export()?;
 
     Ok(())

src/metric.rs

@@ -0,0 +1,505 @@
+//! Metric selection, descriptive statistics, and display units.
+
+use anyhow::{bail, ensure, Result};
+
+use crate::benchmark::measurement::{Measurement, Measurements};
+use crate::quantity::statistics::{max, mean, median, min, standard_deviation};
+use crate::quantity::{byte, ratio, second, Ratio};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Metric {
+    TimeWallClock,
+    TimeCpu,
+    TimeUser,
+    TimeSystem,
+    MemoryPeakResident,
+    CpuCycles,
+    Instructions,
+    CacheReferences,
+    CacheMisses,
+    BranchMisses,
+}
+
+impl Metric {
+    pub const ALL: [Self; 10] = [
+        Self::TimeWallClock,
+        Self::TimeCpu,
+        Self::TimeUser,
+        Self::TimeSystem,
+        Self::MemoryPeakResident,
+        Self::CpuCycles,
+        Self::Instructions,
+        Self::CacheReferences,
+        Self::CacheMisses,
+        Self::BranchMisses,
+    ];
+
+    pub fn name(self) -> &'static str {
+        match self {
+            Self::TimeWallClock => "time_wall_clock",
+            Self::TimeCpu => "time_cpu",
+            Self::TimeUser => "time_user",
+            Self::TimeSystem => "time_system",
+            Self::MemoryPeakResident => "memory_peak_resident",
+            Self::CpuCycles => "cpu_cycles",
+            Self::Instructions => "instructions",
+            Self::CacheReferences => "cache_references",
+            Self::CacheMisses => "cache_misses",
+            Self::BranchMisses => "branch_misses",
+        }
+    }
+
+    pub fn label(self) -> &'static str {
+        match self {
+            Self::TimeWallClock => "Wall Time",
+            Self::TimeCpu => "CPU time",
+            Self::TimeUser => "User time",
+            Self::TimeSystem => "System time",
+            Self::MemoryPeakResident => "Memory",
+            Self::CpuCycles => "CPU cycles",
+            Self::Instructions => "Instructions",
+            Self::CacheReferences => "Cache references",
+            Self::CacheMisses => "Cache misses",
+            Self::BranchMisses => "Branch misses",
+        }
+    }
+
+    /// Extract a value in canonical units: seconds, bytes, or counts.
+    pub fn value(self, measurement: &Measurement) -> Option<f64> {
+        let counters = &measurement.hardware_counters;
+        match self {
+            Self::TimeWallClock => Some(measurement.time_wall_clock.get::<second>()),
+            Self::TimeCpu => Some(measurement.time_cpu.get::<second>()),
+            Self::TimeUser => Some(measurement.time_user.get::<second>()),
+            Self::TimeSystem => Some(measurement.time_system.get::<second>()),
+            Self::MemoryPeakResident => measurement.memory_peak_resident.map(|v| v.get::<byte>()),
+            Self::CpuCycles => counters.cpu_cycles.map(|v| v as f64),
+            Self::Instructions => counters.instructions.map(|v| v as f64),
+            Self::CacheReferences => counters.cache_references.map(|v| v as f64),
+            Self::CacheMisses => counters.cache_misses.map(|v| v as f64),
+            Self::BranchMisses => counters.branch_misses.map(|v| v as f64),
+        }
+    }
+
+    pub fn is_time(self) -> bool {
+        matches!(
+            self,
+            Self::TimeWallClock | Self::TimeCpu | Self::TimeUser | Self::TimeSystem
+        )
+    }
+
+    pub fn is_counter(self) -> bool {
+        !self.is_time() && self != Self::MemoryPeakResident
+    }
+
+    /// Reject known unsupported configurations before any commands execute.
+    /// Availability that depends on permissions or hardware is checked after collection.
+    pub fn ensure_available(self, shell: bool) -> Result<()> {
+        if self.is_counter() {
+            ensure!(
+                !shell,
+                "Metric '{}' is unavailable when a shell is enabled",
+                self.name()
+            );
+            ensure!(
+                cfg!(any(
+                    target_os = "macos",
+                    all(
+                        target_os = "linux",
+                        target_pointer_width = "64",
+                        any(
+                            target_arch = "aarch64",
+                            target_arch = "x86_64",
+                            target_arch = "riscv64",
+                            target_arch = "powerpc64",
+                            target_arch = "loongarch64"
+                        )
+                    )
+                )),
+                "Metric '{}' is unavailable on this platform",
+                self.name()
+            );
+            ensure!(
+                !cfg!(target_os = "macos") || matches!(self, Self::CpuCycles | Self::Instructions),
+                "Metric '{}' is unavailable on macOS",
+                self.name()
+            );
+        }
+        ensure!(
+            !cfg!(windows) || self != Self::MemoryPeakResident,
+            "Metric '{}' is unavailable on Windows",
+            self.name()
+        );
+        Ok(())
+    }
+
+    pub fn base_unit(self) -> Unit {
+        if self.is_time() {
+            Unit::new("s", 1.0)
+        } else if self.is_counter() {
+            Unit::new("count", 1.0)
+        } else {
+            Unit::new("B", 1.0)
+        }
+    }
+
+    fn units(self) -> &'static [Unit] {
+        if self.is_time() {
+            TIME_UNITS
+        } else if self.is_counter() {
+            COUNT_UNITS
+        } else {
+            MEMORY_UNITS
+        }
+    }
+
+    /// Require complete coverage so different metrics describe the same runs.
+    pub fn summarize(self, measurements: &Measurements) -> Option<Stats> {
+        let values = measurements
+            .measurements
+            .iter()
+            .map(|measurement| self.value(measurement))
+            .collect::<Option<Vec<_>>>()?;
+        if values.is_empty() {
+            return None;
+        }
+        Some(Stats {
+            count: values.len(),
+            mean: mean(values.iter().copied()),
+            stddev: (values.len() > 1).then(|| {
+                standard_deviation(values.iter().map(|&v| Ratio::new::<ratio>(v))).get::<ratio>()
+            }),
+            median: median(values.iter().copied()),
+            min: min(values.iter().copied()),
+            max: max(values.iter().copied()),
+        })
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct MetricSelection {
+    pub metric: Metric,
+    pub unit: Option<Unit>,
+}
+
+impl Default for MetricSelection {
+    fn default() -> Self {
+        Self {
+            metric: Metric::TimeWallClock,
+            unit: None,
+        }
+    }
+}
+
+impl MetricSelection {
+    /// Parse an ordered, nonempty list of `METRIC[:UNIT]` selections.
+    pub fn parse_list(input: &str) -> Result<Vec<Self>> {
+        let mut selections = Vec::new();
+        for item in input.split(',') {
+            let (name, unit_name) = match item.split_once(':') {
+                Some((name, unit)) => (name, Some(unit)),
+                None => (item, None),
+            };
+            let metric = match Metric::ALL.iter().copied().find(|m| m.name() == name) {
+                Some(metric) => metric,
+                None => bail!(
+                    "Unknown metric '{name}'. Expected one of: {}",
+                    Metric::ALL.map(Metric::name).join(", ")
+                ),
+            };
+            if selections
+                .iter()
+                .any(|selection: &Self| selection.metric == metric)
+            {
+                bail!("Metric '{name}' was selected more than once");
+            }
+            let unit = match unit_name {
+                None => None,
+                Some(unit_name) => {
+                    // Accept an ASCII spelling for microseconds as well.
+                    let unit_name = if unit_name == "us" { "µs" } else { unit_name };
+                    match metric.units().iter().find(|unit| unit.symbol == unit_name) {
+                        Some(unit) => Some(*unit),
+                        None => bail!(
+                            "Unit '{unit_name}' is not valid for metric '{name}'. Expected one of: {}",
+                            metric.units().iter().map(|u| u.symbol).collect::<Vec<_>>().join(", ")
+                        ),
+                    }
+                }
+            };
+            selections.push(Self { metric, unit });
+        }
+        Ok(selections)
+    }
+
+    pub fn display_unit(self, mean: f64) -> Unit {
+        if let Some(unit) = self.unit {
+            return unit;
+        }
+        if mean == 0.0 {
+            return self.metric.base_unit();
+        }
+        let units = if self.metric == Metric::MemoryPeakResident {
+            BINARY_MEMORY_UNITS
+        } else {
+            self.metric.units()
+        };
+        units
+            .iter()
+            .rev()
+            .find(|unit| mean.abs() >= unit.scale)
+            .copied()
+            .unwrap_or(units[0])
+    }
+
+    /// CSV never automatically scales values.
+    pub fn csv_unit(self) -> Unit {
+        self.unit.unwrap_or_else(|| self.metric.base_unit())
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Unit {
+    pub symbol: &'static str,
+    /// The number of canonical units in this unit (e.g. 1e9 for a billion).
+    pub scale: f64,
+}
+
+impl Unit {
+    const fn new(symbol: &'static str, scale: f64) -> Self {
+        Self { symbol, scale }
+    }
+
+    pub fn format(self, value: f64) -> String {
+        let value = self.format_value(value);
+        if self.symbol == "count" {
+            value
+        } else {
+            format!("{value} {}", self.symbol)
+        }
+    }
+
+    pub fn format_value(self, value: f64) -> String {
+        let precision = if self.symbol == "s" { 3 } else { 1 };
+        format!("{:.precision$}", value / self.scale)
+    }
+}
+
+const TIME_UNITS: &[Unit] = &[
+    Unit::new("ns", 1e-9),
+    Unit::new("µs", 1e-6),
+    Unit::new("ms", 1e-3),
+    Unit::new("s", 1.0),
+    Unit::new("min", 60.0),
+    Unit::new("h", 3600.0),
+];
+
+const BINARY_MEMORY_UNITS: &[Unit] = &[
+    Unit::new("B", 1.0),
+    Unit::new("KiB", 1024.0),
+    Unit::new("MiB", 1_048_576.0),
+    Unit::new("GiB", 1_073_741_824.0),
+    Unit::new("TiB", 1_099_511_627_776.0),
+];
+
+const MEMORY_UNITS: &[Unit] = &[
+    Unit::new("B", 1.0),
+    Unit::new("kB", 1e3),
+    Unit::new("MB", 1e6),
+    Unit::new("GB", 1e9),
+    Unit::new("TB", 1e12),
+    Unit::new("KiB", 1024.0),
+    Unit::new("MiB", 1_048_576.0),
+    Unit::new("GiB", 1_073_741_824.0),
+    Unit::new("TiB", 1_099_511_627_776.0),
+];
+
+const COUNT_UNITS: &[Unit] = &[
+    Unit::new("count", 1.0),
+    Unit::new("k", 1e3),
+    Unit::new("M", 1e6),
+    Unit::new("B", 1e9),
+];
+
+#[derive(Debug, Clone, Copy)]
+pub struct Stats {
+    pub count: usize,
+    pub mean: f64,
+    pub stddev: Option<f64>,
+    pub median: f64,
+    pub min: f64,
+    pub max: f64,
+}
+
+impl Stats {
+    pub fn change_from(&self, reference: &Self) -> Option<f64> {
+        (reference.mean != 0.0).then(|| 100.0 * (self.mean / reference.mean - 1.0))
+    }
+
+    pub fn format_change_from(&self, reference: &Self) -> String {
+        let Some(change) = self.change_from(reference) else {
+            return "N/A".to_owned();
+        };
+        let mut formatted = format!("{change:+.1}%");
+        if formatted == "+0.0%" || formatted == "-0.0%" {
+            formatted = "0.0%".to_owned();
+        }
+        formatted
+    }
+
+    pub fn format_factor_from(&self, reference: &Self, metric: Metric) -> String {
+        if reference.mean > 0.0
+            && (self.mean >= 1.3 * reference.mean || self.mean <= 0.7 * reference.mean)
+        {
+            let decrease = self.mean < reference.mean;
+            let factor = if decrease {
+                reference.mean / self.mean
+            } else {
+                self.mean / reference.mean
+            };
+            let direction = match (metric.is_time(), decrease) {
+                (true, true) => "faster",
+                (true, false) => "slower",
+                (false, true) => "less",
+                (false, false) => "more",
+            };
+            let factor = if factor.is_infinite() {
+                "∞".to_owned()
+            } else {
+                format!("{factor:.1}")
+            };
+            format!("({factor}x {direction})")
+        } else {
+            String::new()
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::quantity::{Information, Time};
+
+    #[test]
+    fn large_changes_include_factors() {
+        let reference = Stats {
+            count: 1,
+            mean: 100.0,
+            stddev: None,
+            median: 100.0,
+            min: 100.0,
+            max: 100.0,
+        };
+        for (mean, metric, expected) in [
+            (130.0, Metric::TimeWallClock, "+30.0% (1.3x slower)"),
+            (70.0, Metric::TimeCpu, "-30.0% (1.4x faster)"),
+            (10.0, Metric::Instructions, "-90.0% (10.0x less)"),
+            (1000.0, Metric::MemoryPeakResident, "+900.0% (10.0x more)"),
+            (129.99, Metric::TimeWallClock, "+30.0%"),
+            (70.01, Metric::Instructions, "-30.0%"),
+            (100.0, Metric::Instructions, "0.0%"),
+            (0.0, Metric::Instructions, "-100.0% (∞x less)"),
+        ] {
+            let stats = Stats { mean, ..reference };
+            let (percentage, factor) = expected
+                .split_once(" (")
+                .map_or((expected, String::new()), |(p, f)| (p, format!("({f}")));
+            assert_eq!(stats.format_change_from(&reference), percentage);
+            assert_eq!(stats.format_factor_from(&reference, metric), factor);
+        }
+        let zero = Stats {
+            mean: 0.0,
+            ..reference
+        };
+        assert_eq!(reference.format_change_from(&zero), "N/A");
+    }
+
+    #[test]
+    fn cpu_time_statistics_use_per_run_totals() {
+        let measurements = Measurements::new(
+            [(1.0, 3.0), (3.0, 1.0)]
+                .iter()
+                .map(|&(user, system)| Measurement {
+                    time_cpu: Time::new::<second>(user + system),
+                    time_user: Time::new::<second>(user),
+                    time_system: Time::new::<second>(system),
+                    ..Measurement::default()
+                })
+                .collect(),
+        );
+        let selection = MetricSelection::parse_list("time_cpu:ms").unwrap()[0];
+        let stats = selection.metric.summarize(&measurements).unwrap();
+        assert_eq!(selection.csv_unit().format(stats.mean), "4000.0 ms");
+        // User and system time vary, but their per-run total is constant.
+        assert_eq!(stats.stddev, Some(0.0));
+        assert_eq!(stats.min, 4.0);
+        assert_eq!(stats.max, 4.0);
+        let json = serde_json::to_value(&measurements).unwrap();
+        for sample in json["measurements"].as_array().unwrap() {
+            assert_eq!(sample["time_cpu"]["value"], 4.0);
+            assert_eq!(sample["time_cpu"]["unit"], "second");
+        }
+    }
+
+    #[test]
+    fn selection_units_and_validation() {
+        let selections = MetricSelection::parse_list(
+            "instructions:B,memory_peak_resident:B,time_wall_clock:ms,time_user:us",
+        )
+        .unwrap();
+        assert_eq!(selections[0].csv_unit().scale, 1e9);
+        assert_eq!(selections[1].csv_unit().scale, 1.0);
+        assert_eq!(selections[2].csv_unit().format(0.125), "125.0 ms");
+        assert_eq!(selections[3].csv_unit().symbol, "µs");
+
+        let instructions = MetricSelection::parse_list("instructions").unwrap()[0];
+        assert_eq!(instructions.csv_unit().symbol, "count");
+        assert_eq!(
+            instructions.display_unit(143.2e9).format(143.2e9),
+            "143.2 B"
+        );
+        for invalid in [
+            "",
+            "unknown",
+            "time_wall_clock,",
+            "instructions,instructions:M",
+            "instructions:MiB",
+            "memory_peak_resident:ms",
+            "time_wall_clock:",
+        ] {
+            assert!(MetricSelection::parse_list(invalid).is_err(), "{}", invalid);
+        }
+    }
+
+    #[test]
+    fn summaries_require_complete_samples() {
+        let mut measurements = Measurements::new(vec![Measurement {
+            memory_peak_resident: Some(Information::new::<byte>(10.0)),
+            ..Measurement::default()
+        }]);
+        let single = Metric::MemoryPeakResident.summarize(&measurements).unwrap();
+        assert_eq!(single.stddev, None);
+        measurements.push(Measurement::default());
+        assert!(Metric::MemoryPeakResident
+            .summarize(&measurements)
+            .is_none());
+        measurements.measurements[1].memory_peak_resident = Some(Information::new::<byte>(30.0));
+        let stats = Metric::MemoryPeakResident.summarize(&measurements).unwrap();
+        assert_eq!(stats.mean, 20.0);
+        assert_eq!(stats.change_from(&single), Some(100.0));
+        assert_eq!(stats.format_change_from(&single), "+100.0%");
+        assert_eq!(single.format_change_from(&stats), "-50.0%");
+        assert_eq!(
+            Stats {
+                mean: 9.999,
+                ..single
+            }
+            .format_change_from(&single),
+            "0.0%"
+        );
+        approx::assert_relative_eq!(stats.stddev.unwrap(), 200.0_f64.sqrt());
+        let zero = Metric::TimeWallClock.summarize(&measurements).unwrap();
+        assert_eq!(zero.change_from(&zero), None);
+        assert_eq!(zero.format_change_from(&zero), "N/A");
+    }
+}

src/options.rs

@@ -4,12 +4,13 @@ use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::{cmp, env, fmt, io};
 
-use anyhow::{bail, ensure};
+use anyhow::ensure;
 use clap::ArgMatches;
 
 use crate::command::Commands;
 use crate::error::OptionsError;
-use crate::quantity::{second, Time, TimeUnit};
+use crate::metric::{Metric, MetricSelection};
+use crate::quantity::{second, Time};
 
 use anyhow::Result;
 
@@ -97,12 +98,6 @@ pub enum OutputStyleOption {
     Disabled,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SortOrder {
-    Command,
-    MeanTime,
-}
-
 /// Bounds for the number of benchmark runs
 pub struct RunBounds {
     /// Minimum number of benchmark runs
@@ -202,36 +197,21 @@ pub struct Options {
     /// Whether or not to ignore non-zero exit codes
     pub command_failure_action: CmdFailureAction,
 
-    // Standalone reference command, cleared when an existing benchmark is selected.
-    pub reference_command: Option<String>,
-
-    // Name of the reference command
-    pub reference_name: Option<String>,
-
-    // Index of the reference in the benchmark sequence, if any.
-    pub reference_index: Option<usize>,
-
-    /// Command(s) to run before each timing run
+    /// Command(s) to run before each benchmark run
     pub preparation_command: Option<Vec<String>>,
 
-    /// Command(s) to run after each timing run
+    /// Command(s) to run after each benchmark run
     pub conclusion_command: Option<Vec<String>>,
 
-    /// Command to run before each *batch* of timing runs, i.e. before each individual benchmark
+    /// Command to run before each *batch* of benchmark runs, i.e. before each individual benchmark
     pub setup_command: Option<String>,
 
-    /// Command to run after each *batch* of timing runs, i.e. after each individual benchmark
+    /// Command to run after each *batch* of benchmark runs, i.e. after each individual benchmark
     pub cleanup_command: Option<String>,
 
     /// What color mode to use for the terminal output
     pub output_style: OutputStyleOption,
 
-    /// How to order benchmarks in the relative speed comparison
-    pub sort_order_speed_comparison: SortOrder,
-
-    /// How to order benchmarks in the markup format exports
-    pub sort_order_exports: SortOrder,
-
     /// Determines how we run commands
     pub executor_kind: ExecutorKind,
 
@@ -241,8 +221,11 @@ pub struct Options {
     /// What to do with the output of the benchmarked commands
     pub command_output_policies: Vec<CommandOutputPolicy>,
 
-    /// Which time unit to use when displaying results
-    pub time_unit: Option<TimeUnit>,
+    /// Displayed metrics, with the primary metric first.
+    pub metrics: Vec<MetricSelection>,
+
+    /// The `all` preset displays only metrics with complete measurements.
+    pub skip_unavailable_metrics: bool,
 }
 
 impl Default for Options {
@@ -252,19 +235,16 @@ impl Default for Options {
             warmup_count: 0,
             min_benchmarking_time: Time::new::<second>(3.0),
             command_failure_action: CmdFailureAction::RaiseError,
-            reference_command: None,
-            reference_name: None,
-            reference_index: None,
             preparation_command: None,
             conclusion_command: None,
             setup_command: None,
             cleanup_command: None,
             output_style: OutputStyleOption::Full,
-            sort_order_speed_comparison: SortOrder::MeanTime,
-            sort_order_exports: SortOrder::Command,
             executor_kind: ExecutorKind::default(),
             command_output_policies: vec![CommandOutputPolicy::Null],
-            time_unit: None,
+            metrics: MetricSelection::parse_list(crate::cli::DEFAULT_METRICS)
+                .expect("valid default metrics"),
+            skip_unavailable_metrics: false,
             command_input_policy: CommandInputPolicy::Null,
         }
     }
@@ -307,11 +287,6 @@ impl Options {
 
         options.setup_command = matches.get_one::<String>("setup").map(String::from);
 
-        options.reference_command = matches.get_one::<String>("reference").map(String::from);
-        options.reference_name = matches
-            .get_one::<String>("reference-name")
-            .map(String::from);
-
         options.preparation_command = matches
             .get_many::<String>("prepare")
             .map(|values| values.map(String::from).collect::<Vec<String>>());
@@ -383,16 +358,6 @@ impl Options {
             OutputStyleOption::Disabled => {}
         };
 
-        (
-            options.sort_order_speed_comparison,
-            options.sort_order_exports,
-        ) = match matches.get_one::<String>("sort").map(|s| s.as_str()) {
-            None | Some("auto") => (SortOrder::MeanTime, SortOrder::Command),
-            Some("command") => (SortOrder::Command, SortOrder::Command),
-            Some("mean-time") => (SortOrder::MeanTime, SortOrder::MeanTime),
-            Some(_) => unreachable!("Unknown sort order"),
-        };
-
         options.executor_kind = if matches.get_flag("no-shell") {
             ExecutorKind::Raw
         } else if matches.get_flag("default-shell") {
@@ -430,14 +395,36 @@ impl Options {
             };
         }
 
-        options.time_unit = match matches.get_one::<String>("time-unit").map(|s| s.as_str()) {
-            Some("µs" | "us" | "microsecond" | "microseconds") => Some(TimeUnit::MicroSecond),
-            Some("ms" | "millisecond" | "milliseconds") => Some(TimeUnit::MilliSecond),
-            Some("s" | "second" | "seconds") => Some(TimeUnit::Second),
-            Some("min" | "minute" | "minutes") => Some(TimeUnit::Minute),
-            Some("h" | "hour" | "hours") => Some(TimeUnit::Hour),
-            _ => None,
-        };
+        match matches.get_one::<String>("metrics").unwrap().as_str() {
+            "default" => {}
+            "time" => {
+                options.metrics = [
+                    Metric::TimeWallClock,
+                    Metric::TimeCpu,
+                    Metric::TimeUser,
+                    Metric::TimeSystem,
+                ]
+                .map(|metric| MetricSelection { metric, unit: None })
+                .to_vec();
+            }
+            "all" => {
+                let shell = matches!(
+                    options.executor_kind,
+                    ExecutorKind::Shell(_) | ExecutorKind::Mock(Some(_))
+                );
+                options.metrics = Metric::ALL
+                    .iter()
+                    .copied()
+                    .filter(|metric| metric.ensure_available(shell).is_ok())
+                    .map(|metric| MetricSelection { metric, unit: None })
+                    .collect();
+                options.skip_unavailable_metrics = true;
+            }
+            list => {
+                options.metrics = MetricSelection::parse_list(list)
+                    .map_err(|error| OptionsError::InvalidMetrics(error.to_string()))?;
+            }
+        }
 
         if let Some(time) = matches.get_one::<String>("min-benchmarking-time") {
             options.min_benchmarking_time = Time::new::<second>(
@@ -466,43 +453,20 @@ impl Options {
     }
 
     pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> {
-        if let Some(reference) = &self.reference_command {
-            if commands
-                .iter()
-                .next()
-                .is_some_and(|command| !command.get_parameters().is_empty())
-            {
-                ensure!(
-                    self.reference_name.is_none(),
-                    "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark"
-                );
-                let mut matches = commands
-                    .iter()
-                    .enumerate()
-                    .filter(|(_, command)| command.get_name_with_unused_parameters() == *reference);
-                let Some((index, _)) = matches.next() else {
-                    bail!(
-                        "Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name."
-                    );
-                };
-                ensure!(
-                    matches.next().is_none(),
-                    "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names."
-                );
-                self.reference_index = Some(index);
-                self.reference_command = None;
-            } else {
-                self.reference_index = Some(0);
+        if !matches!(self.executor_kind, ExecutorKind::Mock(_)) {
+            for selection in &self.metrics {
+                selection
+                    .metric
+                    .ensure_available(matches!(self.executor_kind, ExecutorKind::Shell(_)))?;
             }
         }
-        let has_reference_command = self.reference_command.is_some();
-        let num_commands = commands.num_commands(has_reference_command);
+        let num_commands = commands.num_commands();
 
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
                 preparation_command.len() <= 1 || num_commands == preparation_command.len(),
                 "The '--prepare' option has to be provided just once or N times, where N={num_commands} is the \
-                 number of benchmark commands (including a potential reference)."
+                 number of benchmark commands."
             );
         }
 
@@ -510,7 +474,7 @@ impl Options {
             ensure!(
                 conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
                 "The '--conclude' option has to be provided just once or N times, where N={num_commands} is the \
-                 number of benchmark commands (including a potential reference)."
+                 number of benchmark commands."
             );
         }
 
@@ -521,7 +485,7 @@ impl Options {
             ensure!(
                 self.command_output_policies.len() == num_commands,
                 "The '--output' option has to be provided just once or N times, where N={num_commands} is the \
-                 number of benchmark commands (including a potential reference)."
+                 number of benchmark commands."
             );
         }
 

src/output/mod.rs

@@ -1,4 +1,5 @@
 pub mod progress_bar;
+pub mod report;
 pub mod warnings;
 
 /// Mark console write errors so broken pipes can be handled separately from file errors.

src/output/progress_bar.rs

@@ -16,29 +16,36 @@ const DEFAULT_TEMPLATE: &str = " {spinner} {msg:<30} {wide_bar} ETA {eta_precise
 const INITIAL_TEMPLATE: &str =
     " {spinner} Initial run: {initial_elapsed:<17} {wide_bar} ETA {eta_precise} ";
 
+fn style_with_template(bar: &ProgressBar, template: &str) -> ProgressStyle {
+    let template = if bar.prefix().is_empty() {
+        template.to_owned()
+    } else {
+        format!("{{prefix}}\n{template}")
+    };
+    bar.style().template(&template).expect("no template error")
+}
+
+/// Show the benchmark heading above the progress bar, clearing both on completion.
+pub fn set_benchmark_header(bar: &ProgressBar, header: String) {
+    bar.set_prefix(header);
+    bar.set_style(style_with_template(bar, DEFAULT_TEMPLATE));
+}
+
 /// Show the time elapsed since the initial benchmark command started.
 pub fn start_initial_measurement(bar: &ProgressBar, started: Instant) {
-    let style = bar
-        .style()
-        .template(INITIAL_TEMPLATE)
-        .expect("no template error")
-        .with_key(
-            "initial_elapsed",
-            move |_: &ProgressState, w: &mut dyn fmt::Write| {
-                write!(w, "{}", FormattedDuration(started.elapsed())).unwrap();
-            },
-        );
+    let style = style_with_template(bar, INITIAL_TEMPLATE).with_key(
+        "initial_elapsed",
+        move |_: &ProgressState, w: &mut dyn fmt::Write| {
+            write!(w, "{}", FormattedDuration(started.elapsed())).unwrap();
+        },
+    );
     bar.set_style(style);
 }
 
 /// Stop showing the initial timer and display the next phase's message.
 pub fn finish_initial_measurement(bar: &ProgressBar, message: String) {
     bar.set_message(message);
-    bar.set_style(
-        bar.style()
-            .template(DEFAULT_TEMPLATE)
-            .expect("no template error"),
-    );
+    bar.set_style(style_with_template(bar, DEFAULT_TEMPLATE));
 }
 
 /// Return a pre-configured progress bar

src/output/report.rs

@@ -0,0 +1,197 @@
+use std::io::{self, Write};
+
+use anyhow::{ensure, Result};
+use colored::Colorize;
+
+use crate::benchmark::{benchmark_result::BenchmarkResult, measurement::Measurements};
+use crate::metric::MetricSelection;
+use crate::output::console_writeln;
+
+// Reserve enough space for typical values and every supported unit suffix so
+// independently printed benchmarks keep the same layout.
+const MIN_VALUE_WIDTH: usize = 7;
+const MIN_UNIT_WIDTH: usize = 3;
+const MIN_CHANGE_WIDTH: usize = 14;
+
+/// Print a completed benchmark immediately, comparing every metric with benchmark 1.
+pub fn print(
+    number: usize,
+    name: &str,
+    measurements: &Measurements,
+    metrics: &[MetricSelection],
+    skip_unavailable_metrics: bool,
+    reference: Option<&BenchmarkResult>,
+) -> Result<()> {
+    let comparison = reference.is_some();
+    let multiple_runs = measurements.len() > 1;
+    let value_columns = 1..if multiple_runs { 5 } else { 2 };
+    let mut header = vec![
+        String::new(),
+        "mean".to_owned(),
+        "σ".to_owned(),
+        "min".to_owned(),
+        "max".to_owned(),
+    ];
+    if !multiple_runs {
+        header.truncate(2);
+        header[1] = "Value".to_owned();
+    }
+    if comparison {
+        header.extend([String::new(), String::new()]);
+    }
+    let mut rows = vec![(header, "")];
+    for selection in metrics {
+        let Some(summary) = selection.metric.summarize(measurements) else {
+            ensure!(
+                skip_unavailable_metrics,
+                "Metric '{}' is unavailable for '{name}'",
+                selection.metric.name()
+            );
+            continue;
+        };
+        let baseline = reference.and_then(|r| selection.metric.summarize(&r.measurements));
+        // Keep auto units consistent with the first command for easier comparison.
+        let scale_mean = baseline
+            .as_ref()
+            .filter(|stats| stats.mean != 0.0)
+            .unwrap_or(&summary)
+            .mean;
+        let unit = selection.display_unit(scale_mean);
+        let mut row = vec![
+            selection.metric.label().to_owned(),
+            unit.format_value(summary.mean),
+        ];
+        if let Some(stddev) = summary.stddev {
+            row.push(unit.format_value(stddev));
+            row.push(unit.format_value(summary.min));
+            row.push(unit.format_value(summary.max));
+        }
+        if comparison {
+            row.push(baseline.as_ref().map_or_else(
+                || "N/A".to_owned(),
+                |baseline| summary.format_change_from(baseline),
+            ));
+            row.push(baseline.as_ref().map_or_else(String::new, |baseline| {
+                summary.format_factor_from(baseline, selection.metric)
+            }));
+        }
+        rows.push((
+            row,
+            if unit.symbol == "count" {
+                ""
+            } else {
+                unit.symbol
+            },
+        ));
+    }
+    // Align numbers and unit suffixes independently so mixed units do not shift
+    // the values or the ± and … separators.
+    let unit_width = rows
+        .iter()
+        .map(|(_, unit)| unit.chars().count())
+        .max()
+        .unwrap()
+        .max(MIN_UNIT_WIDTH);
+    // Keep labels aligned across benchmarks even when some metrics are skipped.
+    let metric_width = metrics
+        .iter()
+        .map(|selection| selection.metric.label().chars().count())
+        .max()
+        .unwrap_or(0);
+    let widths: Vec<_> = (0..rows[0].0.len())
+        .map(|column| {
+            let width = rows
+                .iter()
+                .map(|(row, _)| row[column].chars().count())
+                .max()
+                .unwrap();
+            if column == 0 {
+                width.max(metric_width)
+            } else if value_columns.contains(&column) {
+                width.max(MIN_VALUE_WIDTH)
+            } else {
+                width.max(MIN_CHANGE_WIDTH)
+            }
+        })
+        .collect();
+    let mut stdout = io::stdout().lock();
+    console_writeln!(
+        stdout,
+        "{}: {} {}",
+        format!("Benchmark {}", number + 1).bold(),
+        name.white().bold(),
+        format!(
+            "({} {})",
+            measurements.len(),
+            if measurements.len() == 1 {
+                "run"
+            } else {
+                "runs"
+            }
+        )
+        .dimmed()
+    )?;
+    for (index, (row, unit)) in rows.iter().enumerate() {
+        let mut line = String::new();
+        for (column, value) in row.iter().enumerate() {
+            let is_factor = comparison && column == row.len() - 1;
+            let is_change = comparison && column >= row.len() - 2;
+            if index == 0 && is_change {
+                break;
+            }
+            if column > 0 {
+                line.push_str(match (multiple_runs, column) {
+                    (true, 2) => " ± ",
+                    (true, 4) => " … ",
+                    _ => "  ",
+                });
+            }
+            let padding = " ".repeat(widths[column] - value.chars().count());
+            let has_unit = value_columns.contains(&column);
+            let text = if has_unit && !unit.is_empty() {
+                format!("{value} {unit}")
+            } else {
+                value.clone()
+            };
+            let styled = if is_change {
+                if row[row.len() - 2].starts_with('-') {
+                    text.green().to_string()
+                } else if row[row.len() - 2].starts_with('+') {
+                    text.red().to_string()
+                } else {
+                    text.normal().to_string()
+                }
+            } else if column == 1 || column == 2 {
+                text.green().to_string()
+            } else if column == 3 {
+                text.cyan().to_string()
+            } else if column == 4 {
+                text.purple().to_string()
+            } else {
+                text.normal().to_string()
+            };
+            if column == 0 || is_factor {
+                line.push_str(&format!("{styled}{padding}"));
+            } else {
+                line.push_str(&format!("{padding}{styled}"));
+            }
+            if has_unit && unit_width > 0 {
+                // Empty suffixes (headers and unscaled counters) also need the
+                // space normally separating a number from its unit.
+                let unit_padding = unit_width - unit.chars().count() + usize::from(unit.is_empty());
+                line.push_str(&" ".repeat(unit_padding));
+            }
+        }
+        let line = line.trim_end();
+        console_writeln!(
+            stdout,
+            "  {}",
+            if index == 1 {
+                line.bold()
+            } else {
+                line.normal()
+            }
+        )?;
+    }
+    Ok(())
+}

src/output/warnings.rs

@@ -1,7 +1,8 @@
 use std::fmt;
 
 use crate::benchmark::MIN_EXECUTION_TIME;
-use crate::quantity::{FormatQuantity, Time, TimeUnit};
+use crate::metric::{Metric, MetricSelection};
+use crate::quantity::millisecond;
 
 pub struct OutlierWarningOptions {
     pub warmup_in_use: bool,
@@ -12,8 +13,7 @@ pub struct OutlierWarningOptions {
 pub enum Warnings {
     FastExecutionTime,
     NonZeroExitCode,
-    SlowInitialRun(Time, OutlierWarningOptions),
-    OutliersDetected(OutlierWarningOptions),
+    InitialRunOutlier(MetricSelection, f64, OutlierWarningOptions),
 }
 
 impl fmt::Display for Warnings {
@@ -21,19 +21,25 @@ impl fmt::Display for Warnings {
         match *self {
             Warnings::FastExecutionTime => write!(
                 f,
-                "Command took less than {} ms to complete. Note that the results might be \
+                "Command took less than {} ms to complete. Note that the wall-clock measurements might be \
                 inaccurate because hyperfine can not calibrate the shell startup time much \
                 more precise than this limit. You can try to use the `-N`/`--shell=none` \
                 option to disable the shell completely.",
-                MIN_EXECUTION_TIME.format_with_precision(TimeUnit::MilliSecond, 0)
+                MIN_EXECUTION_TIME.get::<millisecond>()
             ),
             Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
-            Warnings::SlowInitialRun(time_first_run, ref options) => write!(
+            Warnings::InitialRunOutlier(selection, value, _) if selection.metric != Metric::TimeWallClock => write!(
+                f,
+                "The first benchmarking run for this command had an unusually high value for the primary '{}' metric ({}) compared with the rest.",
+                selection.metric.name(),
+                selection.display_unit(value).format(value),
+            ),
+            Warnings::InitialRunOutlier(selection, value, ref options) => write!(
                 f,
                 "The first benchmarking run for this command was significantly slower than the \
                  rest ({time}). This could be caused by (filesystem) caches that were not filled until \
                  after the first run. {hints}",
-                time=time_first_run.format_auto(),
+                time=selection.display_unit(value).format(value),
                 hints=match (options.warmup_in_use, options.prepare_in_use) {
                     (true, true) => "You are already using both the '--warmup' option as well \
                     as the '--prepare' option. Consider re-running the benchmark on a quiet system. \
@@ -53,16 +59,7 @@ impl fmt::Display for Warnings {
                     option to clear the caches before each timing run."
                 }
             ),
-            Warnings::OutliersDetected(ref options) => write!(
-                f,
-                "Statistical outliers were detected. Consider re-running this benchmark on a quiet \
-                 system without any interferences from other programs.{hint}",
-                hint=if options.warmup_in_use && options.prepare_in_use {
-                    ""
-                } else {
-                    " It might help to use the '--warmup' or '--prepare' options."
-                }
-            ),
+
         }
     }
 }

src/quantity/mod.rs

@@ -11,80 +11,7 @@ pub use si::ratio::ratio;
 pub use si::time::{hour, microsecond, millisecond, minute, nanosecond, second};
 pub use uom::num_traits::Zero;
 
-pub use units::{InformationUnit, IsUnit, TimeUnit};
-
 pub mod statistics;
-mod units;
-
-pub trait QuantityInUnit<U> {
-    fn value_in_unit(self) -> f64;
-}
-
-impl<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U> for Time {
-    fn value_in_unit(self) -> f64 {
-        self.get::<U>()
-    }
-}
-
-impl<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U>
-    for Information
-{
-    fn value_in_unit(self) -> f64 {
-        self.get::<U>()
-    }
-}
-
-impl<U: uom::si::ratio::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U> for Ratio {
-    fn value_in_unit(self) -> f64 {
-        self.get::<U>()
-    }
-}
-
-pub trait FormatQuantity {
-    type Unit;
-
-    fn suitable_unit(&self) -> Self::Unit;
-
-    fn format_with_precision(&self, unit: Self::Unit, precision: usize) -> String;
-    fn format(&self, unit: Self::Unit) -> String;
-    fn format_auto(&self) -> String;
-    fn format_value(&self, unit: Self::Unit) -> String;
-}
-
-impl FormatQuantity for Time {
-    type Unit = TimeUnit;
-
-    fn suitable_unit(&self) -> TimeUnit {
-        if *self < Time::new::<millisecond>(1.0) {
-            TimeUnit::MicroSecond
-        } else if *self < Time::new::<second>(1.0) {
-            TimeUnit::MilliSecond
-        } else {
-            TimeUnit::Second
-        }
-    }
-
-    /// Format the time duration in the given unit with the given precision.
-    fn format_with_precision(&self, u: TimeUnit, precision: usize) -> String {
-        u.format(*self, precision)
-    }
-
-    /// Format the time duration in the given unit.
-    fn format(&self, unit: TimeUnit) -> String {
-        let value = self.format_with_precision(unit, unit.preferred_precision());
-        format!("{} {}", value, unit.short_name())
-    }
-
-    /// Format the given time duration. The unit will be determined automatically.
-    fn format_auto(&self) -> String {
-        self.format(self.suitable_unit())
-    }
-
-    /// Like `format`, but without displaying the unit.
-    fn format_value(&self, unit: TimeUnit) -> String {
-        self.format_with_precision(unit, unit.preferred_precision())
-    }
-}
 
 pub const fn const_time_from_seconds(value: f64) -> Time {
     // Quantity::new in uom is not yet const: https://docs.rs/uom/0.36.0/uom/si/struct.Quantity.html
@@ -95,43 +22,6 @@ pub const fn const_time_from_seconds(value: f64) -> Time {
     }
 }
 
-impl FormatQuantity for Information {
-    type Unit = InformationUnit;
-
-    fn suitable_unit(&self) -> InformationUnit {
-        if *self < Information::new::<kibibyte>(1.0) {
-            InformationUnit::Byte
-        } else if *self < Information::new::<mebibyte>(1.0) {
-            InformationUnit::KibiByte
-        } else if *self < Information::new::<gibibyte>(1.0) {
-            InformationUnit::MebiByte
-        } else {
-            InformationUnit::GibiByte
-        }
-    }
-
-    /// Format the information in the given unit with the given precision.
-    fn format_with_precision(&self, u: InformationUnit, precision: usize) -> String {
-        u.format(*self, precision)
-    }
-
-    /// Format the information in the given unit.
-    fn format(&self, unit: InformationUnit) -> String {
-        let value = self.format_with_precision(unit, unit.preferred_precision());
-        format!("{} {}", value, unit.short_name())
-    }
-
-    /// Format the given information. The unit will be determined automatically.
-    fn format_auto(&self) -> String {
-        self.format(self.suitable_unit())
-    }
-
-    /// Like `format`, but without displaying the unit.
-    fn format_value(&self, unit: InformationUnit) -> String {
-        self.format_with_precision(unit, unit.preferred_precision())
-    }
-}
-
 pub fn serialize_time<S>(t: &Time, s: S) -> Result<S::Ok, S::Error>
 where
     S: Serializer,
@@ -177,74 +67,3 @@ fn test_information() {
     let largest_exactly_representable = Information::new::<byte>(9_007_199_254_740_992.);
     assert_eq!(largest_exactly_representable.get::<pebibyte>(), 8.);
 }
-
-#[test]
-fn test_suiteable_unit_time() {
-    assert_eq!(Time::new::<second>(1.3).suitable_unit(), TimeUnit::Second);
-    assert_eq!(Time::new::<second>(1.0).suitable_unit(), TimeUnit::Second);
-    assert_eq!(
-        Time::new::<second>(0.999).suitable_unit(),
-        TimeUnit::MilliSecond
-    );
-    assert_eq!(
-        Time::new::<second>(0.0005).suitable_unit(),
-        TimeUnit::MicroSecond
-    );
-    assert_eq!(
-        Time::new::<second>(0.).suitable_unit(),
-        TimeUnit::MicroSecond
-    );
-    assert_eq!(
-        Time::new::<second>(1000.0).suitable_unit(),
-        TimeUnit::Second
-    );
-}
-
-#[test]
-fn test_suitable_unit_information() {
-    assert_eq!(
-        Information::new::<byte>(512.0).suitable_unit(),
-        InformationUnit::Byte
-    );
-    assert_eq!(
-        Information::new::<byte>(2048.0).suitable_unit(),
-        InformationUnit::KibiByte
-    );
-    assert_eq!(
-        Information::new::<byte>(2_097_152.0).suitable_unit(),
-        InformationUnit::MebiByte
-    );
-    assert_eq!(
-        Information::new::<byte>(2_147_483_648.0).suitable_unit(),
-        InformationUnit::GibiByte
-    );
-}
-
-#[test]
-fn test_format() {
-    let time = Time::new::<millisecond>(123.4);
-    assert_eq!(time.format(TimeUnit::Second), "0.123 s");
-    assert_eq!(time.format(TimeUnit::MilliSecond), "123.4 ms");
-    assert_eq!(time.format_auto(), "123.4 ms");
-    assert_eq!(time.format(TimeUnit::MicroSecond), "123400.0 µs");
-
-    let memory_peak_resident = Information::new::<kibibyte>(8.);
-    assert_eq!(memory_peak_resident.format_auto(), "8.0 KiB");
-    assert_eq!(memory_peak_resident.format(InformationUnit::Byte), "8192 B");
-    assert_eq!(
-        memory_peak_resident.format(InformationUnit::KibiByte),
-        "8.0 KiB"
-    );
-}
-
-#[test]
-fn test_format_duration_unit_with_unit() {
-    let out = Time::new::<second>(1.3).format(TimeUnit::Second);
-    assert_eq!("1.300 s", out);
-
-    let out = Time::new::<second>(1.3).format(TimeUnit::MilliSecond);
-    assert_eq!("1300.0 ms", out);
-
-    let out = Time::new::<second>(1.3).format(TimeUnit::MicroSecond);
-    assert_eq!("1300000.0 µs", out);
-}

src/quantity/statistics.rs

@@ -146,11 +146,6 @@ pub fn modified_zscores_f64(xs: &[f64]) -> Vec<f64> {
     xs.iter().map(|&x| (x - x_median) / mad).collect()
 }
 
-pub fn modified_zscores<Q: UnsafeRawValue>(values: &[Q]) -> Vec<f64> {
-    let values: Vec<_> = values.iter().map(|q| q.unsafe_raw_value()).collect();
-    modified_zscores_f64(&values)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

src/quantity/units.rs

@@ -1,166 +0,0 @@
-//! This module contains common units for time and information quantities.
-
-use std::marker::PhantomData;
-
-use crate::quantity::{
-    byte, gibibyte, hour, kibibyte, mebibyte, microsecond, millisecond, minute, second, tebibyte,
-    Information, Time,
-};
-
-pub trait IsUnit {
-    type Quantity;
-
-    fn dispatch(&self) -> Box<dyn UnitImpl<Quantity = Self::Quantity>>;
-
-    fn preferred_precision(&self) -> usize;
-
-    fn short_name(&self) -> &'static str {
-        self.dispatch().short_name()
-    }
-
-    fn format(&self, value: Self::Quantity, precision: usize) -> String {
-        self.dispatch().format_value(value, precision)
-    }
-}
-
-/// Supported time units
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TimeUnit {
-    MicroSecond,
-    MilliSecond,
-    Second,
-    Minute,
-    Hour,
-}
-
-impl IsUnit for TimeUnit {
-    type Quantity = Time;
-
-    fn dispatch(&self) -> Box<dyn UnitImpl<Quantity = Time>> {
-        match self {
-            TimeUnit::MicroSecond => Box::new(TimeUnitDispatcher::<microsecond>::new()),
-            TimeUnit::MilliSecond => Box::new(TimeUnitDispatcher::<millisecond>::new()),
-            TimeUnit::Second => Box::new(TimeUnitDispatcher::<second>::new()),
-            TimeUnit::Minute => Box::new(TimeUnitDispatcher::<minute>::new()),
-            TimeUnit::Hour => Box::new(TimeUnitDispatcher::<hour>::new()),
-        }
-    }
-
-    fn preferred_precision(&self) -> usize {
-        match self {
-            TimeUnit::Second => 3,
-            _ => 1,
-        }
-    }
-}
-
-/// Supported information units
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-
-pub enum InformationUnit {
-    Byte,
-    KibiByte,
-    MebiByte,
-    GibiByte,
-    TebiByte,
-}
-
-impl IsUnit for InformationUnit {
-    type Quantity = Information;
-
-    fn dispatch(&self) -> Box<dyn UnitImpl<Quantity = Information>> {
-        match self {
-            InformationUnit::Byte => Box::new(InformationUnitDispatcher::<byte>::new()),
-            InformationUnit::KibiByte => Box::new(InformationUnitDispatcher::<kibibyte>::new()),
-            InformationUnit::MebiByte => Box::new(InformationUnitDispatcher::<mebibyte>::new()),
-            InformationUnit::GibiByte => Box::new(InformationUnitDispatcher::<gibibyte>::new()),
-            InformationUnit::TebiByte => Box::new(InformationUnitDispatcher::<tebibyte>::new()),
-        }
-    }
-
-    fn preferred_precision(&self) -> usize {
-        match self {
-            InformationUnit::Byte => 0,
-            _ => 1,
-        }
-    }
-}
-
-pub trait UnitImpl {
-    type Quantity;
-
-    fn short_name(&self) -> &'static str;
-    fn format_value(&self, value: Self::Quantity, precision: usize) -> String;
-}
-
-struct TimeUnitDispatcher<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> {
-    u: PhantomData<U>,
-}
-
-impl<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> TimeUnitDispatcher<U> {
-    fn new() -> Self {
-        TimeUnitDispatcher { u: PhantomData }
-    }
-}
-
-impl<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> UnitImpl for TimeUnitDispatcher<U> {
-    type Quantity = Time;
-
-    fn short_name(&self) -> &'static str {
-        U::abbreviation()
-    }
-
-    fn format_value(&self, value: Time, precision: usize) -> String {
-        format!("{value:.precision$}", value = value.get::<U>())
-    }
-}
-
-struct InformationUnitDispatcher<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> {
-    u: PhantomData<U>,
-}
-
-impl<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> InformationUnitDispatcher<U> {
-    fn new() -> Self {
-        InformationUnitDispatcher { u: PhantomData }
-    }
-}
-
-impl<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> UnitImpl
-    for InformationUnitDispatcher<U>
-{
-    type Quantity = Information;
-
-    fn short_name(&self) -> &'static str {
-        U::abbreviation()
-    }
-
-    fn format_value(&self, value: Information, precision: usize) -> String {
-        format!("{value:.precision$}", value = value.get::<U>())
-    }
-}
-
-#[test]
-fn test_time_unit_short_name() {
-    assert_eq!("s", TimeUnit::Second.short_name());
-    assert_eq!("ms", TimeUnit::MilliSecond.short_name());
-    assert_eq!("µs", TimeUnit::MicroSecond.short_name());
-    assert_eq!("min", TimeUnit::Minute.short_name());
-    assert_eq!("h", TimeUnit::Hour.short_name());
-}
-
-// Note - the values are rounded when formatted.
-#[test]
-fn test_unit_format() {
-    use crate::quantity::FormatQuantity;
-
-    let value = Time::new::<second>(123.456789);
-    assert_eq!("123.457", value.format_value(TimeUnit::Second));
-    assert_eq!("123456.8", value.format_value(TimeUnit::MilliSecond));
-
-    assert_eq!(
-        "1234.6",
-        Time::new::<second>(0.00123456).format_value(TimeUnit::MicroSecond)
-    );
-    assert_eq!("2.1", value.format_value(TimeUnit::Minute));
-    assert_eq!("0.0", value.format_value(TimeUnit::Hour));
-}

src/timer/mod.rs

@@ -118,6 +118,7 @@ pub fn execute_and_measure(
 
     Ok(Measurement {
         time_wall_clock,
+        time_cpu: time_user + time_system,
         time_user,
         time_system,
         memory_peak_resident,