Alternative performance metrics in the terminal (#986)

src/benchmark/benchmark_result.rs

@@ -3,7 +3,6 @@ use std::collections::BTreeMap;
 use serde::Serialize;
 
 use crate::benchmark::measurement::Measurements;
-use crate::quantity::Time;
 
 /// Parameter value
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
@@ -47,9 +46,4 @@ impl BenchmarkResult {
     pub fn get_name(&self) -> &str {
         self.name.as_deref().unwrap_or(&self.command)
     }
-
-    /// The average wall clock time
-    pub fn mean_wall_clock_time(&self) -> Time {
-        self.measurements.time_wall_clock_mean()
-    }
 }

src/benchmark/executor.rs

@@ -220,6 +220,7 @@ impl Executor for ShellExecutor<'_> {
                 ensure_non_negative(measurement.time_user - spawning_time.time_user);
             measurement.time_system =
                 ensure_non_negative(measurement.time_system - spawning_time.time_system);
+            measurement.time_cpu = measurement.time_user + measurement.time_system;
         }
 
         Ok(measurement)
@@ -278,6 +279,7 @@ impl Executor for ShellExecutor<'_> {
 
         self.shell_spawning_time = Some(Measurement {
             time_wall_clock: measurements.time_wall_clock_mean(),
+            time_cpu: measurements.time_user_mean() + measurements.time_system_mean(),
             time_user: measurements.time_user_mean(),
             time_system: measurements.time_system_mean(),
             memory_peak_resident: None,
@@ -337,6 +339,7 @@ impl Executor for MockExecutor {
 
         Ok(Measurement {
             time_wall_clock: Self::extract_time(command.get_command_line()),
+            time_cpu: Time::zero(),
             time_user: Time::zero(),
             time_system: Time::zero(),
             memory_peak_resident: Some(Information::zero()),

src/benchmark/measurement.rs

@@ -2,7 +2,8 @@ use std::process::ExitStatus;
 
 use serde::Serialize;
 
-use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation};
+use crate::metric::Metric;
+use crate::quantity::statistics::{mean, modified_zscores_f64};
 use crate::quantity::{serialize_information, serialize_time, Information, Time};
 use crate::util::exit_code::extract_exit_code;
 
@@ -77,6 +78,10 @@ pub struct Measurement {
     #[serde(serialize_with = "serialize_time")]
     pub time_wall_clock: Time,
 
+    /// Total CPU time (user and kernel mode)
+    #[serde(serialize_with = "serialize_time")]
+    pub time_cpu: Time,
+
     /// Time spent in user mode
     #[serde(serialize_with = "serialize_time")]
     pub time_user: Time,
@@ -131,35 +136,14 @@ impl Measurements {
         mean(self.wall_clock_times())
     }
 
-    /// The standard deviation of all wall clock times. Not available if only one run has been performed
-    pub fn stddev(&self) -> Option<Time> {
-        let times: Vec<_> = self.wall_clock_times().collect(); // TODO: Avoid collecting
-
-        if times.len() < 2 {
-            None
-        } else {
-            Some(standard_deviation(times))
-        }
-    }
-
-    /// The median wall clock time
-    pub fn median(&self) -> Time {
-        median(self.wall_clock_times())
-    }
-
-    /// The minimum wall clock time
-    pub fn min(&self) -> Time {
-        min(self.wall_clock_times())
-    }
-
-    /// The maximum wall clock time
-    pub fn max(&self) -> Time {
-        max(self.wall_clock_times())
-    }
-
-    /// Compute modified Z-scores for the wall clock times
-    pub fn modified_zscores(&self) -> Vec<f64> {
-        modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
+    /// Compute modified Z-scores for a metric validated to be available in every run.
+    pub fn modified_zscores(&self, metric: Metric) -> Vec<f64> {
+        let values: Vec<_> = self
+            .measurements
+            .iter()
+            .map(|m| metric.value(m).expect("Validated metric is available"))
+            .collect();
+        modified_zscores_f64(&values)
     }
 
     /// The average user time
@@ -172,3 +156,28 @@ impl Measurements {
         mean(self.measurements.iter().map(|m| m.time_system))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::outlier_detection::OUTLIER_THRESHOLD;
+    use crate::quantity::{byte, second};
+
+    #[test]
+    fn outliers_follow_the_selected_metric() {
+        let measurements = Measurements::new(
+            [(1.0, 100.0), (1.0, 1.0), (1.0, 1.0), (100.0, 1.0)]
+                .iter()
+                .copied()
+                .map(|(time, memory)| Measurement {
+                    time_wall_clock: Time::new::<second>(time),
+                    memory_peak_resident: Some(Information::new::<byte>(memory)),
+                    ..Measurement::default()
+                })
+                .collect(),
+        );
+
+        assert!(measurements.modified_zscores(Metric::TimeWallClock)[0] < OUTLIER_THRESHOLD);
+        assert!(measurements.modified_zscores(Metric::MemoryPeakResident)[0] > OUTLIER_THRESHOLD);
+    }
+}

src/benchmark/mod.rs

@@ -1,7 +1,6 @@
 pub mod benchmark_result;
 pub mod executor;
 pub mod measurement;
-pub mod relative_speed;
 pub mod scheduler;
 
 use std::cmp;
@@ -18,14 +17,15 @@ use crate::options::{
 use crate::outlier_detection::OUTLIER_THRESHOLD;
 use crate::output::console_writeln;
 use crate::output::progress_bar::{
-    finish_initial_measurement, get_progress_bar, start_initial_measurement,
+    finish_initial_measurement, get_progress_bar, set_benchmark_header, start_initial_measurement,
 };
+use crate::output::report;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
-use crate::quantity::{self, const_time_from_seconds, FormatQuantity, Time, Zero};
+use crate::quantity::{self, const_time_from_seconds, Time, Zero};
 use benchmark_result::BenchmarkResult;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, ensure, Result};
 use colored::*;
 
 use self::executor::Executor;
@@ -155,20 +155,21 @@ impl<'a> Benchmark<'a> {
         self.run_intermediate_command(command, error_output, output_policy, iteration)
     }
 
-    /// Run the benchmark for a single command
-    pub fn run(&self) -> Result<BenchmarkResult> {
-        if self.options.output_style != OutputStyleOption::Disabled {
-            console_writeln!(
-                io::stdout(),
-                "{}{}: {}",
-                "Benchmark ".bold(),
-                (self.number + 1).to_string().bold(),
-                self.command.get_name_with_unused_parameters(),
-            )?;
+    fn validate_measurement(&self, measurement: &Measurement) -> Result<()> {
+        if self.options.skip_unavailable_metrics {
+            return Ok(());
         }
+        for selection in &self.options.metrics {
+            ensure!(selection.metric.value(measurement).is_some(),
+                "Metric '{}' is unavailable for '{}'. Check platform support and hardware-counter permissions.",
+                selection.metric.name(), self.command.get_command_line());
+        }
+        Ok(())
+    }
 
+    /// Run the benchmark for a single command
+    pub fn run(&self, reference: Option<&BenchmarkResult>) -> Result<BenchmarkResult> {
         let mut measurements = Measurements::default();
-        let mut all_succeeded = true;
 
         let output_policy = &self.options.command_output_policies[self.number];
 
@@ -213,6 +214,15 @@ impl<'a> Benchmark<'a> {
 
         self.run_setup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
 
+        let progress_header = format!(
+            "{}: {}",
+            format!("Benchmark {}", self.number + 1).bold(),
+            self.command
+                .get_name_with_unused_parameters()
+                .white()
+                .bold()
+        );
+
         // Warmup phase
         if self.options.warmup_count > 0 {
             let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
@@ -225,6 +235,10 @@ impl<'a> Benchmark<'a> {
                 None
             };
 
+            if let Some(bar) = &progress_bar {
+                set_benchmark_header(bar, progress_header.clone());
+            }
+
             for i in 0..self.options.warmup_count {
                 let warmup_iteration = BenchmarkIteration::Warmup(i);
                 let _ = run_preparation_command(warmup_iteration)?;
@@ -259,6 +273,10 @@ impl<'a> Benchmark<'a> {
             None
         };
 
+        if let Some(bar) = &progress_bar {
+            set_benchmark_header(bar, progress_header);
+        }
+
         let benchmark_iteration = BenchmarkIteration::Benchmark(0);
         let preparation_result = run_preparation_command(benchmark_iteration)?;
         let preparation_overhead = preparation_result.map_or(Time::zero(), |res| {
@@ -275,18 +293,14 @@ impl<'a> Benchmark<'a> {
             None,
             output_policy,
         )?;
-        let success = measurement.exit_status.success();
+        self.validate_measurement(&measurement)?;
+        let mut all_succeeded = measurement.exit_status.success();
 
         if let Some(bar) = progress_bar.as_ref() {
-            let time_unit = self
-                .options
-                .time_unit
-                .unwrap_or(measurement.time_wall_clock.suitable_unit());
-            let estimate = measurement.time_wall_clock.format(time_unit);
-            finish_initial_measurement(
-                bar,
-                format!("Current estimate: {}", estimate.to_string().green()),
-            );
+            let primary = self.options.metrics[0];
+            let value = primary.metric.value(&measurement).unwrap();
+            let estimate = primary.display_unit(value).format(value);
+            finish_initial_measurement(bar, format!("Current estimate: {}", estimate.green()));
         }
 
         let conclusion_result = run_conclusion_command(benchmark_iteration)?;
@@ -316,10 +330,10 @@ impl<'a> Benchmark<'a> {
         let count_remaining = count - 1;
 
         // Save the first result
+        let primary = self.options.metrics[0];
+        let mut primary_total = primary.metric.value(&measurement).unwrap();
         measurements.push(measurement);
 
-        all_succeeded = all_succeeded && success;
-
         // Re-configure the progress bar
         if let Some(bar) = progress_bar.as_ref() {
             bar.set_length(count);
@@ -330,18 +344,12 @@ impl<'a> Benchmark<'a> {
         for i in 0..count_remaining {
             let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
 
-            let msg = {
-                let t_wall_clock_mean = measurements.time_wall_clock_mean();
-                let time_unit = self
-                    .options
-                    .time_unit
-                    .unwrap_or(t_wall_clock_mean.suitable_unit());
-                let mean = t_wall_clock_mean.format(time_unit);
-                format!("Current estimate: {}", mean.to_string().green())
-            };
-
             if let Some(bar) = progress_bar.as_ref() {
-                bar.set_message(msg.to_owned())
+                let mean = primary_total / measurements.len() as f64;
+                bar.set_message(format!(
+                    "Current estimate: {}",
+                    primary.display_unit(mean).format(mean).green()
+                ));
             }
 
             run_preparation_command(benchmark_iteration)?;
@@ -352,7 +360,9 @@ impl<'a> Benchmark<'a> {
                 None,
                 output_policy,
             )?;
+            self.validate_measurement(&measurement)?;
             let success = measurement.exit_status.success();
+            primary_total += primary.metric.value(&measurement).unwrap();
             measurements.push(measurement);
 
             all_succeeded = all_succeeded && success;
@@ -368,63 +378,27 @@ impl<'a> Benchmark<'a> {
             bar.finish_and_clear()
         }
 
-        // Formatting and console output
-        let t_wall_clock_mean = measurements.time_wall_clock_mean();
-        let time_unit = self
-            .options
-            .time_unit
-            .unwrap_or(t_wall_clock_mean.suitable_unit());
-        let mean_str = t_wall_clock_mean.format(time_unit);
-        let min_str = measurements.min().format(time_unit);
-        let max_str = measurements.max().format(time_unit);
-        let num_str = format!("{num_runs} runs", num_runs = measurements.len());
-
-        let user_str = measurements.time_user_mean().format(time_unit);
-        let system_str = measurements.time_system_mean().format(time_unit);
-
         if self.options.output_style != OutputStyleOption::Disabled {
-            let mut stdout = io::stdout().lock();
-            if measurements.len() == 1 {
-                console_writeln!(
-                    stdout,
-                    "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
-                    "abs".green().bold(),
-                    mean_str.green().bold(),
-                    "        ", // alignment
-                    user_str.blue(),
-                    system_str.blue()
-                )?;
-            } else {
-                let stddev_str = measurements.stddev().unwrap().format(time_unit);
-
-                console_writeln!(
-                    stdout,
-                    "  Time ({} ± {}):     {:>8} ± {:>8}    [User: {}, System: {}]",
-                    "mean".green().bold(),
-                    "σ".green(),
-                    mean_str.green().bold(),
-                    stddev_str.green(),
-                    user_str.blue(),
-                    system_str.blue()
-                )?;
-
-                console_writeln!(
-                    stdout,
-                    "  Range ({} … {}):   {:>8} … {:>8}    {}",
-                    "min".cyan(),
-                    "max".purple(),
-                    min_str.cyan(),
-                    max_str.purple(),
-                    num_str.dimmed()
-                )?;
-            }
+            report::print(
+                self.number,
+                &self.command.get_name_with_unused_parameters(),
+                &measurements,
+                &self.options.metrics,
+                self.options.skip_unavailable_metrics,
+                reference,
+            )?;
         }
 
         // Warnings
         let mut warnings = vec![];
 
         // Check execution time
-        if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
+        if self
+            .options
+            .metrics
+            .iter()
+            .any(|m| m.metric == crate::metric::Metric::TimeWallClock)
+            && matches!(self.options.executor_kind, ExecutorKind::Shell(_))
             && measurements
                 .wall_clock_times()
                 .any(|t| t < MIN_EXECUTION_TIME)
@@ -438,7 +412,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = measurements.modified_zscores();
+        let scores = measurements.modified_zscores(primary.metric);
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -452,12 +426,11 @@ impl<'a> Benchmark<'a> {
         };
 
         if scores[0] > OUTLIER_THRESHOLD {
-            warnings.push(Warnings::SlowInitialRun(
-                measurements.wall_clock_times().next().unwrap(),
+            warnings.push(Warnings::InitialRunOutlier(
+                primary,
+                primary.metric.value(&measurements.measurements[0]).unwrap(),
                 outlier_warning_options,
             ));
-        } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
-            warnings.push(Warnings::OutliersDetected(outlier_warning_options));
         }
 
         if !warnings.is_empty() {

src/benchmark/relative_speed.rs

@@ -1,237 +0,0 @@
-use std::cmp::Ordering;
-
-use super::benchmark_result::BenchmarkResult;
-use crate::{
-    options::SortOrder,
-    quantity::{self, Ratio, Time, Zero},
-};
-
-#[derive(Debug)]
-pub struct BenchmarkResultWithRelativeSpeed<'a> {
-    pub result: &'a BenchmarkResult,
-    pub relative_speed: f64,
-    pub relative_speed_stddev: Option<f64>,
-    pub is_reference: bool,
-    // Less means faster
-    pub relative_ordering: Ordering,
-}
-
-impl BenchmarkResultWithRelativeSpeed<'_> {
-    pub fn reference_label(&self) -> &'static str {
-        if self.is_reference {
-            " (reference)"
-        } else {
-            match self.relative_ordering {
-                Ordering::Less => " (faster)",
-                Ordering::Equal => " (same speed)",
-                Ordering::Greater => " (slower)",
-            }
-        }
-    }
-}
-
-pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
-    l.mean_wall_clock_time()
-        .partial_cmp(&r.mean_wall_clock_time())
-        .unwrap_or(Ordering::Equal)
-}
-
-pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
-    results
-        .iter()
-        .min_by(|&l, &r| compare_mean_time(l, r))
-        .expect("at least one benchmark result")
-}
-
-fn compute_relative_speeds<'a>(
-    results: &'a [BenchmarkResult],
-    reference: &'a BenchmarkResult,
-    sort_order: SortOrder,
-) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    let mut results: Vec<_> = results
-        .iter()
-        .map(|result| {
-            // Separate benchmarks can have identical names and measurements.
-            let is_reference = std::ptr::eq(result, reference);
-            let relative_ordering = compare_mean_time(result, reference);
-
-            if result.mean_wall_clock_time() == Time::zero()
-                || reference.mean_wall_clock_time() == Time::zero()
-            {
-                return BenchmarkResultWithRelativeSpeed {
-                    result,
-                    relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
-                    relative_speed_stddev: None,
-                    is_reference,
-                    relative_ordering,
-                };
-            }
-
-            let ratio = match relative_ordering {
-                Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
-                Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
-                Ordering::Greater => {
-                    result.mean_wall_clock_time() / reference.mean_wall_clock_time()
-                }
-            };
-
-            // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
-            // Covariance assumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (
-                result.measurements.stddev(),
-                reference.measurements.stddev(),
-            ) {
-                (Some(result_stddev), Some(reference_stddev)) => Some(
-                    ratio
-                        * ((result_stddev / result.mean_wall_clock_time())
-                            .powi(uom::typenum::P2::new())
-                            + (reference_stddev / reference.mean_wall_clock_time())
-                                .powi(uom::typenum::P2::new()))
-                        .sqrt(),
-                ),
-                _ => None,
-            };
-
-            BenchmarkResultWithRelativeSpeed {
-                result,
-                relative_speed: ratio.get::<quantity::ratio>(),
-                relative_speed_stddev: ratio_stddev.map(|r| r.get::<quantity::ratio>()),
-                is_reference,
-                relative_ordering,
-            }
-        })
-        .collect();
-
-    match sort_order {
-        SortOrder::Command => {}
-        SortOrder::MeanTime => {
-            results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));
-        }
-    }
-
-    results
-}
-
-pub fn compute_with_check_from_reference<'a>(
-    results: &'a [BenchmarkResult],
-    reference: &'a BenchmarkResult,
-    sort_order: SortOrder,
-) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
-    if fastest_of(results).mean_wall_clock_time() == Time::zero()
-        || reference.mean_wall_clock_time() == Time::zero()
-    {
-        return None;
-    }
-
-    Some(compute_relative_speeds(results, reference, sort_order))
-}
-
-pub fn compute_with_check(
-    results: &[BenchmarkResult],
-    sort_order: SortOrder,
-) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
-    let fastest = fastest_of(results);
-
-    if fastest.mean_wall_clock_time() == Time::zero() {
-        return None;
-    }
-
-    Some(compute_relative_speeds(results, fastest, sort_order))
-}
-
-/// Compute relative speeds against the given reference, or the fastest result.
-pub fn compute<'a>(
-    results: &'a [BenchmarkResult],
-    sort_order: SortOrder,
-    reference: Option<&'a BenchmarkResult>,
-) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
-    let reference = reference.unwrap_or_else(|| fastest_of(results));
-    compute_relative_speeds(results, reference, sort_order)
-}
-
-#[cfg(test)]
-fn create_result(name: &str, mean: f64) -> BenchmarkResult {
-    use std::collections::BTreeMap;
-
-    use crate::benchmark::measurement::{Measurement, Measurements};
-    use crate::quantity::second;
-
-    BenchmarkResult {
-        command: name.into(),
-        display_name: name.into(),
-        name: None,
-        measurements: Measurements {
-            measurements: vec![Measurement {
-                time_wall_clock: Time::new::<second>(mean),
-                time_user: Time::new::<second>(mean),
-                ..Default::default()
-            }],
-        },
-        parameters: BTreeMap::new(),
-    }
-}
-
-#[test]
-fn test_compute_relative_speed() {
-    use approx::assert_relative_eq;
-
-    let results = vec![
-        create_result("cmd1", 3.0),
-        create_result("cmd2", 2.0),
-        create_result("cmd3", 5.0),
-    ];
-
-    let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();
-
-    assert_relative_eq!(1.5, annotated_results[0].relative_speed);
-    assert_relative_eq!(1.0, annotated_results[1].relative_speed);
-    assert_relative_eq!(2.5, annotated_results[2].relative_speed);
-}
-
-#[test]
-fn test_compute_relative_speed_with_reference() {
-    use approx::assert_relative_eq;
-
-    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
-    let reference = create_result("cmd2", 4.0);
-
-    let annotated_results =
-        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
-
-    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
-    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
-}
-
-#[test]
-fn test_compute_relative_speed_for_zero_times() {
-    let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
-
-    let annotated_results = compute_with_check(&results, SortOrder::Command);
-
-    assert!(annotated_results.is_none());
-}
-
-#[test]
-fn reference_identity_distinguishes_equal_results() {
-    let results = vec![create_result("same", 1.0), create_result("same", 1.0)];
-    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
-
-    assert!(entries[0].is_reference);
-    assert!(!entries[1].is_reference);
-}
-
-#[test]
-fn reference_ratios_handle_zero_times() {
-    let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
-    for result in &mut results {
-        let measurement = result.measurements.measurements[0].clone();
-        result.measurements.measurements.push(measurement);
-    }
-    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
-    assert_eq!(entries[1].relative_speed, f64::INFINITY);
-
-    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
-    assert_eq!(entries[0].relative_speed, f64::INFINITY);
-    assert_eq!(entries[0].relative_speed_stddev, None);
-    assert_eq!(entries[1].relative_speed, 1.0);
-}

src/benchmark/scheduler.rs

@@ -1,14 +1,10 @@
 use super::benchmark_result::BenchmarkResult;
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
-use super::{relative_speed, Benchmark};
-use colored::*;
-use std::cmp::Ordering;
-use std::io::{self, Write};
+use super::Benchmark;
 
-use crate::command::{Command, Commands};
+use crate::command::Commands;
 use crate::export::ExportManager;
-use crate::options::{ExecutorKind, Options, OutputStyleOption, SortOrder};
-use crate::output::console_writeln;
+use crate::options::{ExecutorKind, Options};
 
 use anyhow::Result;
 
@@ -40,133 +36,23 @@ impl<'a> Scheduler<'a> {
             ExecutorKind::Shell(ref shell) => Box::new(ShellExecutor::new(shell, self.options)),
         };
 
-        let reference = self
-            .options
-            .reference_command
-            .as_ref()
-            .map(|cmd| Command::new(self.options.reference_name.as_deref(), cmd));
-
         executor.calibrate()?;
 
-        for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
-            self.results
-                .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
+        for (number, cmd) in self.commands.iter().enumerate() {
+            self.results.push(
+                Benchmark::new(number, cmd, self.options, &*executor).run(self.results.first())?,
+            );
 
             // We export results after each individual benchmark, because
             // we would risk losing them if a later benchmark fails.
-            self.export_manager
-                .write_results(&self.results, true, self.options.reference_index)?;
-        }
-
-        Ok(())
-    }
-
-    pub fn print_relative_speed_comparison(&self) -> Result<()> {
-        if self.options.output_style == OutputStyleOption::Disabled {
-            return Ok(());
-        }
-
-        if self.results.len() < 2 {
-            return Ok(());
-        }
-
-        let reference = self
-            .options
-            .reference_index
-            .map(|index| &self.results[index])
-            .unwrap_or_else(|| relative_speed::fastest_of(&self.results));
-
-        if let Some(annotated_results) = relative_speed::compute_with_check_from_reference(
-            &self.results,
-            reference,
-            self.options.sort_order_speed_comparison,
-        ) {
-            let mut stdout = io::stdout().lock();
-            match self.options.sort_order_speed_comparison {
-                SortOrder::MeanTime => {
-                    console_writeln!(stdout, "{}", "Summary".bold())?;
-
-                    let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
-                    let others = annotated_results.iter().filter(|r| !r.is_reference);
-
-                    console_writeln!(stdout, "  {} ran", reference.result.display_name.cyan())?;
-
-                    for item in others {
-                        let stddev = if let Some(stddev) = item.relative_speed_stddev {
-                            format!(" ± {}", format!("{:.2}", stddev).green())
-                        } else {
-                            "".into()
-                        };
-                        let comparator = match item.relative_ordering {
-                            Ordering::Less => format!(
-                                "{}{} times slower than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                            Ordering::Greater => format!(
-                                "{}{} times faster than",
-                                format!("{:8.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                            Ordering::Equal => format!(
-                                "    As fast ({}{}) as",
-                                format!("{:.2}", item.relative_speed).bold().green(),
-                                stddev
-                            ),
-                        };
-                        console_writeln!(
-                            stdout,
-                            "{} {}",
-                            comparator,
-                            &item.result.display_name.magenta()
-                        )?;
-                    }
-                }
-                SortOrder::Command => {
-                    console_writeln!(stdout, "{}", "Relative speed comparison".bold())?;
-
-                    for item in annotated_results {
-                        let relationship = if self.options.reference_index.is_some() {
-                            item.reference_label()
-                        } else {
-                            ""
-                        };
-                        console_writeln!(
-                            stdout,
-                            "  {}{}  {}{}",
-                            format!("{:10.2}", item.relative_speed).bold().green(),
-                            if item.is_reference {
-                                "        ".into()
-                            } else if let Some(stddev) = item.relative_speed_stddev {
-                                format!(" ± {}", format!("{stddev:5.2}").green())
-                            } else {
-                                "        ".into()
-                            },
-                            item.result.display_name,
-                            relationship,
-                        )?;
-                    }
-                }
-            }
-        } else {
-            console_writeln!(
-                io::stderr(),
-                "{}: The benchmark comparison could not be computed as some benchmark times are zero. \
-                 This could be caused by background interference during the initial calibration phase \
-                 of hyperfine, in combination with very fast commands (faster than a few milliseconds). \
-                 Try to re-run the benchmark on a quiet system. If you did not do so already, try the \
-                 --shell=none/-N option. If it does not help either, you command is most likely too fast \
-                 to be accurately benchmarked by hyperfine.",
-                 "Note".bold().red()
-            )?;
+            self.export_manager.write_results(&self.results, true)?;
         }
 
         Ok(())
     }
 
     pub fn final_export(&self) -> Result<()> {
-        self.export_manager
-            .write_results(&self.results, false, self.options.reference_index)
+        self.export_manager.write_results(&self.results, false)
     }
 }
 
@@ -183,11 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
     assert_eq!(options.executor_kind, ExecutorKind::Mock(None));
 
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
-    let export_manager = ExportManager::from_cli_arguments(
-        &cli_arguments,
-        options.time_unit,
-        options.sort_order_exports,
-    )?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
 
     options.validate_against_command_list(&commands)?;
 
@@ -199,12 +81,15 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
 
 #[test]
 fn scheduler_basic() -> Result<()> {
-    insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#"
+    insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @"
     - command: sleep 0.123
       measurements:
         - time_wall_clock:
             value: 0.123
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -218,6 +103,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.123
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -233,6 +121,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.456
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -246,6 +137,9 @@ fn scheduler_basic() -> Result<()> {
         - time_wall_clock:
             value: 0.456
             unit: second
+          time_cpu:
+            value: 0
+            unit: second
           time_user:
             value: 0
             unit: second
@@ -256,7 +150,7 @@ fn scheduler_basic() -> Result<()> {
             value: 0
             unit: byte
           exit_code: 0
-    "#);
+    ");
 
     Ok(())
 }