Collect benchmark runs in Measurements

src/benchmark/executor.rs

@@ -9,10 +9,9 @@ use crate::options::{
 use crate::output::progress_bar::get_progress_bar;
 use crate::timer::execute_and_measure;
 use crate::util::randomized_environment_offset;
-use crate::util::statistics::mean;
 use crate::util::units::Second;
 
-use super::measurement::Measurement;
+use super::measurement::{Measurement, Measurements};
 
 use anyhow::{bail, Context, Result};
 
@@ -225,9 +224,7 @@ impl Executor for ShellExecutor<'_> {
             None
         };
 
-        let mut times_real: Vec<Second> = vec![];
-        let mut times_user: Vec<Second> = vec![];
-        let mut times_system: Vec<Second> = vec![];
+        let mut measurements = Measurements::default();
 
         for _ in 0..COUNT {
             // Just run the shell without any command
@@ -252,9 +249,7 @@ impl Executor for ShellExecutor<'_> {
                     );
                 }
                 Ok(r) => {
-                    times_real.push(r.time_wall_clock);
-                    times_user.push(r.time_user);
-                    times_system.push(r.time_system);
+                    measurements.push(r);
                 }
             }
 
@@ -268,9 +263,9 @@ impl Executor for ShellExecutor<'_> {
         }
 
         self.shell_spawning_time = Some(Measurement {
-            time_wall_clock: mean(&times_real),
-            time_user: mean(&times_user),
-            time_system: mean(&times_system),
+            time_wall_clock: measurements.time_wall_clock_mean(),
+            time_user: measurements.time_user_mean(),
+            time_system: measurements.time_system_mean(),
             peak_memory_usage: 0,
             exit_status: ExitStatus::default(),
         });

src/benchmark/measurement.rs

@@ -1,5 +1,7 @@
 use std::process::ExitStatus;
 
+use crate::outlier_detection::modified_zscores;
+use crate::util::statistics::{max, mean, median, min, standard_deviation};
 use crate::util::units::Second;
 
 /// Performance measurements and exit status from running a single command
@@ -20,3 +22,72 @@ pub struct Measurement {
     /// The exit status of the process
     pub exit_status: ExitStatus,
 }
+
+/// Performance measurements and exit statuses for all runs of a command.
+#[derive(Debug, Default, Clone)]
+pub struct Measurements {
+    pub measurements: Vec<Measurement>,
+}
+
+impl Measurements {
+    pub fn len(&self) -> usize {
+        self.measurements.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.measurements.is_empty()
+    }
+
+    pub fn push(&mut self, measurement: Measurement) {
+        self.measurements.push(measurement);
+    }
+
+    pub fn wall_clock_times(&self) -> impl Iterator<Item = Second> + '_ {
+        self.measurements.iter().map(|m| m.time_wall_clock)
+    }
+
+    /// The average wall clock time.
+    pub fn time_wall_clock_mean(&self) -> Second {
+        mean(self.wall_clock_times())
+    }
+
+    /// The standard deviation of wall clock times, if at least two runs were measured.
+    pub fn stddev(&self) -> Option<Second> {
+        if self.len() < 2 {
+            None
+        } else {
+            let times: Vec<_> = self.wall_clock_times().collect();
+            Some(standard_deviation(&times, self.time_wall_clock_mean()))
+        }
+    }
+
+    /// The median wall clock time.
+    pub fn median(&self) -> Second {
+        median(&self.wall_clock_times().collect::<Vec<_>>())
+    }
+
+    /// The minimum wall clock time.
+    pub fn min(&self) -> Second {
+        min(self.wall_clock_times())
+    }
+
+    /// The maximum wall clock time.
+    pub fn max(&self) -> Second {
+        max(self.wall_clock_times())
+    }
+
+    /// Compute modified Z-scores for the wall clock times.
+    pub fn modified_zscores(&self) -> Vec<f64> {
+        modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
+    }
+
+    /// The average user time.
+    pub fn time_user_mean(&self) -> Second {
+        mean(self.measurements.iter().map(|m| m.time_user))
+    }
+
+    /// The average system time.
+    pub fn time_system_mean(&self) -> Second {
+        mean(self.measurements.iter().map(|m| m.time_system))
+    }
+}

src/benchmark/mod.rs

@@ -12,17 +12,16 @@ use crate::command::Command;
 use crate::options::{
     CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
 };
-use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
+use crate::outlier_detection::OUTLIER_THRESHOLD;
 use crate::output::console_writeln;
 use crate::output::format::{format_duration, format_duration_unit};
 use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
 use crate::util::exit_code::extract_exit_code;
-use crate::util::statistics::{max, mean, median, min, standard_deviation};
 use crate::util::units::Second;
 use benchmark_result::BenchmarkResult;
-use measurement::Measurement;
+use measurement::{Measurement, Measurements};
 
 use anyhow::{anyhow, Result};
 use colored::*;
@@ -149,11 +148,7 @@ impl<'a> Benchmark<'a> {
             )?;
         }
 
-        let mut times_real: Vec<Second> = vec![];
-        let mut times_user: Vec<Second> = vec![];
-        let mut times_system: Vec<Second> = vec![];
-        let mut memory_usage_byte: Vec<u64> = vec![];
-        let mut exit_codes: Vec<Option<i32>> = vec![];
+        let mut measurements = Measurements::default();
         let mut all_succeeded = true;
 
         let output_policy = &self.options.command_output_policies[self.number];
@@ -280,11 +275,7 @@ impl<'a> Benchmark<'a> {
         let count_remaining = count - 1;
 
         // Save the first result
-        times_real.push(res.time_wall_clock);
-        times_user.push(res.time_user);
-        times_system.push(res.time_system);
-        memory_usage_byte.push(res.peak_memory_usage);
-        exit_codes.push(extract_exit_code(res.exit_status));
+        measurements.push(res);
 
         all_succeeded = all_succeeded && success;
 
@@ -301,7 +292,8 @@ impl<'a> Benchmark<'a> {
             run_preparation_command()?;
 
             let msg = {
-                let mean = format_duration(mean(&times_real), self.options.time_unit);
+                let mean =
+                    format_duration(measurements.time_wall_clock_mean(), self.options.time_unit);
                 format!("Current estimate: {}", mean.to_string().green())
             };
 
@@ -317,11 +309,7 @@ impl<'a> Benchmark<'a> {
             )?;
             let success = res.exit_status.success();
 
-            times_real.push(res.time_wall_clock);
-            times_user.push(res.time_user);
-            times_system.push(res.time_system);
-            memory_usage_byte.push(res.peak_memory_usage);
-            exit_codes.push(extract_exit_code(res.exit_status));
+            measurements.push(res);
 
             all_succeeded = all_succeeded && success;
 
@@ -337,19 +325,15 @@ impl<'a> Benchmark<'a> {
         }
 
         // Compute statistical quantities
-        let t_num = times_real.len();
-        let t_mean = mean(&times_real);
-        let t_stddev = if times_real.len() > 1 {
-            Some(standard_deviation(&times_real, t_mean))
-        } else {
-            None
-        };
-        let t_median = median(&times_real);
-        let t_min = min(times_real.iter().copied());
-        let t_max = max(times_real.iter().copied());
+        let t_num = measurements.len();
+        let t_mean = measurements.time_wall_clock_mean();
+        let t_stddev = measurements.stddev();
+        let t_median = measurements.median();
+        let t_min = measurements.min();
+        let t_max = measurements.max();
 
-        let user_mean = mean(&times_user);
-        let system_mean = mean(&times_system);
+        let user_mean = measurements.time_user_mean();
+        let system_mean = measurements.time_system_mean();
 
         // Formatting and console output
         let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
@@ -362,7 +346,7 @@ impl<'a> Benchmark<'a> {
 
         if self.options.output_style != OutputStyleOption::Disabled {
             let mut stdout = io::stdout().lock();
-            if times_real.len() == 1 {
+            if measurements.len() == 1 {
                 console_writeln!(
                     stdout,
                     "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
@@ -403,7 +387,9 @@ impl<'a> Benchmark<'a> {
 
         // Check execution time
         if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
-            && times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
+            && measurements
+                .wall_clock_times()
+                .any(|t| t < MIN_EXECUTION_TIME)
         {
             warnings.push(Warnings::FastExecutionTime);
         }
@@ -414,7 +400,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = modified_zscores(&times_real);
+        let scores = measurements.modified_zscores();
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -429,7 +415,7 @@ impl<'a> Benchmark<'a> {
 
         if scores[0] > OUTLIER_THRESHOLD {
             warnings.push(Warnings::SlowInitialRun(
-                times_real[0],
+                measurements.wall_clock_times().next().unwrap(),
                 outlier_warning_options,
             ));
         } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
@@ -461,9 +447,19 @@ impl<'a> Benchmark<'a> {
             system: system_mean,
             min: t_min,
             max: t_max,
-            times: Some(times_real),
-            memory_usage_byte: Some(memory_usage_byte),
-            exit_codes,
+            times: Some(measurements.wall_clock_times().collect()),
+            memory_usage_byte: Some(
+                measurements
+                    .measurements
+                    .iter()
+                    .map(|m| m.peak_memory_usage)
+                    .collect(),
+            ),
+            exit_codes: measurements
+                .measurements
+                .iter()
+                .map(|m| extract_exit_code(m.exit_status))
+                .collect(),
             parameters: self
                 .command
                 .get_parameters()

src/util/statistics.rs

@@ -17,8 +17,14 @@ pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
 }
 
 /// Arithmetic mean, summing measurements in their original order.
-pub fn mean(values: &[f64]) -> f64 {
-    values.iter().fold(0.0, |sum, value| sum + value) / values.len() as f64
+pub fn mean(values: impl IntoIterator<Item = f64>) -> f64 {
+    let mut sum = 0.0;
+    let mut count = 0usize;
+    for value in values {
+        sum += value;
+        count += 1;
+    }
+    sum / count as f64
 }
 
 /// Median without modifying the input sample.
@@ -86,11 +92,11 @@ mod tests {
 
     #[test]
     fn test_mean() {
-        assert_eq!(1.0, mean(&[1.0]));
-        assert_relative_eq!(2.0, mean(&[1.0, 3.0]));
+        assert_eq!(1.0, mean([1.0]));
+        assert_relative_eq!(2.0, mean([1.0, 3.0]));
 
         let values = [0.1, 0.2, 0.6];
-        assert_relative_eq!(mean(&values), 0.3);
+        assert_relative_eq!(mean(values), 0.3);
     }
 
     #[test]
@@ -108,6 +114,6 @@ mod tests {
     #[test]
     fn test_standard_deviation() {
         let values = [0.1, 0.2, 0.3];
-        assert_relative_eq!(standard_deviation(&values, mean(&values)), 0.1);
+        assert_relative_eq!(standard_deviation(&values, mean(values)), 0.1);
     }
 }