Simplify run creation

src/benchmark/benchmark_result.rs

@@ -27,6 +27,87 @@ pub struct Run {
     pub exit_code: Option<i32>,
 }
 
+#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+pub struct Runs {
+    pub runs: Vec<Run>,
+}
+
+impl Runs {
+    pub fn new(runs: Vec<Run>) -> Self {
+        Self { runs }
+    }
+
+    pub fn len(&self) -> usize {
+        self.runs.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.runs.is_empty()
+    }
+
+    pub fn push(&mut self, run: Run) {
+        self.runs.push(run);
+    }
+
+    pub fn wall_clock_times(&self) -> Vec<Second> {
+        self.runs.iter().map(|run| run.wall_clock_time).collect()
+    }
+
+    /// The average wall clock time
+    pub fn mean(&self) -> Second {
+        mean(&self.wall_clock_times())
+    }
+
+    /// The standard deviation of all wall clock times. Not available if only one run has been performed
+    pub fn stddev(&self) -> Option<Second> {
+        let times = self.wall_clock_times();
+
+        let t_mean = mean(&times);
+        if times.len() > 1 {
+            Some(standard_deviation(&times, Some(t_mean)))
+        } else {
+            None
+        }
+    }
+
+    /// The median wall clock time
+    pub fn median(&self) -> Second {
+        median(&self.wall_clock_times())
+    }
+
+    /// The minimum wall clock time
+    pub fn min(&self) -> Second {
+        min(&self.wall_clock_times())
+    }
+
+    /// The maximum wall clock time
+    pub fn max(&self) -> Second {
+        max(&self.wall_clock_times())
+    }
+
+    /// The average user time
+    pub fn user_mean(&self) -> Second {
+        mean(
+            &self
+                .runs
+                .iter()
+                .map(|run| run.user_time)
+                .collect::<Vec<_>>(),
+        )
+    }
+
+    /// The average system time
+    pub fn system_mean(&self) -> Second {
+        mean(
+            &self
+                .runs
+                .iter()
+                .map(|run| run.system_time)
+                .collect::<Vec<_>>(),
+        )
+    }
+}
+
 /// Parameter value and whether it was used in the command line template
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
 pub struct Parameter {
@@ -50,7 +131,8 @@ pub struct BenchmarkResult {
     pub command: String,
 
     /// Performance metrics and exit codes for each run
-    pub runs: Vec<Run>,
+    #[serde(flatten)]
+    pub runs: Runs,
 
     /// Parameter values for this benchmark
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
@@ -58,60 +140,32 @@ pub struct BenchmarkResult {
 }
 
 impl BenchmarkResult {
-    fn wall_clock_times(&self) -> Vec<Second> {
-        self.runs.iter().map(|run| run.wall_clock_time).collect()
-    }
-
-    /// The average run time
     pub fn mean(&self) -> Second {
-        mean(&self.wall_clock_times())
+        self.runs.mean()
     }
 
-    /// The standard deviation of all run times. Not available if only one run has been performed
     pub fn stddev(&self) -> Option<Second> {
-        let times = self.wall_clock_times();
-
-        let t_mean = mean(&times);
-        if times.len() > 1 {
-            Some(standard_deviation(&times, Some(t_mean)))
-        } else {
-            None
-        }
+        self.runs.stddev()
     }
 
-    /// The median run time
     pub fn median(&self) -> Second {
-        median(&self.wall_clock_times())
+        self.runs.median()
     }
 
-    /// The minimum run time
     pub fn min(&self) -> Second {
-        min(&self.wall_clock_times())
+        self.runs.min()
     }
 
-    /// The maximum run time
     pub fn max(&self) -> Second {
-        max(&self.wall_clock_times())
+        self.runs.max()
     }
 
     pub fn user_mean(&self) -> Second {
-        mean(
-            &self
-                .runs
-                .iter()
-                .map(|run| run.user_time)
-                .collect::<Vec<_>>(),
-        )
+        self.runs.user_mean()
     }
 
     pub fn system_mean(&self) -> Second {
-        mean(
-            &self
-                .runs
-                .iter()
-                .map(|run| run.system_time)
-                .collect::<Vec<_>>(),
-        )
+        self.runs.system_mean()
     }
 
     /// The full command line of the program that is being benchmarked, possibly including a list of

src/benchmark/mod.rs

@@ -8,7 +8,7 @@ use std::cmp;
 use std::io::{self, Write};
 use std::time::Instant;
 
-use crate::benchmark::benchmark_result::{Parameter, Run};
+use crate::benchmark::benchmark_result::{Parameter, Run, Runs};
 use crate::benchmark::executor::BenchmarkIteration;
 use crate::command::Command;
 use crate::options::{
@@ -21,13 +21,14 @@ use crate::output::progress_bar::{
 };
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
-use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
-use benchmark_result::{BenchmarkResult, Parameter};
+use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero};
+use crate::util::exit_code::extract_exit_code;
+use crate::util::units::Second;
+use benchmark_result::BenchmarkResult;
 use measurement::{Measurement, Measurements};
 
 use anyhow::{anyhow, Result};
 use colored::*;
-use statistical::{mean, standard_deviation};
 
 use self::executor::Executor;
 
@@ -168,11 +169,7 @@ impl<'a> Benchmark<'a> {
             )?;
         }
 
-        let mut times_wall_clock: Vec<Second> = vec![];
-        let mut times_user: Vec<Second> = vec![];
-        let mut times_system: Vec<Second> = vec![];
-        let mut memory_usage_byte: Vec<u64> = vec![];
-        let mut exit_codes: Vec<Option<i32>> = vec![];
+        let mut runs = Runs::default();
         let mut all_succeeded = true;
 
         let output_policy = &self.options.command_output_policies[self.number];
@@ -320,11 +317,13 @@ impl<'a> Benchmark<'a> {
         let count_remaining = count - 1;
 
         // Save the first result
-        times_wall_clock.push(res.time_wall_clock);
-        times_user.push(res.time_user);
-        times_system.push(res.time_system);
-        memory_usage_byte.push(res.memory_usage_byte);
-        exit_codes.push(extract_exit_code(status));
+        runs.push(Run {
+            wall_clock_time: res.time_wall_clock,
+            user_time: res.time_user,
+            system_time: res.time_system,
+            memory_usage_byte: res.memory_usage_byte,
+            exit_code: extract_exit_code(status),
+        });
 
         all_succeeded = all_succeeded && success;
 
@@ -339,7 +338,7 @@ impl<'a> Benchmark<'a> {
             let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1);
 
             let msg = {
-                let mean = format_duration(mean(&times_wall_clock), self.options.time_unit);
+                let mean = format_duration(runs.mean(), self.options.time_unit);
                 format!("Current estimate: {}", mean.to_string().green())
             };
 
@@ -357,11 +356,13 @@ impl<'a> Benchmark<'a> {
             )?;
             let success = res.exit_status.success();
 
-            times_wall_clock.push(res.time_wall_clock);
-            times_user.push(res.time_user);
-            times_system.push(res.time_system);
-            memory_usage_byte.push(res.memory_usage_byte);
-            exit_codes.push(extract_exit_code(status));
+            runs.push(Run {
+                wall_clock_time: res.time_wall_clock,
+                user_time: res.time_user,
+                system_time: res.time_system,
+                memory_usage_byte: res.memory_usage_byte,
+                exit_code: extract_exit_code(status),
+            });
 
             all_succeeded = all_succeeded && success;
 
@@ -376,32 +377,17 @@ impl<'a> Benchmark<'a> {
             bar.finish_and_clear()
         }
 
-        // Compute statistical quantities
-        let t_num = times_wall_clock.len();
-        let t_mean = mean(&times_wall_clock);
-        let t_stddev = if times_wall_clock.len() > 1 {
-            Some(standard_deviation(&times_wall_clock, Some(t_mean)))
-        } else {
-            None
-        };
-        let t_min = min(&times_wall_clock);
-        let t_max = max(&times_wall_clock);
-
-        let user_mean = measurements.time_user_mean();
-        let system_mean = measurements.time_system_mean();
-
         // Formatting and console output
-        let time_unit = self.options.time_unit.unwrap_or(t_mean.suitable_unit());
-        let mean_str = t_mean.format(time_unit);
-        let min_str = t_min.format(time_unit);
-        let max_str = t_max.format(time_unit);
-        let num_str = format!("{t_num} runs");
+        let (mean_str, time_unit) = format_duration_unit(runs.mean(), self.options.time_unit);
+        let min_str = format_duration(runs.min(), Some(time_unit));
+        let max_str = format_duration(runs.max(), Some(time_unit));
+        let num_str = format!("{num_runs} runs", num_runs = runs.len());
 
-        let user_str = user_mean.format(time_unit);
-        let system_str = system_mean.format(time_unit);
+        let user_str = format_duration(runs.user_mean(), Some(time_unit));
+        let system_str = format_duration(runs.system_mean(), Some(time_unit));
 
         if self.options.output_style != OutputStyleOption::Disabled {
-            if times_wall_clock.len() == 1 {
+            if runs.len() == 1 {
                 println!(
                     "  Time ({} ≡):        {:>8}  {:>8}     [User: {}, System: {}]",
                     "abs".green().bold(),
@@ -411,7 +397,7 @@ impl<'a> Benchmark<'a> {
                     system_str.blue()
                 )?;
             } else {
-                let stddev_str = t_stddev.unwrap().format(time_unit);
+                let stddev_str = format_duration(runs.stddev().unwrap(), Some(time_unit));
 
                 console_writeln!(
                     stdout,
@@ -441,7 +427,10 @@ impl<'a> Benchmark<'a> {
 
         // Check execution time
         if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
-            && times_wall_clock.iter().any(|&t| t < MIN_EXECUTION_TIME)
+            && runs
+                .wall_clock_times()
+                .iter()
+                .any(|&t| t < MIN_EXECUTION_TIME)
         {
             warnings.push(Warnings::FastExecutionTime);
         }
@@ -452,7 +441,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = modified_zscores(&times_wall_clock);
+        let scores = modified_zscores(&runs.wall_clock_times());
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -467,7 +456,7 @@ impl<'a> Benchmark<'a> {
 
         if scores[0] > OUTLIER_THRESHOLD {
             warnings.push(Warnings::SlowInitialRun(
-                times_wall_clock[0],
+                runs.wall_clock_times()[0],
                 outlier_warning_options,
             ));
         } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
@@ -491,27 +480,7 @@ impl<'a> Benchmark<'a> {
 
         Ok(BenchmarkResult {
             command: self.command.get_name(),
-            runs: times_wall_clock
-                .iter()
-                .zip(times_user.iter())
-                .zip(times_system.iter())
-                .zip(memory_usage_byte.iter())
-                .zip(exit_codes.iter())
-                .map(
-                    |(
-                        (((wall_clock_time, user_time), system_time), memory_usage_byte),
-                        exit_code,
-                    )| {
-                        Run {
-                            wall_clock_time: *wall_clock_time,
-                            user_time: *user_time,
-                            system_time: *system_time,
-                            memory_usage_byte: *memory_usage_byte,
-                            exit_code: *exit_code,
-                        }
-                    },
-                )
-                .collect(),
+            runs,
             parameters: self
                 .command
                 .get_parameters()
@@ -521,7 +490,7 @@ impl<'a> Benchmark<'a> {
                         name.to_string(),
                         Parameter {
                             value: value.to_string(),
-                            is_unused: self.command.is_parameter_unused(&name),
+                            is_unused: self.command.is_parameter_unused(name),
                         },
                     )
                 })

src/benchmark/relative_speed.rs

@@ -147,17 +147,19 @@ pub fn compute<'a>(
 fn create_result(name: &str, mean: f64) -> BenchmarkResult {
     use std::collections::BTreeMap;
 
-    use crate::benchmark::benchmark_result::Run;
+    use crate::benchmark::benchmark_result::{Run, Runs};
 
     BenchmarkResult {
         command: name.into(),
-        runs: vec![Run {
-            wall_clock_time: mean,
-            user_time: mean,
-            system_time: 0.,
-            memory_usage_byte: 1024,
-            exit_code: Some(0),
-        }],
+        runs: Runs {
+            runs: vec![Run {
+                wall_clock_time: mean,
+                user_time: mean,
+                system_time: 0.,
+                memory_usage_byte: 1024,
+                exit_code: Some(0),
+            }],
+        },
         parameters: BTreeMap::new(),
     }
 }

src/export/csv.rs

@@ -68,7 +68,7 @@ impl Exporter for CsvExporter {
 }
 
 #[cfg(test)]
-use crate::benchmark::benchmark_result::{Parameter, Run};
+use crate::benchmark::benchmark_result::{Parameter, Run, Runs};
 
 #[test]
 fn test_csv() {
@@ -84,7 +84,7 @@ fn test_csv() {
     let results = vec![
         BenchmarkResult {
             command: String::from("command_a"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 7.0,
                     user_time: 7.0,
@@ -106,7 +106,7 @@ fn test_csv() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: {
                 let mut params = BTreeMap::new();
                 params.insert(
@@ -128,7 +128,7 @@ fn test_csv() {
         },
         BenchmarkResult {
             command: String::from("command_b"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 17.0,
                     user_time: 17.0,
@@ -150,7 +150,7 @@ fn test_csv() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: {
                 let mut params = BTreeMap::new();
                 params.insert(

src/export/tests.rs

@@ -1,5 +1,5 @@
 use super::Exporter;
-use crate::benchmark::benchmark_result::{BenchmarkResult, Run};
+use crate::benchmark::benchmark_result::{BenchmarkResult, Run, Runs};
 use crate::export::asciidoc::AsciidocExporter;
 use crate::export::orgmode::OrgmodeExporter;
 use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero};
@@ -27,7 +27,7 @@ fn test_markup_export_auto_ms() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 0.09,
                     user_time: 0.09,
@@ -49,12 +49,12 @@ fn test_markup_export_auto_ms() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 2"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 2.0,
                     user_time: 2.0,
@@ -76,7 +76,7 @@ fn test_markup_export_auto_ms() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -126,7 +126,7 @@ fn test_markup_export_auto_s() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 2.1,
                     user_time: 2.1,
@@ -148,12 +148,12 @@ fn test_markup_export_auto_s() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 0.1,
                     user_time: 0.1,
@@ -175,7 +175,7 @@ fn test_markup_export_auto_s() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -225,7 +225,7 @@ fn test_markup_export_manual_ms() {
     let timing_results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 2.1,
                     user_time: 2.1,
@@ -247,12 +247,12 @@ fn test_markup_export_manual_ms() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 0.1,
                     user_time: 0.1,
@@ -274,7 +274,7 @@ fn test_markup_export_manual_ms() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -323,7 +323,7 @@ fn test_markup_export_manual_s() {
     let results = [
         BenchmarkResult {
             command: String::from("sleep 2"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 2.01,
                     user_time: 2.01,
@@ -345,12 +345,12 @@ fn test_markup_export_manual_s() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
-            runs: vec![
+            runs: Runs::new(vec![
                 Run {
                     wall_clock_time: 0.11,
                     user_time: 0.11,
@@ -372,7 +372,7 @@ fn test_markup_export_manual_s() {
                     memory_usage_byte: 1024,
                     exit_code: Some(0),
                 },
-            ],
+            ]),
             parameters: BTreeMap::new(),
         },
     ];