Introduce new BenchmarkRun struct

src/benchmark/benchmark_result.rs

@@ -1,7 +1,19 @@
 use std::collections::BTreeMap;
 
-use serde::ser::SerializeStruct;
-use serde::{Serialize, Serializer};
+use serde::Serialize;
+use statistical::{mean, median, standard_deviation};
+
+use crate::util::{
+    min_max::{max, min},
+    units::Second,
+};
+
+#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+pub struct BenchmarkRun {
+    pub wall_clock_time: Second,
+    // user_time: Second,
+    // system_time: Second,
+}
 
 use crate::benchmark::measurement::Measurements;
 use crate::quantity::{byte, second, Time};
@@ -25,8 +37,21 @@ pub struct BenchmarkResult {
     /// The full command line, including parameters not used in the command template.
     pub command_with_unused_parameters: String,
 
-    /// Performance measurements and exit statuses for each run
-    pub measurements: Measurements,
+    /// Time spent in user mode
+    pub user: f64,
+
+    /// Time spent in kernel mode
+    pub system: f64,
+
+    /// All run time measurements
+    pub runs: Vec<BenchmarkRun>,
+
+    /// Maximum memory usage of the process, in bytes
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub memory_usage_byte: Option<Vec<u64>>,
+
+    /// Exit codes of all command invocations
+    pub exit_codes: Vec<Option<i32>>,
 
     /// Parameter values for this benchmark
     pub parameters: BTreeMap<String, Parameter>,
@@ -95,3 +120,41 @@ impl Serialize for BenchmarkResult {
         state.end()
     }
 }
+
+impl BenchmarkResult {
+    fn wall_clock_times(&self) -> Vec<Second> {
+        self.runs.iter().map(|run| run.wall_clock_time).collect()
+    }
+
+    /// The average run time
+    pub fn mean(&self) -> Second {
+        mean(&self.wall_clock_times())
+    }
+
+    /// The standard deviation of all run times. Not available if only one run has been performed
+    pub fn stddev(&self) -> Option<Second> {
+        let times = self.wall_clock_times();
+
+        let t_mean = mean(&times);
+        if times.len() > 1 {
+            Some(standard_deviation(&times, Some(t_mean)))
+        } else {
+            None
+        }
+    }
+
+    /// The median run time
+    pub fn median(&self) -> Second {
+        median(&self.wall_clock_times())
+    }
+
+    /// The minimum run time
+    pub fn min(&self) -> Second {
+        min(&self.wall_clock_times())
+    }
+
+    /// The maximum run time
+    pub fn max(&self) -> Second {
+        max(&self.wall_clock_times())
+    }
+}

src/benchmark/mod.rs

@@ -8,6 +8,7 @@ use std::cmp;
 use std::io::{self, Write};
 use std::time::Instant;
 
+use crate::benchmark::benchmark_result::BenchmarkRun;
 use crate::benchmark::executor::BenchmarkIteration;
 use crate::command::Command;
 use crate::options::{
@@ -26,6 +27,7 @@ use measurement::{Measurement, Measurements};
 
 use anyhow::{anyhow, Result};
 use colored::*;
+use statistical::{mean, standard_deviation};
 
 use self::executor::Executor;
 
@@ -369,11 +371,15 @@ impl<'a> Benchmark<'a> {
         }
 
         // Compute statistical quantities
-        let t_num = measurements.len();
-        let t_mean = measurements.time_wall_clock_mean();
-        let t_stddev = measurements.stddev();
-        let t_min = measurements.min();
-        let t_max = measurements.max();
+        let t_num = times_real.len();
+        let t_mean = mean(&times_real);
+        let t_stddev = if times_real.len() > 1 {
+            Some(standard_deviation(&times_real, Some(t_mean)))
+        } else {
+            None
+        };
+        let t_min = min(&times_real);
+        let t_max = max(&times_real);
 
         let user_mean = measurements.time_user_mean();
         let system_mean = measurements.time_system_mean();
@@ -484,7 +490,16 @@ impl<'a> Benchmark<'a> {
         Ok(BenchmarkResult {
             command: self.command.get_name(),
             command_with_unused_parameters: self.command.get_name_with_unused_parameters(),
-            measurements,
+            user: user_mean,
+            system: system_mean,
+            runs: times_real
+                .iter()
+                .map(|t| BenchmarkRun {
+                    wall_clock_time: *t,
+                })
+                .collect(),
+            memory_usage_byte: Some(memory_usage_byte),
+            exit_codes,
             parameters: self
                 .command
                 .get_parameters()

src/benchmark/relative_speed.rs

@@ -31,9 +31,7 @@ impl BenchmarkResultWithRelativeSpeed<'_> {
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
-    l.mean_wall_clock_time()
-        .partial_cmp(&r.mean_wall_clock_time())
-        .unwrap_or(Ordering::Equal)
+    l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal)
 }
 
 pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
@@ -68,11 +66,9 @@ fn compute_relative_speeds<'a>(
             }
 
             let ratio = match relative_ordering {
-                Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(),
-                Ordering::Equal => Ratio::new::<quantity::ratio>(1.0),
-                Ordering::Greater => {
-                    result.mean_wall_clock_time() / reference.mean_wall_clock_time()
-                }
+                Ordering::Less => reference.mean() / result.mean(),
+                Ordering::Equal => 1.0,
+                Ordering::Greater => result.mean() / reference.mean(),
             };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
@@ -117,9 +113,7 @@ pub fn compute_with_check_from_reference<'a>(
     reference: &'a BenchmarkResult,
     sort_order: SortOrder,
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
-    if fastest_of(results).mean_wall_clock_time() == Time::zero()
-        || reference.mean_wall_clock_time() == Time::zero()
-    {
+    if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 {
         return None;
     }
 
@@ -132,7 +126,7 @@ pub fn compute_with_check(
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
     let fastest = fastest_of(results);
 
-    if fastest.mean_wall_clock_time() == Time::zero() {
+    if fastest.mean() == 0.0 {
         return None;
     }
 
@@ -153,19 +147,18 @@ pub fn compute<'a>(
 fn create_result(name: &str, mean: f64) -> BenchmarkResult {
     use std::collections::BTreeMap;
 
-    use crate::benchmark::measurement::{Measurement, Measurements};
-    use crate::quantity::{second, Time};
+    use crate::benchmark::benchmark_result::BenchmarkRun;
 
     BenchmarkResult {
         command: name.into(),
         command_with_unused_parameters: name.into(),
-        measurements: Measurements {
-            measurements: vec![Measurement {
-                time_wall_clock: Time::new::<second>(mean),
-                time_user: Time::new::<second>(mean),
-                ..Default::default()
-            }],
-        },
+        user: mean,
+        system: 0.0,
+        runs: vec![BenchmarkRun {
+            wall_clock_time: mean,
+        }],
+        memory_usage_byte: None,
+        exit_codes: Vec::new(),
         parameters: BTreeMap::new(),
     }
 }

src/benchmark/scheduler.rs

@@ -205,16 +205,11 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
 fn scheduler_basic() -> Result<()> {
     insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#"
     - command: sleep 0.123
-      mean: 0.123
-      stddev: 0
-      median: 0.123
       user: 0
       system: 0
-      min: 0.123
-      max: 0.123
-      times:
-        - 0.123
-        - 0.123
+      runs:
+        - wall_clock_time: 0.123
+        - wall_clock_time: 0.123
       memory_usage_byte:
         - 0
         - 0
@@ -222,16 +217,11 @@ fn scheduler_basic() -> Result<()> {
         - 0
         - 0
     - command: sleep 0.456
-      mean: 0.456
-      stddev: 0
-      median: 0.456
       user: 0
       system: 0
-      min: 0.456
-      max: 0.456
-      times:
-        - 0.456
-        - 0.456
+      runs:
+        - wall_clock_time: 0.456
+        - wall_clock_time: 0.456
       memory_usage_byte:
         - 0
         - 0