Store typed Measurements in BenchmarkResult

src/benchmark/benchmark_result.rs

@@ -1,6 +1,11 @@
 use std::collections::BTreeMap;
 
-use serde::Serialize;
+use serde::ser::SerializeStruct;
+use serde::{Serialize, Serializer};
+
+use crate::benchmark::measurement::Measurements;
+use crate::quantity::{byte, second, Time};
+use crate::util::exit_code::extract_exit_code;
 
 /// Parameter value and whether it was used in the command line template
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
@@ -11,52 +16,82 @@ pub struct Parameter {
     pub is_unused: bool,
 }
 
-/// Set of values that will be exported.
-// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
-// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
-#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+/// Meta data and performance metrics for a single benchmark
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct BenchmarkResult {
     /// The full command line of the program that is being benchmarked
     pub command: String,
 
-    /// The full command line of the program that is being benchmarked, possibly including a list of
-    /// parameters that were not used in the command line template.
-    #[serde(skip_serializing)]
+    /// The full command line, including parameters not used in the command template.
     pub command_with_unused_parameters: String,
 
-    /// The average run time
-    pub mean: f64,
-
-    /// The standard deviation of all run times. Not available if only one run has been performed
-    pub stddev: Option<f64>,
-
-    /// The median run time
-    pub median: f64,
-
-    /// Time spent in user mode
-    pub user: f64,
-
-    /// Time spent in kernel mode
-    pub system: f64,
-
-    /// Minimum of all measured times
-    pub min: f64,
-
-    /// Maximum of all measured times
-    pub max: f64,
-
-    /// All run time measurements
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub times: Option<Vec<f64>>,
-
-    /// Maximum memory usage of the process, in bytes
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub memory_usage_byte: Option<Vec<u64>>,
-
-    /// Exit codes of all command invocations
-    pub exit_codes: Vec<Option<i32>>,
+    /// Performance measurements and exit statuses for each run
+    pub measurements: Measurements,
 
     /// Parameter values for this benchmark
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, Parameter>,
 }
+
+impl BenchmarkResult {
+    /// The average wall clock time
+    pub fn mean_wall_clock_time(&self) -> Time {
+        self.measurements.time_wall_clock_mean()
+    }
+}
+
+// Preserve the existing export format while storing typed measurements internally.
+impl Serialize for BenchmarkResult {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct(
+            "BenchmarkResult",
+            if self.parameters.is_empty() { 11 } else { 12 },
+        )?;
+        state.serialize_field("command", &self.command)?;
+        state.serialize_field("mean", &self.mean_wall_clock_time().get::<second>())?;
+        state.serialize_field(
+            "stddev",
+            &self.measurements.stddev().map(|time| time.get::<second>()),
+        )?;
+        state.serialize_field("median", &self.measurements.median().get::<second>())?;
+        state.serialize_field("user", &self.measurements.time_user_mean().get::<second>())?;
+        state.serialize_field(
+            "system",
+            &self.measurements.time_system_mean().get::<second>(),
+        )?;
+        state.serialize_field("min", &self.measurements.min().get::<second>())?;
+        state.serialize_field("max", &self.measurements.max().get::<second>())?;
+        state.serialize_field(
+            "times",
+            &self
+                .measurements
+                .wall_clock_times()
+                .map(|time| time.get::<second>())
+                .collect::<Vec<_>>(),
+        )?;
+        state.serialize_field(
+            "memory_usage_byte",
+            &self
+                .measurements
+                .measurements
+                .iter()
+                .map(|measurement| measurement.peak_memory_usage.get::<byte>() as u64)
+                .collect::<Vec<_>>(),
+        )?;
+        state.serialize_field(
+            "exit_codes",
+            &self
+                .measurements
+                .measurements
+                .iter()
+                .map(|measurement| extract_exit_code(measurement.exit_status))
+                .collect::<Vec<_>>(),
+        )?;
+        if !self.parameters.is_empty() {
+            state.serialize_field("parameters", &self.parameters)?;
+        }
+        state.end()
+    }
+}

src/benchmark/measurement.rs

@@ -4,7 +4,7 @@ use crate::quantity::statistics::{max, mean, median, min, modified_zscores, stan
 use crate::quantity::{Information, Time};
 
 /// Performance measurements and exit status from running a single command
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, PartialEq)]
 pub struct Measurement {
     /// Wall clock time
     pub time_wall_clock: Time,
@@ -23,12 +23,16 @@ pub struct Measurement {
 }
 
 /// Performance measurements and exit statuses for all runs of a command.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct Measurements {
     pub measurements: Vec<Measurement>,
 }
 
 impl Measurements {
+    pub fn new(measurements: Vec<Measurement>) -> Self {
+        Self { measurements }
+    }
+
     pub fn len(&self) -> usize {
         self.measurements.len()
     }

src/benchmark/mod.rs

@@ -17,8 +17,7 @@ use crate::output::console_writeln;
 use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
-use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero};
-use crate::util::exit_code::extract_exit_code;
+use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
 use benchmark_result::{BenchmarkResult, Parameter};
 use measurement::{Measurement, Measurements};
 
@@ -332,7 +331,6 @@ impl<'a> Benchmark<'a> {
         let t_num = measurements.len();
         let t_mean = measurements.time_wall_clock_mean();
         let t_stddev = measurements.stddev();
-        let t_median = measurements.median();
         let t_min = measurements.min();
         let t_max = measurements.max();
 
@@ -445,31 +443,7 @@ impl<'a> Benchmark<'a> {
         Ok(BenchmarkResult {
             command: self.command.get_name(),
             command_with_unused_parameters: self.command.get_name_with_unused_parameters(),
-            mean: t_mean.get::<second>(),
-            stddev: t_stddev.map(|t| t.get::<second>()),
-            median: t_median.get::<second>(),
-            user: user_mean.get::<second>(),
-            system: system_mean.get::<second>(),
-            min: t_min.get::<second>(),
-            max: t_max.get::<second>(),
-            times: Some(
-                measurements
-                    .wall_clock_times()
-                    .map(|t| t.get::<second>())
-                    .collect(),
-            ),
-            memory_usage_byte: Some(
-                measurements
-                    .measurements
-                    .iter()
-                    .map(|m| m.peak_memory_usage.get::<byte>() as u64)
-                    .collect(),
-            ),
-            exit_codes: measurements
-                .measurements
-                .iter()
-                .map(|m| extract_exit_code(m.exit_status))
-                .collect(),
+            measurements,
             parameters: self
                 .command
                 .get_parameters()

src/benchmark/relative_speed.rs

@@ -2,6 +2,7 @@ use std::cmp::Ordering;
 
 use super::benchmark_result::BenchmarkResult;
 use crate::options::SortOrder;
+use crate::quantity::second;
 
 #[derive(Debug)]
 pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -14,7 +15,10 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
-    l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
+    l.mean_wall_clock_time()
+        .get::<second>()
+        .partial_cmp(&r.mean_wall_clock_time().get::<second>())
+        .unwrap_or(Ordering::Equal)
 }
 
 pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
@@ -35,7 +39,7 @@ fn compute_relative_speeds<'a>(
             let is_reference = result == reference;
             let relative_ordering = compare_mean_time(result, reference);
 
-            if result.mean == 0.0 {
+            if result.mean_wall_clock_time().get::<second>() == 0.0 {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
                     relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -46,18 +50,34 @@ fn compute_relative_speeds<'a>(
             }
 
             let ratio = match relative_ordering {
-                Ordering::Less => reference.mean / result.mean,
+                Ordering::Less => {
+                    reference.mean_wall_clock_time().get::<second>()
+                        / result.mean_wall_clock_time().get::<second>()
+                }
                 Ordering::Equal => 1.0,
-                Ordering::Greater => result.mean / reference.mean,
+                Ordering::Greater => {
+                    result.mean_wall_clock_time().get::<second>()
+                        / reference.mean_wall_clock_time().get::<second>()
+                }
             };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
             // Covariance assumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (result.stddev, reference.stddev) {
+            let ratio_stddev = match (
+                result
+                    .measurements
+                    .stddev()
+                    .map(|time| time.get::<second>()),
+                reference
+                    .measurements
+                    .stddev()
+                    .map(|time| time.get::<second>()),
+            ) {
                 (Some(result_stddev), Some(fastest_stddev)) => Some(
                     ratio
-                        * ((result_stddev / result.mean).powi(2)
-                            + (fastest_stddev / reference.mean).powi(2))
+                        * ((result_stddev / result.mean_wall_clock_time().get::<second>()).powi(2)
+                            + (fastest_stddev / reference.mean_wall_clock_time().get::<second>())
+                                .powi(2))
                         .sqrt(),
                 ),
                 _ => None,
@@ -88,7 +108,9 @@ pub fn compute_with_check_from_reference<'a>(
     reference: &'a BenchmarkResult,
     sort_order: SortOrder,
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
-    if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
+    if fastest_of(results).mean_wall_clock_time().get::<second>() == 0.0
+        || reference.mean_wall_clock_time().get::<second>() == 0.0
+    {
         return None;
     }
 
@@ -101,7 +123,7 @@ pub fn compute_with_check(
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
     let fastest = fastest_of(results);
 
-    if fastest.mean == 0.0 {
+    if fastest.mean_wall_clock_time().get::<second>() == 0.0 {
         return None;
     }
 
@@ -122,19 +144,19 @@ pub fn compute(
 fn create_result(name: &str, mean: f64) -> BenchmarkResult {
     use std::collections::BTreeMap;
 
+    use crate::benchmark::measurement::{Measurement, Measurements};
+    use crate::quantity::{second, Time};
+
     BenchmarkResult {
         command: name.into(),
         command_with_unused_parameters: name.into(),
-        mean,
-        stddev: Some(1.0),
-        median: mean,
-        user: mean,
-        system: 0.0,
-        min: mean,
-        max: mean,
-        times: None,
-        memory_usage_byte: None,
-        exit_codes: Vec::new(),
+        measurements: Measurements {
+            measurements: vec![Measurement {
+                time_wall_clock: Time::new::<second>(mean),
+                time_user: Time::new::<second>(mean),
+                ..Default::default()
+            }],
+        },
         parameters: BTreeMap::new(),
     }
 }