Merge pull request #969 from sharkdp/codex/result-measurements

Store typed Measurements in BenchmarkResult

src/benchmark/benchmark_result.rs

@@ -1,6 +1,11 @@
 use std::collections::BTreeMap;
 
-use serde::Serialize;
+use serde::ser::SerializeStruct;
+use serde::{Serialize, Serializer};
+
+use crate::benchmark::measurement::Measurements;
+use crate::quantity::{byte, second, Time};
+use crate::util::exit_code::extract_exit_code;
 
 /// Parameter value and whether it was used in the command line template
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
@@ -11,52 +16,82 @@ pub struct Parameter {
     pub is_unused: bool,
 }
 
-/// Set of values that will be exported.
-// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
-// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
-#[derive(Debug, Default, Clone, Serialize, PartialEq)]
+/// Meta data and performance metrics for a single benchmark
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct BenchmarkResult {
     /// The full command line of the program that is being benchmarked
     pub command: String,
 
-    /// The full command line of the program that is being benchmarked, possibly including a list of
-    /// parameters that were not used in the command line template.
-    #[serde(skip_serializing)]
+    /// The full command line, including parameters not used in the command template.
     pub command_with_unused_parameters: String,
 
-    /// The average run time
-    pub mean: f64,
-
-    /// The standard deviation of all run times. Not available if only one run has been performed
-    pub stddev: Option<f64>,
-
-    /// The median run time
-    pub median: f64,
-
-    /// Time spent in user mode
-    pub user: f64,
-
-    /// Time spent in kernel mode
-    pub system: f64,
-
-    /// Minimum of all measured times
-    pub min: f64,
-
-    /// Maximum of all measured times
-    pub max: f64,
-
-    /// All run time measurements
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub times: Option<Vec<f64>>,
-
-    /// Maximum memory usage of the process, in bytes
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub memory_usage_byte: Option<Vec<u64>>,
-
-    /// Exit codes of all command invocations
-    pub exit_codes: Vec<Option<i32>>,
+    /// Performance measurements and exit statuses for each run
+    pub measurements: Measurements,
 
     /// Parameter values for this benchmark
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, Parameter>,
 }
+
+impl BenchmarkResult {
+    /// The average wall clock time
+    pub fn mean_wall_clock_time(&self) -> Time {
+        self.measurements.time_wall_clock_mean()
+    }
+}
+
+// Preserve the existing export format while storing typed measurements internally.
+impl Serialize for BenchmarkResult {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct(
+            "BenchmarkResult",
+            if self.parameters.is_empty() { 11 } else { 12 },
+        )?;
+        state.serialize_field("command", &self.command)?;
+        state.serialize_field("mean", &self.mean_wall_clock_time().get::<second>())?;
+        state.serialize_field(
+            "stddev",
+            &self.measurements.stddev().map(|time| time.get::<second>()),
+        )?;
+        state.serialize_field("median", &self.measurements.median().get::<second>())?;
+        state.serialize_field("user", &self.measurements.time_user_mean().get::<second>())?;
+        state.serialize_field(
+            "system",
+            &self.measurements.time_system_mean().get::<second>(),
+        )?;
+        state.serialize_field("min", &self.measurements.min().get::<second>())?;
+        state.serialize_field("max", &self.measurements.max().get::<second>())?;
+        state.serialize_field(
+            "times",
+            &self
+                .measurements
+                .wall_clock_times()
+                .map(|time| time.get::<second>())
+                .collect::<Vec<_>>(),
+        )?;
+        state.serialize_field(
+            "memory_usage_byte",
+            &self
+                .measurements
+                .measurements
+                .iter()
+                .map(|measurement| measurement.peak_memory_usage.get::<byte>() as u64)
+                .collect::<Vec<_>>(),
+        )?;
+        state.serialize_field(
+            "exit_codes",
+            &self
+                .measurements
+                .measurements
+                .iter()
+                .map(|measurement| extract_exit_code(measurement.exit_status))
+                .collect::<Vec<_>>(),
+        )?;
+        if !self.parameters.is_empty() {
+            state.serialize_field("parameters", &self.parameters)?;
+        }
+        state.end()
+    }
+}

src/benchmark/measurement.rs

@@ -4,7 +4,7 @@ use crate::quantity::statistics::{max, mean, median, min, modified_zscores, stan
 use crate::quantity::{Information, Time};
 
 /// Performance measurements and exit status from running a single command
-#[derive(Debug, Default, Copy, Clone)]
+#[derive(Debug, Default, Copy, Clone, PartialEq)]
 pub struct Measurement {
     /// Wall clock time
     pub time_wall_clock: Time,
@@ -23,12 +23,16 @@ pub struct Measurement {
 }
 
 /// Performance measurements and exit statuses for all runs of a command.
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct Measurements {
     pub measurements: Vec<Measurement>,
 }
 
 impl Measurements {
+    pub fn new(measurements: Vec<Measurement>) -> Self {
+        Self { measurements }
+    }
+
     pub fn len(&self) -> usize {
         self.measurements.len()
     }

src/benchmark/mod.rs

@@ -17,8 +17,7 @@ use crate::output::console_writeln;
 use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
-use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero};
-use crate::util::exit_code::extract_exit_code;
+use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
 use benchmark_result::{BenchmarkResult, Parameter};
 use measurement::{Measurement, Measurements};
 
@@ -352,7 +351,6 @@ impl<'a> Benchmark<'a> {
         let t_num = measurements.len();
         let t_mean = measurements.time_wall_clock_mean();
         let t_stddev = measurements.stddev();
-        let t_median = measurements.median();
         let t_min = measurements.min();
         let t_max = measurements.max();
 
@@ -465,31 +463,7 @@ impl<'a> Benchmark<'a> {
         Ok(BenchmarkResult {
             command: self.command.get_name(),
             command_with_unused_parameters: self.command.get_name_with_unused_parameters(),
-            mean: t_mean.get::<second>(),
-            stddev: t_stddev.map(|t| t.get::<second>()),
-            median: t_median.get::<second>(),
-            user: user_mean.get::<second>(),
-            system: system_mean.get::<second>(),
-            min: t_min.get::<second>(),
-            max: t_max.get::<second>(),
-            times: Some(
-                measurements
-                    .wall_clock_times()
-                    .map(|t| t.get::<second>())
-                    .collect(),
-            ),
-            memory_usage_byte: Some(
-                measurements
-                    .measurements
-                    .iter()
-                    .map(|m| m.peak_memory_usage.get::<byte>() as u64)
-                    .collect(),
-            ),
-            exit_codes: measurements
-                .measurements
-                .iter()
-                .map(|m| extract_exit_code(m.exit_status))
-                .collect(),
+            measurements,
             parameters: self
                 .command
                 .get_parameters()

src/benchmark/relative_speed.rs

@@ -2,6 +2,7 @@ use std::cmp::Ordering;
 
 use super::benchmark_result::BenchmarkResult;
 use crate::options::SortOrder;
+use crate::quantity::second;
 
 #[derive(Debug)]
 pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -14,7 +15,10 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
-    l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
+    l.mean_wall_clock_time()
+        .get::<second>()
+        .partial_cmp(&r.mean_wall_clock_time().get::<second>())
+        .unwrap_or(Ordering::Equal)
 }
 
 pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
@@ -35,7 +39,7 @@ fn compute_relative_speeds<'a>(
             let is_reference = result == reference;
             let relative_ordering = compare_mean_time(result, reference);
 
-            if result.mean == 0.0 {
+            if result.mean_wall_clock_time().get::<second>() == 0.0 {
                 return BenchmarkResultWithRelativeSpeed {
                     result,
                     relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -46,18 +50,34 @@ fn compute_relative_speeds<'a>(
             }
 
             let ratio = match relative_ordering {
-                Ordering::Less => reference.mean / result.mean,
+                Ordering::Less => {
+                    reference.mean_wall_clock_time().get::<second>()
+                        / result.mean_wall_clock_time().get::<second>()
+                }
                 Ordering::Equal => 1.0,
-                Ordering::Greater => result.mean / reference.mean,
+                Ordering::Greater => {
+                    result.mean_wall_clock_time().get::<second>()
+                        / reference.mean_wall_clock_time().get::<second>()
+                }
             };
 
             // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
             // Covariance assumed to be 0, i.e. variables are assumed to be independent
-            let ratio_stddev = match (result.stddev, reference.stddev) {
+            let ratio_stddev = match (
+                result
+                    .measurements
+                    .stddev()
+                    .map(|time| time.get::<second>()),
+                reference
+                    .measurements
+                    .stddev()
+                    .map(|time| time.get::<second>()),
+            ) {
                 (Some(result_stddev), Some(fastest_stddev)) => Some(
                     ratio
-                        * ((result_stddev / result.mean).powi(2)
-                            + (fastest_stddev / reference.mean).powi(2))
+                        * ((result_stddev / result.mean_wall_clock_time().get::<second>()).powi(2)
+                            + (fastest_stddev / reference.mean_wall_clock_time().get::<second>())
+                                .powi(2))
                         .sqrt(),
                 ),
                 _ => None,
@@ -88,7 +108,9 @@ pub fn compute_with_check_from_reference<'a>(
     reference: &'a BenchmarkResult,
     sort_order: SortOrder,
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
-    if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
+    if fastest_of(results).mean_wall_clock_time().get::<second>() == 0.0
+        || reference.mean_wall_clock_time().get::<second>() == 0.0
+    {
         return None;
     }
 
@@ -101,7 +123,7 @@ pub fn compute_with_check(
 ) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
     let fastest = fastest_of(results);
 
-    if fastest.mean == 0.0 {
+    if fastest.mean_wall_clock_time().get::<second>() == 0.0 {
         return None;
     }
 
@@ -122,19 +144,19 @@ pub fn compute(
 fn create_result(name: &str, mean: f64) -> BenchmarkResult {
     use std::collections::BTreeMap;
 
+    use crate::benchmark::measurement::{Measurement, Measurements};
+    use crate::quantity::{second, Time};
+
     BenchmarkResult {
         command: name.into(),
         command_with_unused_parameters: name.into(),
-        mean,
-        stddev: Some(1.0),
-        median: mean,
-        user: mean,
-        system: 0.0,
-        min: mean,
-        max: mean,
-        times: None,
-        memory_usage_byte: None,
-        exit_codes: Vec::new(),
+        measurements: Measurements {
+            measurements: vec![Measurement {
+                time_wall_clock: Time::new::<second>(mean),
+                time_user: Time::new::<second>(mean),
+                ..Default::default()
+            }],
+        },
         parameters: BTreeMap::new(),
     }
 }

src/export/csv.rs

@@ -5,7 +5,7 @@ use csv::WriterBuilder;
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::options::SortOrder;
-use crate::quantity::TimeUnit;
+use crate::quantity::{second, TimeUnit};
 
 use anyhow::Result;
 
@@ -42,13 +42,16 @@ impl Exporter for CsvExporter {
         for res in results {
             let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
             for f in &[
-                res.mean,
-                res.stddev.unwrap_or(0.0),
-                res.median,
-                res.user,
-                res.system,
-                res.min,
-                res.max,
+                res.mean_wall_clock_time().get::<second>(),
+                res.measurements
+                    .stddev()
+                    .unwrap_or_default()
+                    .get::<second>(),
+                res.measurements.median().get::<second>(),
+                res.measurements.time_user_mean().get::<second>(),
+                res.measurements.time_system_mean().get::<second>(),
+                res.measurements.min().get::<second>(),
+                res.measurements.max().get::<second>(),
             ] {
                 fields.push(Cow::Owned(format!("{f:.CSV_PRECISION$}").into_bytes()))
             }
@@ -65,23 +68,41 @@ impl Exporter for CsvExporter {
 #[test]
 fn test_csv() {
     use crate::benchmark::benchmark_result::Parameter;
+    use crate::benchmark::measurement::{Measurement, Measurements};
+    use crate::quantity::{byte, second, Information, Time, Zero};
+
     use std::collections::BTreeMap;
+    use std::process::ExitStatus;
+
     let exporter = CsvExporter::default();
 
     let results = vec![
         BenchmarkResult {
             command: String::from("command_a"),
             command_with_unused_parameters: String::from("command_a"),
-            mean: 1.0,
-            stddev: Some(2.0),
-            median: 1.0,
-            user: 3.0,
-            system: 4.0,
-            min: 5.0,
-            max: 6.0,
-            times: Some(vec![7.0, 8.0, 9.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(7.0),
+                    time_user: Time::new::<second>(7.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(8.0),
+                    time_user: Time::new::<second>(8.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(12.0),
+                    time_user: Time::new::<second>(12.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: {
                 let mut params = BTreeMap::new();
                 params.insert(
@@ -104,16 +125,29 @@ fn test_csv() {
         BenchmarkResult {
             command: String::from("command_b"),
             command_with_unused_parameters: String::from("command_b"),
-            mean: 11.0,
-            stddev: Some(12.0),
-            median: 11.0,
-            user: 13.0,
-            system: 14.0,
-            min: 15.0,
-            max: 16.5,
-            times: Some(vec![17.0, 18.0, 19.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(17.0),
+                    time_user: Time::new::<second>(17.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(18.0),
+                    time_user: Time::new::<second>(18.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(19.0),
+                    time_user: Time::new::<second>(19.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: {
                 let mut params = BTreeMap::new();
                 params.insert(
@@ -144,7 +178,7 @@ fn test_csv() {
 
     insta::assert_snapshot!(actual, @r#"
     command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo
-    command_a,1.000000,2.000000,1.000000,3.000000,4.000000,5.000000,6.000000,two,one
-    command_b,11.000000,12.000000,11.000000,13.000000,14.000000,15.000000,16.500000,seven,one
+    command_a,9.000000,2.645751,8.000000,9.000000,0.000000,7.000000,12.000000,two,one
+    command_b,18.000000,1.000000,18.000000,18.000000,0.000000,17.000000,19.000000,seven,one
     "#);
 }

src/export/markup.rs

@@ -1,7 +1,7 @@
 use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed;
 use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed};
 use crate::options::SortOrder;
-use crate::quantity::{second, FormatQuantity, IsUnit, Time, TimeUnit};
+use crate::quantity::{FormatQuantity, IsUnit, TimeUnit};
 
 use super::Exporter;
 use anyhow::Result;
@@ -50,14 +50,14 @@ pub trait MarkupExporter {
             let cmd_str = measurement
                 .command_with_unused_parameters
                 .replace('|', "\\|");
-            let mean_str = Time::new::<second>(measurement.mean).format_value(unit);
-            let stddev_str = if let Some(stddev) = measurement.stddev {
-                format!(" ± {}", Time::new::<second>(stddev).format_value(unit))
+            let mean_str = measurement.mean_wall_clock_time().format_value(unit);
+            let stddev_str = if let Some(stddev) = measurement.measurements.stddev() {
+                format!(" ± {}", stddev.format_value(unit))
             } else {
                 "".into()
             };
-            let min_str = Time::new::<second>(measurement.min).format_value(unit);
-            let max_str = Time::new::<second>(measurement.max).format_value(unit);
+            let min_str = measurement.measurements.min().format_value(unit);
+            let max_str = measurement.measurements.max().format_value(unit);
             let rel_str = format!("{:.2}", entry.relative_speed);
             let rel_stddev_str = if entry.is_reference {
                 "".into()
@@ -101,7 +101,7 @@ pub trait MarkupExporter {
 fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit {
     if let Some(first_result) = results.first() {
         // Use the first BenchmarkResult entry to determine the unit for all entries.
-        Time::new::<second>(first_result.mean).suitable_unit()
+        first_result.mean_wall_clock_time().suitable_unit()
     } else {
         // Default to `Second`.
         TimeUnit::Second

src/export/tests.rs

@@ -1,10 +1,12 @@
 use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
+use crate::benchmark::measurement::{Measurement, Measurements};
 use crate::export::asciidoc::AsciidocExporter;
 use crate::export::orgmode::OrgmodeExporter;
-use crate::quantity::TimeUnit;
+use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero};
 use crate::{export::markdown::MarkdownExporter, options::SortOrder};
 use std::collections::BTreeMap;
+use std::process::ExitStatus;
 
 fn get_output<E: Exporter + Default>(
     results: &[BenchmarkResult],
@@ -27,31 +29,57 @@ fn test_markup_export_auto_ms() {
         BenchmarkResult {
             command: String::from("sleep 0.1"),
             command_with_unused_parameters: String::from("sleep 0.1"),
-            mean: 0.1057,
-            stddev: Some(0.0016),
-            median: 0.1057,
-            user: 0.0009,
-            system: 0.0011,
-            min: 0.1023,
-            max: 0.1080,
-            times: Some(vec![0.1, 0.1, 0.1]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.09),
+                    time_user: Time::new::<second>(0.09),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.10),
+                    time_user: Time::new::<second>(0.10),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.14),
+                    time_user: Time::new::<second>(0.14),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 2"),
             command_with_unused_parameters: String::from("sleep 2"),
-            mean: 2.0050,
-            stddev: Some(0.0020),
-            median: 2.0050,
-            user: 0.0009,
-            system: 0.0012,
-            min: 2.0020,
-            max: 2.0080,
-            times: Some(vec![2.0, 2.0, 2.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.0),
+                    time_user: Time::new::<second>(2.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(3.0),
+                    time_user: Time::new::<second>(3.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(4.0),
+                    time_user: Time::new::<second>(4.0),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -59,8 +87,8 @@ fn test_markup_export_auto_ms() {
     insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None, SortOrder::Command), @r#"
     | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
     |:---|---:|---:|---:|---:|
-    | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 |
-    | `sleep 2` | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 |
+    | `sleep 0.1` | 110.0 ± 26.5 | 90.0 | 140.0 | 1.00 |
+    | `sleep 2` | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | 27.27 ± 11.21 |
     "#);
 
     insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None, SortOrder::Command), @r#"
@@ -73,24 +101,24 @@ fn test_markup_export_auto_ms() {
     | Relative 
 
     | `sleep 0.1` 
-    | 105.7 ± 1.6 
-    | 102.3 
-    | 108.0 
+    | 110.0 ± 26.5 
+    | 90.0 
+    | 140.0 
     | 1.00 
 
     | `sleep 2` 
-    | 2005.0 ± 2.0 
-    | 2002.0 
-    | 2008.0 
-    | 18.97 ± 0.29 
+    | 3000.0 ± 1000.0 
+    | 2000.0 
+    | 4000.0 
+    | 27.27 ± 11.21 
     |===
     "#);
 
     insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None, SortOrder::Command), @r#"
     | Command  |  Mean [ms] |  Min [ms] |  Max [ms] |  Relative |
     |--+--+--+--+--|
-    | =sleep 0.1=  |  105.7 ± 1.6 |  102.3 |  108.0 |  1.00 |
-    | =sleep 2=  |  2005.0 ± 2.0 |  2002.0 |  2008.0 |  18.97 ± 0.29 |
+    | =sleep 0.1=  |  110.0 ± 26.5 |  90.0 |  140.0 |  1.00 |
+    | =sleep 2=  |  3000.0 ± 1000.0 |  2000.0 |  4000.0 |  27.27 ± 11.21 |
     "#);
 }
 
@@ -102,31 +130,57 @@ fn test_markup_export_auto_s() {
         BenchmarkResult {
             command: String::from("sleep 2"),
             command_with_unused_parameters: String::from("sleep 2"),
-            mean: 2.0050,
-            stddev: Some(0.0020),
-            median: 2.0050,
-            user: 0.0009,
-            system: 0.0012,
-            min: 2.0020,
-            max: 2.0080,
-            times: Some(vec![2.0, 2.0, 2.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.1),
+                    time_user: Time::new::<second>(2.1),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.2),
+                    time_user: Time::new::<second>(2.2),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.3),
+                    time_user: Time::new::<second>(2.3),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
             command_with_unused_parameters: String::from("sleep 0.1"),
-            mean: 0.1057,
-            stddev: Some(0.0016),
-            median: 0.1057,
-            user: 0.0009,
-            system: 0.0011,
-            min: 0.1023,
-            max: 0.1080,
-            times: Some(vec![0.1, 0.1, 0.1]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.1),
+                    time_user: Time::new::<second>(0.1),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.2),
+                    time_user: Time::new::<second>(0.2),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.3),
+                    time_user: Time::new::<second>(0.3),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -134,8 +188,8 @@ fn test_markup_export_auto_s() {
     insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, None, SortOrder::Command), @r#"
     | Command | Mean [s] | Min [s] | Max [s] | Relative |
     |:---|---:|---:|---:|---:|
-    | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
-    | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 |
+    | `sleep 2` | 2.200 ± 0.100 | 2.100 | 2.300 | 11.00 ± 5.52 |
+    | `sleep 0.1` | 0.200 ± 0.100 | 0.100 | 0.300 | 1.00 |
     "#);
 
     insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, None, SortOrder::Command), @r#"
@@ -148,15 +202,15 @@ fn test_markup_export_auto_s() {
     | Relative 
 
     | `sleep 2` 
-    | 2.005 ± 0.002 
-    | 2.002 
-    | 2.008 
-    | 18.97 ± 0.29 
+    | 2.200 ± 0.100 
+    | 2.100 
+    | 2.300 
+    | 11.00 ± 5.52 
 
     | `sleep 0.1` 
-    | 0.106 ± 0.002 
-    | 0.102 
-    | 0.108 
+    | 0.200 ± 0.100 
+    | 0.100 
+    | 0.300 
     | 1.00 
     |===
     "#);
@@ -164,8 +218,8 @@ fn test_markup_export_auto_s() {
     insta::assert_snapshot!(get_output::<OrgmodeExporter>(&results, None, SortOrder::Command), @r#"
     | Command  |  Mean [s] |  Min [s] |  Max [s] |  Relative |
     |--+--+--+--+--|
-    | =sleep 2=  |  2.005 ± 0.002 |  2.002 |  2.008 |  18.97 ± 0.29 |
-    | =sleep 0.1=  |  0.106 ± 0.002 |  0.102 |  0.108 |  1.00 |
+    | =sleep 2=  |  2.200 ± 0.100 |  2.100 |  2.300 |  11.00 ± 5.52 |
+    | =sleep 0.1=  |  0.200 ± 0.100 |  0.100 |  0.300 |  1.00 |
     "#);
 }
 
@@ -177,31 +231,57 @@ fn test_markup_export_manual_ms() {
         BenchmarkResult {
             command: String::from("sleep 2"),
             command_with_unused_parameters: String::from("sleep 2"),
-            mean: 2.0050,
-            stddev: Some(0.0020),
-            median: 2.0050,
-            user: 0.0009,
-            system: 0.0012,
-            min: 2.0020,
-            max: 2.0080,
-            times: Some(vec![2.0, 2.0, 2.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.1),
+                    time_user: Time::new::<second>(2.1),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.2),
+                    time_user: Time::new::<second>(2.2),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.3),
+                    time_user: Time::new::<second>(2.3),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
             command_with_unused_parameters: String::from("sleep 0.1"),
-            mean: 0.1057,
-            stddev: Some(0.0016),
-            median: 0.1057,
-            user: 0.0009,
-            system: 0.0011,
-            min: 0.1023,
-            max: 0.1080,
-            times: Some(vec![0.1, 0.1, 0.1]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.1),
+                    time_user: Time::new::<second>(0.1),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.2),
+                    time_user: Time::new::<second>(0.2),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.3),
+                    time_user: Time::new::<second>(0.3),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
@@ -209,8 +289,8 @@ fn test_markup_export_manual_ms() {
     insta::assert_snapshot!(get_output::<MarkdownExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
     | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
     |:---|---:|---:|---:|---:|
-    | `sleep 2` | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 |
-    | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 |
+    | `sleep 2` | 2200.0 ± 100.0 | 2100.0 | 2300.0 | 11.00 ± 5.52 |
+    | `sleep 0.1` | 200.0 ± 100.0 | 100.0 | 300.0 | 1.00 |
     "#);
 
     insta::assert_snapshot!(get_output::<AsciidocExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
@@ -223,15 +303,15 @@ fn test_markup_export_manual_ms() {
     | Relative 
 
     | `sleep 2` 
-    | 2005.0 ± 2.0 
-    | 2002.0 
-    | 2008.0 
-    | 18.97 ± 0.29 
+    | 2200.0 ± 100.0 
+    | 2100.0 
+    | 2300.0 
+    | 11.00 ± 5.52 
 
     | `sleep 0.1` 
-    | 105.7 ± 1.6 
-    | 102.3 
-    | 108.0 
+    | 200.0 ± 100.0 
+    | 100.0 
+    | 300.0 
     | 1.00 
     |===
     "#);
@@ -239,8 +319,8 @@ fn test_markup_export_manual_ms() {
     insta::assert_snapshot!(get_output::<OrgmodeExporter>(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#"
     | Command  |  Mean [ms] |  Min [ms] |  Max [ms] |  Relative |
     |--+--+--+--+--|
-    | =sleep 2=  |  2005.0 ± 2.0 |  2002.0 |  2008.0 |  18.97 ± 0.29 |
-    | =sleep 0.1=  |  105.7 ± 1.6 |  102.3 |  108.0 |  1.00 |
+    | =sleep 2=  |  2200.0 ± 100.0 |  2100.0 |  2300.0 |  11.00 ± 5.52 |
+    | =sleep 0.1=  |  200.0 ± 100.0 |  100.0 |  300.0 |  1.00 |
     "#);
 }
 
@@ -251,48 +331,74 @@ fn test_markup_export_manual_s() {
         BenchmarkResult {
             command: String::from("sleep 2"),
             command_with_unused_parameters: String::from("sleep 2"),
-            mean: 2.0050,
-            stddev: Some(0.0020),
-            median: 2.0050,
-            user: 0.0009,
-            system: 0.0012,
-            min: 2.0020,
-            max: 2.0080,
-            times: Some(vec![2.0, 2.0, 2.0]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.01),
+                    time_user: Time::new::<second>(2.01),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.02),
+                    time_user: Time::new::<second>(2.02),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(2.03),
+                    time_user: Time::new::<second>(2.03),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
         BenchmarkResult {
             command: String::from("sleep 0.1"),
             command_with_unused_parameters: String::from("sleep 0.1"),
-            mean: 0.1057,
-            stddev: Some(0.0016),
-            median: 0.1057,
-            user: 0.0009,
-            system: 0.0011,
-            min: 0.1023,
-            max: 0.1080,
-            times: Some(vec![0.1, 0.1, 0.1]),
-            memory_usage_byte: None,
-            exit_codes: vec![Some(0), Some(0), Some(0)],
+            measurements: Measurements::new(vec![
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.11),
+                    time_user: Time::new::<second>(0.11),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.12),
+                    time_user: Time::new::<second>(0.12),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+                Measurement {
+                    time_wall_clock: Time::new::<second>(0.13),
+                    time_user: Time::new::<second>(0.13),
+                    time_system: Time::zero(),
+                    peak_memory_usage: Information::new::<byte>(1024.),
+                    exit_status: ExitStatus::default(),
+                },
+            ]),
             parameters: BTreeMap::new(),
         },
     ];
 
     insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, Some(TimeUnit::Second), SortOrder::Command), @r#"
-        | Command | Mean [s] | Min [s] | Max [s] | Relative |
-        |:---|---:|---:|---:|---:|
-        | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
-        | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 |
-        "#);
+    | Command | Mean [s] | Min [s] | Max [s] | Relative |
+    |:---|---:|---:|---:|---:|
+    | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 |
+    | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 |
+    "#);
 
     insta::assert_snapshot!(get_output::<MarkdownExporter>(&results, Some(TimeUnit::Second), SortOrder::MeanTime), @r#"
-        | Command | Mean [s] | Min [s] | Max [s] | Relative |
-        |:---|---:|---:|---:|---:|
-        | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 |
-        | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 |
-        "#);
+    | Command | Mean [s] | Min [s] | Max [s] | Relative |
+    |:---|---:|---:|---:|---:|
+    | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 |
+    | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 |
+    "#);
 
     insta::assert_snapshot!(get_output::<AsciidocExporter>(&results, Some(TimeUnit::Second), SortOrder::Command), @r#"
     [cols="<,>,>,>,>"]
@@ -304,15 +410,15 @@ fn test_markup_export_manual_s() {
     | Relative 
 
     | `sleep 2` 
-    | 2.005 ± 0.002 
-    | 2.002 
-    | 2.008 
-    | 18.97 ± 0.29 
+    | 2.020 ± 0.010 
+    | 2.010 
+    | 2.030 
+    | 16.83 ± 1.41 
 
     | `sleep 0.1` 
-    | 0.106 ± 0.002 
-    | 0.102 
-    | 0.108 
+    | 0.120 ± 0.010 
+    | 0.110 
+    | 0.130 
     | 1.00 
     |===
     "#);