From b23372c5e297b91f37b3c26fc44a22eab2e8e5f4 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 12:46:10 +0200 Subject: [PATCH] Store typed Measurements in BenchmarkResult --- src/benchmark/benchmark_result.rs | 115 ++++++---- src/benchmark/measurement.rs | 8 +- src/benchmark/mod.rs | 30 +-- src/benchmark/relative_speed.rs | 60 +++-- src/export/csv.rs | 94 +++++--- src/export/markup.rs | 14 +- src/export/tests.rs | 368 +++++++++++++++++++----------- 7 files changed, 432 insertions(+), 257 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 27d41a4..387884d 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,6 +1,11 @@ use std::collections::BTreeMap; -use serde::Serialize; +use serde::ser::SerializeStruct; +use serde::{Serialize, Serializer}; + +use crate::benchmark::measurement::Measurements; +use crate::quantity::{byte, second, Time}; +use crate::util::exit_code::extract_exit_code; /// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] @@ -11,52 +16,82 @@ pub struct Parameter { pub is_unused: bool, } -/// Set of values that will be exported. -// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the -// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. -#[derive(Debug, Default, Clone, Serialize, PartialEq)] +/// Meta data and performance metrics for a single benchmark +#[derive(Debug, Default, Clone, PartialEq)] pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, - /// The full command line of the program that is being benchmarked, possibly including a list of - /// parameters that were not used in the command line template. - #[serde(skip_serializing)] + /// The full command line, including parameters not used in the command template. pub command_with_unused_parameters: String, - /// The average run time - pub mean: f64, - - /// The standard deviation of all run times. Not available if only one run has been performed - pub stddev: Option, - - /// The median run time - pub median: f64, - - /// Time spent in user mode - pub user: f64, - - /// Time spent in kernel mode - pub system: f64, - - /// Minimum of all measured times - pub min: f64, - - /// Maximum of all measured times - pub max: f64, - - /// All run time measurements - #[serde(skip_serializing_if = "Option::is_none")] - pub times: Option>, - - /// Maximum memory usage of the process, in bytes - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_usage_byte: Option>, - - /// Exit codes of all command invocations - pub exit_codes: Vec>, + /// Performance measurements and exit statuses for each run + pub measurements: Measurements, /// Parameter values for this benchmark - #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, } + +impl BenchmarkResult { + /// The average wall clock time + pub fn mean_wall_clock_time(&self) -> Time { + self.measurements.time_wall_clock_mean() + } +} + +// Preserve the existing export format while storing typed measurements internally. +impl Serialize for BenchmarkResult { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_struct( + "BenchmarkResult", + if self.parameters.is_empty() { 11 } else { 12 }, + )?; + state.serialize_field("command", &self.command)?; + state.serialize_field("mean", &self.mean_wall_clock_time().get::())?; + state.serialize_field( + "stddev", + &self.measurements.stddev().map(|time| time.get::()), + )?; + state.serialize_field("median", &self.measurements.median().get::())?; + state.serialize_field("user", &self.measurements.time_user_mean().get::())?; + state.serialize_field( + "system", + &self.measurements.time_system_mean().get::(), + )?; + state.serialize_field("min", &self.measurements.min().get::())?; + state.serialize_field("max", &self.measurements.max().get::())?; + state.serialize_field( + "times", + &self + .measurements + .wall_clock_times() + .map(|time| time.get::()) + .collect::>(), + )?; + state.serialize_field( + "memory_usage_byte", + &self + .measurements + .measurements + .iter() + .map(|measurement| measurement.peak_memory_usage.get::() as u64) + .collect::>(), + )?; + state.serialize_field( + "exit_codes", + &self + .measurements + .measurements + .iter() + .map(|measurement| extract_exit_code(measurement.exit_status)) + .collect::>(), + )?; + if !self.parameters.is_empty() { + state.serialize_field("parameters", &self.parameters)?; + } + state.end() + } +} diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index 58d522c..718ca88 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -4,7 +4,7 @@ use crate::quantity::statistics::{max, mean, median, min, modified_zscores, stan use crate::quantity::{Information, Time}; /// Performance measurements and exit status from running a single command -#[derive(Debug, Default, Copy, Clone)] +#[derive(Debug, Default, Copy, Clone, PartialEq)] pub struct Measurement { /// Wall clock time pub time_wall_clock: Time, @@ -23,12 +23,16 @@ pub struct Measurement { } /// Performance measurements and exit statuses for all runs of a command. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct Measurements { pub measurements: Vec, } impl Measurements { + pub fn new(measurements: Vec) -> Self { + Self { measurements } + } + pub fn len(&self) -> usize { self.measurements.len() } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index ecf1302..50ee744 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -17,8 +17,7 @@ use crate::output::console_writeln; use crate::output::progress_bar::get_progress_bar; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; -use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero}; -use crate::util::exit_code::extract_exit_code; +use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero}; use benchmark_result::{BenchmarkResult, Parameter}; use measurement::{Measurement, Measurements}; @@ -332,7 +331,6 @@ impl<'a> Benchmark<'a> { let t_num = measurements.len(); let t_mean = measurements.time_wall_clock_mean(); let t_stddev = measurements.stddev(); - let t_median = measurements.median(); let t_min = measurements.min(); let t_max = measurements.max(); @@ -445,31 +443,7 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), command_with_unused_parameters: self.command.get_name_with_unused_parameters(), - mean: t_mean.get::(), - stddev: t_stddev.map(|t| t.get::()), - median: t_median.get::(), - user: user_mean.get::(), - system: system_mean.get::(), - min: t_min.get::(), - max: t_max.get::(), - times: Some( - measurements - .wall_clock_times() - .map(|t| t.get::()) - .collect(), - ), - memory_usage_byte: Some( - measurements - .measurements - .iter() - .map(|m| m.peak_memory_usage.get::() as u64) - .collect(), - ), - exit_codes: measurements - .measurements - .iter() - .map(|m| extract_exit_code(m.exit_status)) - .collect(), + measurements, parameters: self .command .get_parameters() diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index f40c85a..1efea5f 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -2,6 +2,7 @@ use std::cmp::Ordering; use super::benchmark_result::BenchmarkResult; use crate::options::SortOrder; +use crate::quantity::second; #[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { @@ -14,7 +15,10 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { - l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal) + l.mean_wall_clock_time() + .get::() + .partial_cmp(&r.mean_wall_clock_time().get::()) + .unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { @@ -35,7 +39,7 @@ fn compute_relative_speeds<'a>( let is_reference = result == reference; let relative_ordering = compare_mean_time(result, reference); - if result.mean == 0.0 { + if result.mean_wall_clock_time().get::() == 0.0 { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, @@ -46,18 +50,34 @@ fn compute_relative_speeds<'a>( } let ratio = match relative_ordering { - Ordering::Less => reference.mean / result.mean, + Ordering::Less => { + reference.mean_wall_clock_time().get::() + / result.mean_wall_clock_time().get::() + } Ordering::Equal => 1.0, - Ordering::Greater => result.mean / reference.mean, + Ordering::Greater => { + result.mean_wall_clock_time().get::() + / reference.mean_wall_clock_time().get::() + } }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance assumed to be 0, i.e. variables are assumed to be independent - let ratio_stddev = match (result.stddev, reference.stddev) { + let ratio_stddev = match ( + result + .measurements + .stddev() + .map(|time| time.get::()), + reference + .measurements + .stddev() + .map(|time| time.get::()), + ) { (Some(result_stddev), Some(fastest_stddev)) => Some( ratio - * ((result_stddev / result.mean).powi(2) - + (fastest_stddev / reference.mean).powi(2)) + * ((result_stddev / result.mean_wall_clock_time().get::()).powi(2) + + (fastest_stddev / reference.mean_wall_clock_time().get::()) + .powi(2)) .sqrt(), ), _ => None, @@ -88,7 +108,9 @@ pub fn compute_with_check_from_reference<'a>( reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { - if fastest_of(results).mean == 0.0 || reference.mean == 0.0 { + if fastest_of(results).mean_wall_clock_time().get::() == 0.0 + || reference.mean_wall_clock_time().get::() == 0.0 + { return None; } @@ -101,7 +123,7 @@ pub fn compute_with_check( ) -> Option>> { let fastest = fastest_of(results); - if fastest.mean == 0.0 { + if fastest.mean_wall_clock_time().get::() == 0.0 { return None; } @@ -122,19 +144,19 @@ pub fn compute( fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; + use crate::benchmark::measurement::{Measurement, Measurements}; + use crate::quantity::{second, Time}; + BenchmarkResult { command: name.into(), command_with_unused_parameters: name.into(), - mean, - stddev: Some(1.0), - median: mean, - user: mean, - system: 0.0, - min: mean, - max: mean, - times: None, - memory_usage_byte: None, - exit_codes: Vec::new(), + measurements: Measurements { + measurements: vec![Measurement { + time_wall_clock: Time::new::(mean), + time_user: Time::new::(mean), + ..Default::default() + }], + }, parameters: BTreeMap::new(), } } diff --git a/src/export/csv.rs b/src/export/csv.rs index 331b52e..fbb0c2e 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -5,7 +5,7 @@ use csv::WriterBuilder; use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; use crate::options::SortOrder; -use crate::quantity::TimeUnit; +use crate::quantity::{second, TimeUnit}; use anyhow::Result; @@ -42,13 +42,16 @@ impl Exporter for CsvExporter { for res in results { let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; for f in &[ - res.mean, - res.stddev.unwrap_or(0.0), - res.median, - res.user, - res.system, - res.min, - res.max, + res.mean_wall_clock_time().get::(), + res.measurements + .stddev() + .unwrap_or_default() + .get::(), + res.measurements.median().get::(), + res.measurements.time_user_mean().get::(), + res.measurements.time_system_mean().get::(), + res.measurements.min().get::(), + res.measurements.max().get::(), ] { fields.push(Cow::Owned(format!("{f:.CSV_PRECISION$}").into_bytes())) } @@ -65,23 +68,41 @@ impl Exporter for CsvExporter { #[test] fn test_csv() { use crate::benchmark::benchmark_result::Parameter; + use crate::benchmark::measurement::{Measurement, Measurements}; + use crate::quantity::{byte, second, Information, Time, Zero}; + use std::collections::BTreeMap; + use std::process::ExitStatus; + let exporter = CsvExporter::default(); let results = vec![ BenchmarkResult { command: String::from("command_a"), command_with_unused_parameters: String::from("command_a"), - mean: 1.0, - stddev: Some(2.0), - median: 1.0, - user: 3.0, - system: 4.0, - min: 5.0, - max: 6.0, - times: Some(vec![7.0, 8.0, 9.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(7.0), + time_user: Time::new::(7.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(8.0), + time_user: Time::new::(8.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(12.0), + time_user: Time::new::(12.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: { let mut params = BTreeMap::new(); params.insert( @@ -104,16 +125,29 @@ fn test_csv() { BenchmarkResult { command: String::from("command_b"), command_with_unused_parameters: String::from("command_b"), - mean: 11.0, - stddev: Some(12.0), - median: 11.0, - user: 13.0, - system: 14.0, - min: 15.0, - max: 16.5, - times: Some(vec![17.0, 18.0, 19.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(17.0), + time_user: Time::new::(17.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(18.0), + time_user: Time::new::(18.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(19.0), + time_user: Time::new::(19.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: { let mut params = BTreeMap::new(); params.insert( @@ -144,7 +178,7 @@ fn test_csv() { insta::assert_snapshot!(actual, @r#" command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo - command_a,1.000000,2.000000,1.000000,3.000000,4.000000,5.000000,6.000000,two,one - command_b,11.000000,12.000000,11.000000,13.000000,14.000000,15.000000,16.500000,seven,one + command_a,9.000000,2.645751,8.000000,9.000000,0.000000,7.000000,12.000000,two,one + command_b,18.000000,1.000000,18.000000,18.000000,0.000000,17.000000,19.000000,seven,one "#); } diff --git a/src/export/markup.rs b/src/export/markup.rs index e14ffe4..f3f8711 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -1,7 +1,7 @@ use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed; use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed}; use crate::options::SortOrder; -use crate::quantity::{second, FormatQuantity, IsUnit, Time, TimeUnit}; +use crate::quantity::{FormatQuantity, IsUnit, TimeUnit}; use super::Exporter; use anyhow::Result; @@ -50,14 +50,14 @@ pub trait MarkupExporter { let cmd_str = measurement .command_with_unused_parameters .replace('|', "\\|"); - let mean_str = Time::new::(measurement.mean).format_value(unit); - let stddev_str = if let Some(stddev) = measurement.stddev { - format!(" ± {}", Time::new::(stddev).format_value(unit)) + let mean_str = measurement.mean_wall_clock_time().format_value(unit); + let stddev_str = if let Some(stddev) = measurement.measurements.stddev() { + format!(" ± {}", stddev.format_value(unit)) } else { "".into() }; - let min_str = Time::new::(measurement.min).format_value(unit); - let max_str = Time::new::(measurement.max).format_value(unit); + let min_str = measurement.measurements.min().format_value(unit); + let max_str = measurement.measurements.max().format_value(unit); let rel_str = format!("{:.2}", entry.relative_speed); let rel_stddev_str = if entry.is_reference { "".into() @@ -101,7 +101,7 @@ pub trait MarkupExporter { fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit { if let Some(first_result) = results.first() { // Use the first BenchmarkResult entry to determine the unit for all entries. - Time::new::(first_result.mean).suitable_unit() + first_result.mean_wall_clock_time().suitable_unit() } else { // Default to `Second`. TimeUnit::Second diff --git a/src/export/tests.rs b/src/export/tests.rs index d57dfec..9566f9d 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -1,10 +1,12 @@ use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::benchmark::measurement::{Measurement, Measurements}; use crate::export::asciidoc::AsciidocExporter; use crate::export::orgmode::OrgmodeExporter; -use crate::quantity::TimeUnit; +use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero}; use crate::{export::markdown::MarkdownExporter, options::SortOrder}; use std::collections::BTreeMap; +use std::process::ExitStatus; fn get_output( results: &[BenchmarkResult], @@ -27,31 +29,57 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - mean: 0.1057, - stddev: Some(0.0016), - median: 0.1057, - user: 0.0009, - system: 0.0011, - min: 0.1023, - max: 0.1080, - times: Some(vec![0.1, 0.1, 0.1]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(0.09), + time_user: Time::new::(0.09), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.10), + time_user: Time::new::(0.10), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.14), + time_user: Time::new::(0.14), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - mean: 2.0050, - stddev: Some(0.0020), - median: 2.0050, - user: 0.0009, - system: 0.0012, - min: 2.0020, - max: 2.0080, - times: Some(vec![2.0, 2.0, 2.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(2.0), + time_user: Time::new::(2.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(3.0), + time_user: Time::new::(3.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(4.0), + time_user: Time::new::(4.0), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, ]; @@ -59,8 +87,8 @@ fn test_markup_export_auto_ms() { insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| - | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | - | `sleep 2` | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 | + | `sleep 0.1` | 110.0 ± 26.5 | 90.0 | 140.0 | 1.00 | + | `sleep 2` | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | 27.27 ± 11.21 | "#); insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" @@ -73,24 +101,24 @@ fn test_markup_export_auto_ms() { | Relative | `sleep 0.1` - | 105.7 ± 1.6 - | 102.3 - | 108.0 + | 110.0 ± 26.5 + | 90.0 + | 140.0 | 1.00 | `sleep 2` - | 2005.0 ± 2.0 - | 2002.0 - | 2008.0 - | 18.97 ± 0.29 + | 3000.0 ± 1000.0 + | 2000.0 + | 4000.0 + | 27.27 ± 11.21 |=== "#); insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |--+--+--+--+--| - | =sleep 0.1= | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | - | =sleep 2= | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 | + | =sleep 0.1= | 110.0 ± 26.5 | 90.0 | 140.0 | 1.00 | + | =sleep 2= | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | 27.27 ± 11.21 | "#); } @@ -102,31 +130,57 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - mean: 2.0050, - stddev: Some(0.0020), - median: 2.0050, - user: 0.0009, - system: 0.0012, - min: 2.0020, - max: 2.0080, - times: Some(vec![2.0, 2.0, 2.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(2.1), + time_user: Time::new::(2.1), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.2), + time_user: Time::new::(2.2), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.3), + time_user: Time::new::(2.3), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - mean: 0.1057, - stddev: Some(0.0016), - median: 0.1057, - user: 0.0009, - system: 0.0011, - min: 0.1023, - max: 0.1080, - times: Some(vec![0.1, 0.1, 0.1]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(0.1), + time_user: Time::new::(0.1), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.2), + time_user: Time::new::(0.2), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.3), + time_user: Time::new::(0.3), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, ]; @@ -134,8 +188,8 @@ fn test_markup_export_auto_s() { insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| - | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | - | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 | + | `sleep 2` | 2.200 ± 0.100 | 2.100 | 2.300 | 11.00 ± 5.52 | + | `sleep 0.1` | 0.200 ± 0.100 | 0.100 | 0.300 | 1.00 | "#); insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" @@ -148,15 +202,15 @@ fn test_markup_export_auto_s() { | Relative | `sleep 2` - | 2.005 ± 0.002 - | 2.002 - | 2.008 - | 18.97 ± 0.29 + | 2.200 ± 0.100 + | 2.100 + | 2.300 + | 11.00 ± 5.52 | `sleep 0.1` - | 0.106 ± 0.002 - | 0.102 - | 0.108 + | 0.200 ± 0.100 + | 0.100 + | 0.300 | 1.00 |=== "#); @@ -164,8 +218,8 @@ fn test_markup_export_auto_s() { insta::assert_snapshot!(get_output::(&results, None, SortOrder::Command), @r#" | Command | Mean [s] | Min [s] | Max [s] | Relative | |--+--+--+--+--| - | =sleep 2= | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | - | =sleep 0.1= | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 | + | =sleep 2= | 2.200 ± 0.100 | 2.100 | 2.300 | 11.00 ± 5.52 | + | =sleep 0.1= | 0.200 ± 0.100 | 0.100 | 0.300 | 1.00 | "#); } @@ -177,31 +231,57 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - mean: 2.0050, - stddev: Some(0.0020), - median: 2.0050, - user: 0.0009, - system: 0.0012, - min: 2.0020, - max: 2.0080, - times: Some(vec![2.0, 2.0, 2.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(2.1), + time_user: Time::new::(2.1), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.2), + time_user: Time::new::(2.2), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.3), + time_user: Time::new::(2.3), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - mean: 0.1057, - stddev: Some(0.0016), - median: 0.1057, - user: 0.0009, - system: 0.0011, - min: 0.1023, - max: 0.1080, - times: Some(vec![0.1, 0.1, 0.1]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(0.1), + time_user: Time::new::(0.1), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.2), + time_user: Time::new::(0.2), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.3), + time_user: Time::new::(0.3), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, ]; @@ -209,8 +289,8 @@ fn test_markup_export_manual_ms() { insta::assert_snapshot!(get_output::(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#" | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| - | `sleep 2` | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 | - | `sleep 0.1` | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | + | `sleep 2` | 2200.0 ± 100.0 | 2100.0 | 2300.0 | 11.00 ± 5.52 | + | `sleep 0.1` | 200.0 ± 100.0 | 100.0 | 300.0 | 1.00 | "#); insta::assert_snapshot!(get_output::(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#" @@ -223,15 +303,15 @@ fn test_markup_export_manual_ms() { | Relative | `sleep 2` - | 2005.0 ± 2.0 - | 2002.0 - | 2008.0 - | 18.97 ± 0.29 + | 2200.0 ± 100.0 + | 2100.0 + | 2300.0 + | 11.00 ± 5.52 | `sleep 0.1` - | 105.7 ± 1.6 - | 102.3 - | 108.0 + | 200.0 ± 100.0 + | 100.0 + | 300.0 | 1.00 |=== "#); @@ -239,8 +319,8 @@ fn test_markup_export_manual_ms() { insta::assert_snapshot!(get_output::(&timing_results, Some(TimeUnit::MilliSecond), SortOrder::Command), @r#" | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |--+--+--+--+--| - | =sleep 2= | 2005.0 ± 2.0 | 2002.0 | 2008.0 | 18.97 ± 0.29 | - | =sleep 0.1= | 105.7 ± 1.6 | 102.3 | 108.0 | 1.00 | + | =sleep 2= | 2200.0 ± 100.0 | 2100.0 | 2300.0 | 11.00 ± 5.52 | + | =sleep 0.1= | 200.0 ± 100.0 | 100.0 | 300.0 | 1.00 | "#); } @@ -251,48 +331,74 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - mean: 2.0050, - stddev: Some(0.0020), - median: 2.0050, - user: 0.0009, - system: 0.0012, - min: 2.0020, - max: 2.0080, - times: Some(vec![2.0, 2.0, 2.0]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(2.01), + time_user: Time::new::(2.01), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.02), + time_user: Time::new::(2.02), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(2.03), + time_user: Time::new::(2.03), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - mean: 0.1057, - stddev: Some(0.0016), - median: 0.1057, - user: 0.0009, - system: 0.0011, - min: 0.1023, - max: 0.1080, - times: Some(vec![0.1, 0.1, 0.1]), - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], + measurements: Measurements::new(vec![ + Measurement { + time_wall_clock: Time::new::(0.11), + time_user: Time::new::(0.11), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.12), + time_user: Time::new::(0.12), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + Measurement { + time_wall_clock: Time::new::(0.13), + time_user: Time::new::(0.13), + time_system: Time::zero(), + peak_memory_usage: Information::new::(1024.), + exit_status: ExitStatus::default(), + }, + ]), parameters: BTreeMap::new(), }, ]; insta::assert_snapshot!(get_output::(&results, Some(TimeUnit::Second), SortOrder::Command), @r#" - | Command | Mean [s] | Min [s] | Max [s] | Relative | - |:---|---:|---:|---:|---:| - | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | - | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 | - "#); + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 | + | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 | + "#); insta::assert_snapshot!(get_output::(&results, Some(TimeUnit::Second), SortOrder::MeanTime), @r#" - | Command | Mean [s] | Min [s] | Max [s] | Relative | - |:---|---:|---:|---:|---:| - | `sleep 0.1` | 0.106 ± 0.002 | 0.102 | 0.108 | 1.00 | - | `sleep 2` | 2.005 ± 0.002 | 2.002 | 2.008 | 18.97 ± 0.29 | - "#); + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 | + | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 | + "#); insta::assert_snapshot!(get_output::(&results, Some(TimeUnit::Second), SortOrder::Command), @r#" [cols="<,>,>,>,>"] @@ -304,15 +410,15 @@ fn test_markup_export_manual_s() { | Relative | `sleep 2` - | 2.005 ± 0.002 - | 2.002 - | 2.008 - | 18.97 ± 0.29 + | 2.020 ± 0.010 + | 2.010 + | 2.030 + | 16.83 ± 1.41 | `sleep 0.1` - | 0.106 ± 0.002 - | 0.102 - | 0.108 + | 0.120 ± 0.010 + | 0.110 + | 0.130 | 1.00 |=== "#);