From 14d773998a732a49d9867b67b06f9d95704706bc Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:13:58 +0200 Subject: [PATCH 01/55] Introduce new BenchmarkRun struct --- src/benchmark/benchmark_result.rs | 71 +++++++- src/benchmark/mod.rs | 27 ++- src/benchmark/relative_speed.rs | 35 ++-- src/benchmark/scheduler.rs | 22 +-- src/export/csv.rs | 85 ++++------ src/export/markup.rs | 8 +- src/export/tests.rs | 263 +++++++++++------------------- 7 files changed, 247 insertions(+), 264 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 387884d..6a2eea5 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,7 +1,19 @@ use std::collections::BTreeMap; -use serde::ser::SerializeStruct; -use serde::{Serialize, Serializer}; +use serde::Serialize; +use statistical::{mean, median, standard_deviation}; + +use crate::util::{ + min_max::{max, min}, + units::Second, +}; + +#[derive(Debug, Default, Clone, Serialize, PartialEq)] +pub struct BenchmarkRun { + pub wall_clock_time: Second, + // user_time: Second, + // system_time: Second, +} use crate::benchmark::measurement::Measurements; use crate::quantity::{byte, second, Time}; @@ -25,8 +37,21 @@ pub struct BenchmarkResult { /// The full command line, including parameters not used in the command template. pub command_with_unused_parameters: String, - /// Performance measurements and exit statuses for each run - pub measurements: Measurements, + /// Time spent in user mode + pub user: f64, + + /// Time spent in kernel mode + pub system: f64, + + /// All run time measurements + pub runs: Vec, + + /// Maximum memory usage of the process, in bytes + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_usage_byte: Option>, + + /// Exit codes of all command invocations + pub exit_codes: Vec>, /// Parameter values for this benchmark pub parameters: BTreeMap, @@ -95,3 +120,41 @@ impl Serialize for BenchmarkResult { state.end() } } + +impl BenchmarkResult { + fn wall_clock_times(&self) -> Vec { + self.runs.iter().map(|run| run.wall_clock_time).collect() + } + + /// The average run time + pub fn mean(&self) -> Second { + mean(&self.wall_clock_times()) + } + + /// The standard deviation of all run times. Not available if only one run has been performed + pub fn stddev(&self) -> Option { + let times = self.wall_clock_times(); + + let t_mean = mean(×); + if times.len() > 1 { + Some(standard_deviation(×, Some(t_mean))) + } else { + None + } + } + + /// The median run time + pub fn median(&self) -> Second { + median(&self.wall_clock_times()) + } + + /// The minimum run time + pub fn min(&self) -> Second { + min(&self.wall_clock_times()) + } + + /// The maximum run time + pub fn max(&self) -> Second { + max(&self.wall_clock_times()) + } +} diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index caa589f..f5c7e28 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,6 +8,7 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; +use crate::benchmark::benchmark_result::BenchmarkRun; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ @@ -26,6 +27,7 @@ use measurement::{Measurement, Measurements}; use anyhow::{anyhow, Result}; use colored::*; +use statistical::{mean, standard_deviation}; use self::executor::Executor; @@ -369,11 +371,15 @@ impl<'a> Benchmark<'a> { } // Compute statistical quantities - let t_num = measurements.len(); - let t_mean = measurements.time_wall_clock_mean(); - let t_stddev = measurements.stddev(); - let t_min = measurements.min(); - let t_max = measurements.max(); + let t_num = times_real.len(); + let t_mean = mean(×_real); + let t_stddev = if times_real.len() > 1 { + Some(standard_deviation(×_real, Some(t_mean))) + } else { + None + }; + let t_min = min(×_real); + let t_max = max(×_real); let user_mean = measurements.time_user_mean(); let system_mean = measurements.time_system_mean(); @@ -484,7 +490,16 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), command_with_unused_parameters: self.command.get_name_with_unused_parameters(), - measurements, + user: user_mean, + system: system_mean, + runs: times_real + .iter() + .map(|t| BenchmarkRun { + wall_clock_time: *t, + }) + .collect(), + memory_usage_byte: Some(memory_usage_byte), + exit_codes, parameters: self .command .get_parameters() diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index e62a591..fc18ae3 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -31,9 +31,7 @@ impl BenchmarkResultWithRelativeSpeed<'_> { } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { - l.mean_wall_clock_time() - .partial_cmp(&r.mean_wall_clock_time()) - .unwrap_or(Ordering::Equal) + l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { @@ -68,11 +66,9 @@ fn compute_relative_speeds<'a>( } let ratio = match relative_ordering { - Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(), - Ordering::Equal => Ratio::new::(1.0), - Ordering::Greater => { - result.mean_wall_clock_time() / reference.mean_wall_clock_time() - } + Ordering::Less => reference.mean() / result.mean(), + Ordering::Equal => 1.0, + Ordering::Greater => result.mean() / reference.mean(), }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas @@ -117,9 +113,7 @@ pub fn compute_with_check_from_reference<'a>( reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { - if fastest_of(results).mean_wall_clock_time() == Time::zero() - || reference.mean_wall_clock_time() == Time::zero() - { + if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 { return None; } @@ -132,7 +126,7 @@ pub fn compute_with_check( ) -> Option>> { let fastest = fastest_of(results); - if fastest.mean_wall_clock_time() == Time::zero() { + if fastest.mean() == 0.0 { return None; } @@ -153,19 +147,18 @@ pub fn compute<'a>( fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; - use crate::benchmark::measurement::{Measurement, Measurements}; - use crate::quantity::{second, Time}; + use crate::benchmark::benchmark_result::BenchmarkRun; BenchmarkResult { command: name.into(), command_with_unused_parameters: name.into(), - measurements: Measurements { - measurements: vec![Measurement { - time_wall_clock: Time::new::(mean), - time_user: Time::new::(mean), - ..Default::default() - }], - }, + user: mean, + system: 0.0, + runs: vec![BenchmarkRun { + wall_clock_time: mean, + }], + memory_usage_byte: None, + exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index db036d6..56d75c1 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -205,16 +205,11 @@ fn generate_results(args: &[&'static str]) -> Result> { fn scheduler_basic() -> Result<()> { insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#" - command: sleep 0.123 - mean: 0.123 - stddev: 0 - median: 0.123 user: 0 system: 0 - min: 0.123 - max: 0.123 - times: - - 0.123 - - 0.123 + runs: + - wall_clock_time: 0.123 + - wall_clock_time: 0.123 memory_usage_byte: - 0 - 0 @@ -222,16 +217,11 @@ fn scheduler_basic() -> Result<()> { - 0 - 0 - command: sleep 0.456 - mean: 0.456 - stddev: 0 - median: 0.456 user: 0 system: 0 - min: 0.456 - max: 0.456 - times: - - 0.456 - - 0.456 + runs: + - wall_clock_time: 0.456 + - wall_clock_time: 0.456 memory_usage_byte: - 0 - 0 diff --git a/src/export/csv.rs b/src/export/csv.rs index 15cc3a5..27e8bf6 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -44,13 +44,13 @@ impl Exporter for CsvExporter { for res in results { let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; for f in &[ - res.mean_wall_clock_time(), - res.measurements.stddev().unwrap_or_default(), - res.measurements.median(), - res.measurements.time_user_mean(), - res.measurements.time_system_mean(), - res.measurements.min(), - res.measurements.max(), + res.mean(), + res.stddev().unwrap_or(0.0), + res.median(), + res.user, + res.system, + res.min(), + res.max(), ] { fields.push(Cow::Owned( f.format_with_precision(CSV_UNIT, CSV_PRECISION) @@ -67,6 +67,9 @@ impl Exporter for CsvExporter { } } +#[cfg(test)] +use crate::benchmark::benchmark_result::BenchmarkRun; + #[test] fn test_csv() { use crate::benchmark::benchmark_result::Parameter; @@ -82,29 +85,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_a"), command_with_unused_parameters: String::from("command_a"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(7.0), - time_user: Time::new::(7.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 3.0, + system: 4.0, + runs: vec![ + BenchmarkRun { + wall_clock_time: 7.0, }, - Measurement { - time_wall_clock: Time::new::(8.0), - time_user: Time::new::(8.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 8.0, }, - Measurement { - time_wall_clock: Time::new::(12.0), - time_user: Time::new::(12.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 12.0, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert( @@ -127,29 +122,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_b"), command_with_unused_parameters: String::from("command_b"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(17.0), - time_user: Time::new::(17.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 13.0, + system: 14.0, + runs: vec![ + BenchmarkRun { + wall_clock_time: 17.0, }, - Measurement { - time_wall_clock: Time::new::(18.0), - time_user: Time::new::(18.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 18.0, }, - Measurement { - time_wall_clock: Time::new::(19.0), - time_user: Time::new::(19.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 19.0, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert( @@ -180,7 +167,7 @@ fn test_csv() { insta::assert_snapshot!(actual, @r#" command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo - command_a,9.000000,2.645751,8.000000,9.000000,0.000000,7.000000,12.000000,two,one - command_b,18.000000,1.000000,18.000000,18.000000,0.000000,17.000000,19.000000,seven,one + command_a,9,2.6457513110645907,8,3,4,7,12,two,one + command_b,18,1,18,13,14,17,19,seven,one "#); } diff --git a/src/export/markup.rs b/src/export/markup.rs index 0eea533..aac3795 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -53,9 +53,9 @@ pub trait MarkupExporter { let cmd_str = measurement .command_with_unused_parameters .replace('|', "\\|"); - let mean_str = measurement.mean_wall_clock_time().format_value(unit); - let stddev_str = if let Some(stddev) = measurement.measurements.stddev() { - format!(" ± {}", stddev.format_value(unit)) + let mean_str = format_duration_value(measurement.mean(), Some(unit)).0; + let stddev_str = if let Some(stddev) = measurement.stddev() { + format!(" ± {}", format_duration_value(stddev, Some(unit)).0) } else { "".into() }; @@ -116,7 +116,7 @@ pub trait MarkupExporter { fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit { if let Some(first_result) = results.first() { // Use the first BenchmarkResult entry to determine the unit for all entries. - first_result.mean_wall_clock_time().suitable_unit() + format_duration_value(first_result.mean(), None).1 } else { // Default to `Second`. TimeUnit::Second diff --git a/src/export/tests.rs b/src/export/tests.rs index abe340a..527128c 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -1,6 +1,5 @@ use super::Exporter; -use crate::benchmark::benchmark_result::BenchmarkResult; -use crate::benchmark::measurement::{Measurement, Measurements}; +use crate::benchmark::benchmark_result::{BenchmarkResult, BenchmarkRun}; use crate::export::asciidoc::AsciidocExporter; use crate::export::orgmode::OrgmodeExporter; use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero}; @@ -29,57 +28,41 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(0.09), - time_user: Time::new::(0.09), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0011, + runs: vec![ + BenchmarkRun { + wall_clock_time: 0.09, }, - Measurement { - time_wall_clock: Time::new::(0.10), - time_user: Time::new::(0.10), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.10, }, - Measurement { - time_wall_clock: Time::new::(0.14), - time_user: Time::new::(0.14), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.14, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(2.0), - time_user: Time::new::(2.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0012, + runs: vec![ + BenchmarkRun { + wall_clock_time: 2.0, }, - Measurement { - time_wall_clock: Time::new::(3.0), - time_user: Time::new::(3.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 3.0, }, - Measurement { - time_wall_clock: Time::new::(4.0), - time_user: Time::new::(4.0), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 4.0, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -130,57 +113,41 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(2.1), - time_user: Time::new::(2.1), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0012, + runs: vec![ + BenchmarkRun { + wall_clock_time: 2.1, }, - Measurement { - time_wall_clock: Time::new::(2.2), - time_user: Time::new::(2.2), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.2, }, - Measurement { - time_wall_clock: Time::new::(2.3), - time_user: Time::new::(2.3), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.3, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(0.1), - time_user: Time::new::(0.1), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0011, + runs: vec![ + BenchmarkRun { + wall_clock_time: 0.1, }, - Measurement { - time_wall_clock: Time::new::(0.2), - time_user: Time::new::(0.2), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.2, }, - Measurement { - time_wall_clock: Time::new::(0.3), - time_user: Time::new::(0.3), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.3, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -231,57 +198,41 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(2.1), - time_user: Time::new::(2.1), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0012, + runs: vec![ + BenchmarkRun { + wall_clock_time: 2.1, }, - Measurement { - time_wall_clock: Time::new::(2.2), - time_user: Time::new::(2.2), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.2, }, - Measurement { - time_wall_clock: Time::new::(2.3), - time_user: Time::new::(2.3), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.3, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(0.1), - time_user: Time::new::(0.1), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0011, + runs: vec![ + BenchmarkRun { + wall_clock_time: 0.1, }, - Measurement { - time_wall_clock: Time::new::(0.2), - time_user: Time::new::(0.2), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.2, }, - Measurement { - time_wall_clock: Time::new::(0.3), - time_user: Time::new::(0.3), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.3, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -331,69 +282,53 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(2.01), - time_user: Time::new::(2.01), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0012, + runs: vec![ + BenchmarkRun { + wall_clock_time: 2.01, }, - Measurement { - time_wall_clock: Time::new::(2.02), - time_user: Time::new::(2.02), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.02, }, - Measurement { - time_wall_clock: Time::new::(2.03), - time_user: Time::new::(2.03), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 2.03, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - measurements: Measurements::new(vec![ - Measurement { - time_wall_clock: Time::new::(0.11), - time_user: Time::new::(0.11), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + user: 0.0009, + system: 0.0011, + runs: vec![ + BenchmarkRun { + wall_clock_time: 0.11, }, - Measurement { - time_wall_clock: Time::new::(0.12), - time_user: Time::new::(0.12), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.12, }, - Measurement { - time_wall_clock: Time::new::(0.13), - time_user: Time::new::(0.13), - time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), - exit_status: ExitStatus::default(), + BenchmarkRun { + wall_clock_time: 0.13, }, - ]), + ], + memory_usage_byte: None, + exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; - insta::assert_snapshot!(get_output::(&results, Some(TimeUnit::Second), SortOrder::Command), @r#" + insta::assert_snapshot!(get_output::(&results, Some(Unit::Second), SortOrder::Command), @r#" | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | 16.83 ± 1.41 | | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 | "#); - insta::assert_snapshot!(get_output::(&results, Some(TimeUnit::Second), SortOrder::MeanTime), @r#" + insta::assert_snapshot!(get_output::(&results, Some(Unit::Second), SortOrder::MeanTime), @r#" | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | 1.00 | From 50232feb22df2cb9fed1180b4407c7fe5f503fff Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:14:10 +0200 Subject: [PATCH 02/55] Move user and system time --- src/benchmark/benchmark_result.rs | 35 +++++++++++++---- src/benchmark/mod.rs | 9 +++-- src/benchmark/relative_speed.rs | 4 +- src/benchmark/scheduler.rs | 12 ++++-- src/export/csv.rs | 24 ++++++++---- src/export/tests.rs | 64 +++++++++++++++++++++++-------- 6 files changed, 106 insertions(+), 42 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 6a2eea5..5913c83 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -10,9 +10,14 @@ use crate::util::{ #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct BenchmarkRun { + /// Wall clock time measurement pub wall_clock_time: Second, - // user_time: Second, - // system_time: Second, + + /// Time spent in user mode + pub user_time: Second, + + /// Time spent in kernel mode + pub system_time: Second, } use crate::benchmark::measurement::Measurements; @@ -37,12 +42,6 @@ pub struct BenchmarkResult { /// The full command line, including parameters not used in the command template. pub command_with_unused_parameters: String, - /// Time spent in user mode - pub user: f64, - - /// Time spent in kernel mode - pub system: f64, - /// All run time measurements pub runs: Vec, @@ -157,4 +156,24 @@ impl BenchmarkResult { pub fn max(&self) -> Second { max(&self.wall_clock_times()) } + + pub fn user_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.user_time) + .collect::>(), + ) + } + + pub fn system_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.system_time) + .collect::>(), + ) + } } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index f5c7e28..4a23dbf 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -490,12 +490,13 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), command_with_unused_parameters: self.command.get_name_with_unused_parameters(), - user: user_mean, - system: system_mean, runs: times_real .iter() - .map(|t| BenchmarkRun { - wall_clock_time: *t, + .zip(times_user.iter().zip(times_system.iter())) + .map(|(wall_clock_time, (user_time, system_time))| BenchmarkRun { + wall_clock_time: *wall_clock_time, + user_time: *user_time, + system_time: *system_time, }) .collect(), memory_usage_byte: Some(memory_usage_byte), diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index fc18ae3..f20b5de 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -152,10 +152,10 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { BenchmarkResult { command: name.into(), command_with_unused_parameters: name.into(), - user: mean, - system: 0.0, runs: vec![BenchmarkRun { wall_clock_time: mean, + user_time: mean, + system_time: 0., }], memory_usage_byte: None, exit_codes: Vec::new(), diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 56d75c1..a1b1953 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -205,11 +205,13 @@ fn generate_results(args: &[&'static str]) -> Result> { fn scheduler_basic() -> Result<()> { insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#" - command: sleep 0.123 - user: 0 - system: 0 runs: - wall_clock_time: 0.123 + user_time: 0 + system_time: 0 - wall_clock_time: 0.123 + user_time: 0 + system_time: 0 memory_usage_byte: - 0 - 0 @@ -217,11 +219,13 @@ fn scheduler_basic() -> Result<()> { - 0 - 0 - command: sleep 0.456 - user: 0 - system: 0 runs: - wall_clock_time: 0.456 + user_time: 0 + system_time: 0 - wall_clock_time: 0.456 + user_time: 0 + system_time: 0 memory_usage_byte: - 0 - 0 diff --git a/src/export/csv.rs b/src/export/csv.rs index 27e8bf6..c9d81b8 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -47,8 +47,8 @@ impl Exporter for CsvExporter { res.mean(), res.stddev().unwrap_or(0.0), res.median(), - res.user, - res.system, + res.user_mean(), + res.system_mean(), res.min(), res.max(), ] { @@ -85,17 +85,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_a"), command_with_unused_parameters: String::from("command_a"), - user: 3.0, - system: 4.0, runs: vec![ BenchmarkRun { wall_clock_time: 7.0, + user_time: 7.0, + system_time: 0.0, }, BenchmarkRun { wall_clock_time: 8.0, + user_time: 8.0, + system_time: 0.0, }, BenchmarkRun { wall_clock_time: 12.0, + user_time: 12.0, + system_time: 0.0, }, ], memory_usage_byte: None, @@ -122,17 +126,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_b"), command_with_unused_parameters: String::from("command_b"), - user: 13.0, - system: 14.0, runs: vec![ BenchmarkRun { wall_clock_time: 17.0, + user_time: 17.0, + system_time: 0.0, }, BenchmarkRun { wall_clock_time: 18.0, + user_time: 18.0, + system_time: 0.0, }, BenchmarkRun { wall_clock_time: 19.0, + user_time: 19.0, + system_time: 0.0, }, ], memory_usage_byte: None, @@ -167,7 +175,7 @@ fn test_csv() { insta::assert_snapshot!(actual, @r#" command,mean,stddev,median,user,system,min,max,parameter_bar,parameter_foo - command_a,9,2.6457513110645907,8,3,4,7,12,two,one - command_b,18,1,18,13,14,17,19,seven,one + command_a,9,2.6457513110645907,8,9,0,7,12,two,one + command_b,18,1,18,18,0,17,19,seven,one "#); } diff --git a/src/export/tests.rs b/src/export/tests.rs index 527128c..298cd0d 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -28,17 +28,21 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - user: 0.0009, - system: 0.0011, runs: vec![ BenchmarkRun { wall_clock_time: 0.09, + user_time: 0.09, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.10, + user_time: 0.10, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.14, + user_time: 0.14, + system_time: 0., }, ], memory_usage_byte: None, @@ -48,17 +52,21 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - user: 0.0009, - system: 0.0012, runs: vec![ BenchmarkRun { wall_clock_time: 2.0, + user_time: 2.0, + system_time: 0., }, BenchmarkRun { wall_clock_time: 3.0, + user_time: 3.0, + system_time: 0., }, BenchmarkRun { wall_clock_time: 4.0, + user_time: 4.0, + system_time: 0., }, ], memory_usage_byte: None, @@ -113,17 +121,21 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - user: 0.0009, - system: 0.0012, runs: vec![ BenchmarkRun { wall_clock_time: 2.1, + user_time: 2.1, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.2, + user_time: 2.2, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.3, + user_time: 2.3, + system_time: 0., }, ], memory_usage_byte: None, @@ -133,17 +145,21 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - user: 0.0009, - system: 0.0011, runs: vec![ BenchmarkRun { wall_clock_time: 0.1, + user_time: 0.1, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.2, + user_time: 0.2, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.3, + user_time: 0.3, + system_time: 0., }, ], memory_usage_byte: None, @@ -198,17 +214,21 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - user: 0.0009, - system: 0.0012, runs: vec![ BenchmarkRun { wall_clock_time: 2.1, + user_time: 2.1, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.2, + user_time: 2.2, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.3, + user_time: 2.3, + system_time: 0., }, ], memory_usage_byte: None, @@ -218,17 +238,21 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - user: 0.0009, - system: 0.0011, runs: vec![ BenchmarkRun { wall_clock_time: 0.1, + user_time: 0.1, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.2, + user_time: 0.2, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.3, + user_time: 0.3, + system_time: 0., }, ], memory_usage_byte: None, @@ -282,17 +306,21 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 2"), command_with_unused_parameters: String::from("sleep 2"), - user: 0.0009, - system: 0.0012, runs: vec![ BenchmarkRun { wall_clock_time: 2.01, + user_time: 2.01, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.02, + user_time: 2.02, + system_time: 0., }, BenchmarkRun { wall_clock_time: 2.03, + user_time: 2.03, + system_time: 0., }, ], memory_usage_byte: None, @@ -302,17 +330,21 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 0.1"), command_with_unused_parameters: String::from("sleep 0.1"), - user: 0.0009, - system: 0.0011, runs: vec![ BenchmarkRun { wall_clock_time: 0.11, + user_time: 0.11, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.12, + user_time: 0.12, + system_time: 0., }, BenchmarkRun { wall_clock_time: 0.13, + user_time: 0.13, + system_time: 0., }, ], memory_usage_byte: None, From ae0035ad56e949cdc36f74e5faf9689f74cea0f0 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:14:25 +0200 Subject: [PATCH 03/55] Use wait4 instead of getrusage --- src/benchmark/benchmark_result.rs | 13 +++--- src/benchmark/mod.rs | 26 +++++++---- src/benchmark/relative_speed.rs | 4 +- src/benchmark/scheduler.rs | 20 ++++----- src/export/csv.rs | 16 +++++-- src/export/tests.rs | 64 ++++++++++++++++++++------- src/timer/mod.rs | 15 ++++++- src/timer/unix_timer.rs | 73 ++++++++++++++----------------- src/timer/windows_timer.rs | 10 ++--- 9 files changed, 143 insertions(+), 98 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 5913c83..5d0e638 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -18,6 +18,12 @@ pub struct BenchmarkRun { /// Time spent in kernel mode pub system_time: Second, + + /// Maximum memory usage of the process, in bytes + pub memory_usage_byte: u64, + + /// Exit codes of the process + pub exit_code: Option, } use crate::benchmark::measurement::Measurements; @@ -45,13 +51,6 @@ pub struct BenchmarkResult { /// All run time measurements pub runs: Vec, - /// Maximum memory usage of the process, in bytes - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_usage_byte: Option>, - - /// Exit codes of all command invocations - pub exit_codes: Vec>, - /// Parameter values for this benchmark pub parameters: BTreeMap, } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 4a23dbf..f4cf90a 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -492,15 +492,25 @@ impl<'a> Benchmark<'a> { command_with_unused_parameters: self.command.get_name_with_unused_parameters(), runs: times_real .iter() - .zip(times_user.iter().zip(times_system.iter())) - .map(|(wall_clock_time, (user_time, system_time))| BenchmarkRun { - wall_clock_time: *wall_clock_time, - user_time: *user_time, - system_time: *system_time, - }) + .zip(times_user.iter()) + .zip(times_system.iter()) + .zip(memory_usage_byte.iter()) + .zip(exit_codes.iter()) + .map( + |( + (((wall_clock_time, user_time), system_time), memory_usage_byte), + exit_code, + )| { + BenchmarkRun { + wall_clock_time: *wall_clock_time, + user_time: *user_time, + system_time: *system_time, + memory_usage_byte: *memory_usage_byte, + exit_code: *exit_code, + } + }, + ) .collect(), - memory_usage_byte: Some(memory_usage_byte), - exit_codes, parameters: self .command .get_parameters() diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index f20b5de..ac8b4e6 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -156,9 +156,9 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { wall_clock_time: mean, user_time: mean, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }], - memory_usage_byte: None, - exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index a1b1953..2944c22 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -209,29 +209,25 @@ fn scheduler_basic() -> Result<()> { - wall_clock_time: 0.123 user_time: 0 system_time: 0 + memory_usage_byte: 0 + exit_code: 0 - wall_clock_time: 0.123 user_time: 0 system_time: 0 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + memory_usage_byte: 0 + exit_code: 0 - command: sleep 0.456 runs: - wall_clock_time: 0.456 user_time: 0 system_time: 0 + memory_usage_byte: 0 + exit_code: 0 - wall_clock_time: 0.456 user_time: 0 system_time: 0 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + memory_usage_byte: 0 + exit_code: 0 "#); Ok(()) diff --git a/src/export/csv.rs b/src/export/csv.rs index c9d81b8..ce54e3a 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -90,20 +90,24 @@ fn test_csv() { wall_clock_time: 7.0, user_time: 7.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 8.0, user_time: 8.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 12.0, user_time: 12.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert( @@ -131,20 +135,24 @@ fn test_csv() { wall_clock_time: 17.0, user_time: 17.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 18.0, user_time: 18.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 19.0, user_time: 19.0, system_time: 0.0, + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: { let mut params = BTreeMap::new(); params.insert( diff --git a/src/export/tests.rs b/src/export/tests.rs index 298cd0d..1366e50 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -33,20 +33,24 @@ fn test_markup_export_auto_ms() { wall_clock_time: 0.09, user_time: 0.09, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.10, user_time: 0.10, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.14, user_time: 0.14, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -57,20 +61,24 @@ fn test_markup_export_auto_ms() { wall_clock_time: 2.0, user_time: 2.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 3.0, user_time: 3.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 4.0, user_time: 4.0, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -126,20 +134,24 @@ fn test_markup_export_auto_s() { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -150,20 +162,24 @@ fn test_markup_export_auto_s() { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -219,20 +235,24 @@ fn test_markup_export_manual_ms() { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -243,20 +263,24 @@ fn test_markup_export_manual_ms() { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; @@ -311,20 +335,24 @@ fn test_markup_export_manual_s() { wall_clock_time: 2.01, user_time: 2.01, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.02, user_time: 2.02, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 2.03, user_time: 2.03, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, BenchmarkResult { @@ -335,20 +363,24 @@ fn test_markup_export_manual_s() { wall_clock_time: 0.11, user_time: 0.11, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.12, user_time: 0.12, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, BenchmarkRun { wall_clock_time: 0.13, user_time: 0.13, system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), }, ], - memory_usage_byte: None, - exit_codes: vec![Some(0), Some(0), Some(0)], parameters: BTreeMap::new(), }, ]; diff --git a/src/timer/mod.rs b/src/timer/mod.rs index be00286..9b7df54 100644 --- a/src/timer/mod.rs +++ b/src/timer/mod.rs @@ -24,6 +24,17 @@ use std::process::{ChildStdout, Command}; use anyhow::Result; +/// Used to indicate the result of running a command +#[derive(Debug, Copy, Clone)] +pub struct TimerResult { + pub time_real: Second, + pub time_user: Second, + pub time_system: Second, + pub memory_usage_byte: u64, + /// The exit status of the process + pub status: ExitStatus, +} + /// Discard the output of a child process. fn discard(output: ChildStdout) { const CHUNK_SIZE: usize = 64 << 10; @@ -82,8 +93,8 @@ pub fn execute_and_measure(mut command: Command) -> Result { discard(output); } - let (time_user, time_system, peak_memory_usage, exit_status) = cpu_timer.stop(child)?; - let time_wall_clock = wallclock_timer.stop(); + let (status, time_user, time_system, memory_usage_byte) = cpu_timer.stop(child)?; + let time_real = wallclock_timer.stop(); Ok(Measurement { time_wall_clock, diff --git a/src/timer/unix_timer.rs b/src/timer/unix_timer.rs index 699a541..b76438c 100644 --- a/src/timer/unix_timer.rs +++ b/src/timer/unix_timer.rs @@ -1,5 +1,6 @@ #![cfg(not(windows))] +use std::convert::TryFrom; use std::io; use std::mem::MaybeUninit; use std::os::unix::process::ExitStatusExt; @@ -7,26 +8,24 @@ use std::process::{Child, ExitStatus}; use anyhow::Result; -use crate::quantity::{byte, kibibyte, microsecond, second, Information, Time}; +use crate::util::units::Second; #[derive(Debug, Copy, Clone)] struct ResourceUsage { /// Total amount of time spent executing in user mode - pub time_user: Time, + pub user_usec: Second, /// Total amount of time spent executing in kernel mode - pub time_system: Time, + pub system_usec: Second, /// Maximum amount of memory used by the process, in bytes - pub memory_usage: Information, + pub memory_usage_byte: i64, } #[allow(clippy::useless_conversion)] -fn convert_timeval(tv: libc::timeval) -> Time { - let sec = tv.tv_sec as f64; - let usec = tv.tv_usec as f64; - - Time::new::(sec) + Time::new::(usec) +fn timeval_to_seconds(tv: libc::timeval) -> Second { + const MICROSEC_PER_SEC: i64 = 1000 * 1000; + (i64::from(tv.tv_sec) * MICROSEC_PER_SEC + i64::from(tv.tv_usec)) as f64 * 1e-6 } #[allow(clippy::useless_conversion)] @@ -37,37 +36,29 @@ fn wait4(mut child: Child) -> io::Result<(ExitStatus, ResourceUsage)> { let mut status = 0; let mut rusage = MaybeUninit::zeroed(); - loop { - let result = unsafe { libc::wait4(pid, &mut status, 0, rusage.as_mut_ptr()) }; - if result >= 0 { - break; - } - let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); - } - } + let result = unsafe { libc::wait4(pid, &mut status, 0, rusage.as_mut_ptr()) }; - let rusage = unsafe { rusage.assume_init() }; - - let memory_usage_byte = if cfg!(target_os = "macos") || cfg!(target_os = "ios") { - // Linux and *BSD return the value in KibiBytes, Darwin flavors in bytes - // A f64 can represent integers up to 2^53 exactly, so we can represent - // all values up to 2^53 bytes = 8 PiB exactly. Beyond that, our precision - // is larger than 1 byte, but that's considered acceptable. - Information::new::(rusage.ru_maxrss as f64) + if result < 0 { + Err(io::Error::last_os_error()) } else { - Information::new::(rusage.ru_maxrss as f64) - }; + let rusage = unsafe { rusage.assume_init() }; - Ok(( - ExitStatus::from_raw(status), - ResourceUsage { - time_user: convert_timeval(rusage.ru_utime), - time_system: convert_timeval(rusage.ru_stime), - memory_usage: memory_usage_byte.into(), - }, - )) + let memory_usage_byte = if cfg!(target_os = "macos") || cfg!(target_os = "ios") { + // Linux and *BSD return the value in KibiBytes, Darwin flavors in bytes + rusage.ru_maxrss + } else { + rusage.ru_maxrss * 1024 + }; + + Ok(( + ExitStatus::from_raw(status), + ResourceUsage { + user_usec: timeval_to_seconds(rusage.ru_utime), + system_usec: timeval_to_seconds(rusage.ru_stime), + memory_usage_byte: memory_usage_byte.into(), + }, + )) + } } pub struct CPUTimer {} @@ -77,13 +68,13 @@ impl CPUTimer { Self {} } - pub fn stop(&self, child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, child: Child) -> Result<(ExitStatus, Second, Second, u64)> { let (status, usage) = wait4(child)?; Ok(( - usage.time_user, - usage.time_system, - usage.memory_usage, status, + usage.user_usec, + usage.system_usec, + u64::try_from(usage.memory_usage_byte).unwrap_or(0), )) } } diff --git a/src/timer/windows_timer.rs b/src/timer/windows_timer.rs index e48e288..5c380dc 100644 --- a/src/timer/windows_timer.rs +++ b/src/timer/windows_timer.rs @@ -84,7 +84,7 @@ impl CPUTimer { Self { job_object } } - pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, mut child: Child) -> Result<(ExitStatus, Second, Second, u64)> { let status = child.wait()?; let mut job_object_info = @@ -113,12 +113,10 @@ impl CPUTimer { // The `TotalKernelTime` is "The total amount of kernel-mode execution time // for all active processes associated with the job, as well as all terminated // processes no longer associated with the job, in 100-nanosecond ticks." - let system_time = - Time::new::((job_object_info.TotalKernelTime as f64) * 100.0); - - Ok((user_time, system_time, Information::zero(), status)) + let kernel: i64 = job_object_info.TotalKernelTime / HUNDRED_NS_PER_MS; + Ok((status, user as f64 * 1e-6, kernel as f64 * 1e-6, 0)) } else { - Ok((Time::zero(), Time::zero(), Information::zero(), status)) + Ok((status, 0.0, 0.0, 0)) } } } From af4422d2126f6b97fa687bec2d3970aa29d65330 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:14:38 +0200 Subject: [PATCH 04/55] Remove command_with_unused_parameters field --- src/benchmark/benchmark_result.rs | 100 +++++++++--------------------- src/benchmark/mod.rs | 5 +- src/benchmark/relative_speed.rs | 1 - src/benchmark/scheduler.rs | 10 +-- src/export/csv.rs | 4 +- src/export/markup.rs | 2 +- src/export/tests.rs | 8 --- 7 files changed, 38 insertions(+), 92 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 5d0e638..9210501 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -26,11 +26,15 @@ pub struct BenchmarkRun { pub exit_code: Option, } -use crate::benchmark::measurement::Measurements; -use crate::quantity::{byte, second, Time}; -use crate::util::exit_code::extract_exit_code; +#[derive(Debug, Default, Clone, Serialize, PartialEq)] +pub struct Parameter { + pub value: String, + pub is_unused: bool, +} -/// Parameter value and whether it was used in the command line template +/// Set of values that will be exported. +// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the +// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. #[derive(Debug, Default, Clone, Serialize, PartialEq)] #[serde(transparent)] pub struct Parameter { @@ -45,80 +49,14 @@ pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, - /// The full command line, including parameters not used in the command template. - pub command_with_unused_parameters: String, - /// All run time measurements pub runs: Vec, /// Parameter values for this benchmark + #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, } -impl BenchmarkResult { - /// The average wall clock time - pub fn mean_wall_clock_time(&self) -> Time { - self.measurements.time_wall_clock_mean() - } -} - -// Preserve the existing export format while storing typed measurements internally. -impl Serialize for BenchmarkResult { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut state = serializer.serialize_struct( - "BenchmarkResult", - if self.parameters.is_empty() { 11 } else { 12 }, - )?; - state.serialize_field("command", &self.command)?; - state.serialize_field("mean", &self.mean_wall_clock_time().get::())?; - state.serialize_field( - "stddev", - &self.measurements.stddev().map(|time| time.get::()), - )?; - state.serialize_field("median", &self.measurements.median().get::())?; - state.serialize_field("user", &self.measurements.time_user_mean().get::())?; - state.serialize_field( - "system", - &self.measurements.time_system_mean().get::(), - )?; - state.serialize_field("min", &self.measurements.min().get::())?; - state.serialize_field("max", &self.measurements.max().get::())?; - state.serialize_field( - "times", - &self - .measurements - .wall_clock_times() - .map(|time| time.get::()) - .collect::>(), - )?; - state.serialize_field( - "memory_usage_byte", - &self - .measurements - .measurements - .iter() - .map(|measurement| measurement.peak_memory_usage.get::() as u64) - .collect::>(), - )?; - state.serialize_field( - "exit_codes", - &self - .measurements - .measurements - .iter() - .map(|measurement| extract_exit_code(measurement.exit_status)) - .collect::>(), - )?; - if !self.parameters.is_empty() { - state.serialize_field("parameters", &self.parameters)?; - } - state.end() - } -} - impl BenchmarkResult { fn wall_clock_times(&self) -> Vec { self.runs.iter().map(|run| run.wall_clock_time).collect() @@ -175,4 +113,24 @@ impl BenchmarkResult { .collect::>(), ) } + + /// The full command line of the program that is being benchmarked, possibly including a list of + /// parameters that were not used in the command line template. + pub fn command_with_unused_parameters(&self) -> String { + let parameters = self + .parameters + .iter() + .filter(|(_, parameter)| parameter.is_unused) + .fold(String::new(), |output, (name, parameter)| { + output + &format!("{name} = {value}, ", value = parameter.value) + }); + let parameters = parameters.trim_end_matches(", "); + let parameters = if parameters.is_empty() { + "".into() + } else { + format!(" ({parameters})") + }; + + format!("{}{}", self.command, parameters) + } } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index f4cf90a..db9a550 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,7 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; -use crate::benchmark::benchmark_result::BenchmarkRun; +use crate::benchmark::benchmark_result::{BenchmarkRun, Parameter}; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ @@ -489,7 +489,6 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), - command_with_unused_parameters: self.command.get_name_with_unused_parameters(), runs: times_real .iter() .zip(times_user.iter()) @@ -520,7 +519,7 @@ impl<'a> Benchmark<'a> { name.to_string(), Parameter { value: value.to_string(), - is_unused: self.command.is_parameter_unused(name), + is_unused: self.command.is_parameter_unused(&name), }, ) }) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index ac8b4e6..71d63da 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -151,7 +151,6 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { BenchmarkResult { command: name.into(), - command_with_unused_parameters: name.into(), runs: vec![BenchmarkRun { wall_clock_time: mean, user_time: mean, diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index 2944c22..e431007 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -92,8 +92,8 @@ impl<'a> Scheduler<'a> { console_writeln!( stdout, " {} ran", - reference.result.command_with_unused_parameters.cyan() - )?; + reference.result.command_with_unused_parameters().cyan() + ); for item in others { let stddev = if let Some(stddev) = item.relative_speed_stddev { @@ -122,8 +122,8 @@ impl<'a> Scheduler<'a> { stdout, "{} {}", comparator, - item.result.command_with_unused_parameters.magenta() - )?; + &item.result.command_with_unused_parameters().magenta() + ); } } SortOrder::Command => { @@ -146,7 +146,7 @@ impl<'a> Scheduler<'a> { } else { " ".into() }, - item.result.command_with_unused_parameters, + item.result.command_with_unused_parameters(), relationship, )?; } diff --git a/src/export/csv.rs b/src/export/csv.rs index ce54e3a..fcd782d 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -68,7 +68,7 @@ impl Exporter for CsvExporter { } #[cfg(test)] -use crate::benchmark::benchmark_result::BenchmarkRun; +use crate::benchmark::benchmark_result::{BenchmarkRun, Parameter}; #[test] fn test_csv() { @@ -84,7 +84,6 @@ fn test_csv() { let results = vec![ BenchmarkResult { command: String::from("command_a"), - command_with_unused_parameters: String::from("command_a"), runs: vec![ BenchmarkRun { wall_clock_time: 7.0, @@ -129,7 +128,6 @@ fn test_csv() { }, BenchmarkResult { command: String::from("command_b"), - command_with_unused_parameters: String::from("command_b"), runs: vec![ BenchmarkRun { wall_clock_time: 17.0, diff --git a/src/export/markup.rs b/src/export/markup.rs index aac3795..e201a2e 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -51,7 +51,7 @@ pub trait MarkupExporter { let measurement = &entry.result; // prepare data row strings let cmd_str = measurement - .command_with_unused_parameters + .command_with_unused_parameters() .replace('|', "\\|"); let mean_str = format_duration_value(measurement.mean(), Some(unit)).0; let stddev_str = if let Some(stddev) = measurement.stddev() { diff --git a/src/export/tests.rs b/src/export/tests.rs index 1366e50..2c303d2 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -27,7 +27,6 @@ fn test_markup_export_auto_ms() { let results = [ BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), runs: vec![ BenchmarkRun { wall_clock_time: 0.09, @@ -55,7 +54,6 @@ fn test_markup_export_auto_ms() { }, BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), runs: vec![ BenchmarkRun { wall_clock_time: 2.0, @@ -128,7 +126,6 @@ fn test_markup_export_auto_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), runs: vec![ BenchmarkRun { wall_clock_time: 2.1, @@ -156,7 +153,6 @@ fn test_markup_export_auto_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), runs: vec![ BenchmarkRun { wall_clock_time: 0.1, @@ -229,7 +225,6 @@ fn test_markup_export_manual_ms() { let timing_results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), runs: vec![ BenchmarkRun { wall_clock_time: 2.1, @@ -257,7 +252,6 @@ fn test_markup_export_manual_ms() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), runs: vec![ BenchmarkRun { wall_clock_time: 0.1, @@ -329,7 +323,6 @@ fn test_markup_export_manual_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), runs: vec![ BenchmarkRun { wall_clock_time: 2.01, @@ -357,7 +350,6 @@ fn test_markup_export_manual_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), runs: vec![ BenchmarkRun { wall_clock_time: 0.11, From c20fcc0bda8045bee6e71a88d078fed6a83882c8 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:14:51 +0200 Subject: [PATCH 05/55] Real => wall clock --- src/benchmark/benchmark_result.rs | 12 +++--- src/benchmark/executor.rs | 67 +++++++++++++++++++++---------- src/benchmark/mod.rs | 66 +++++++++++++++--------------- src/benchmark/relative_speed.rs | 4 +- src/export/csv.rs | 14 +++---- src/export/tests.rs | 50 +++++++++++------------ src/timer/mod.rs | 6 +-- 7 files changed, 122 insertions(+), 97 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 9210501..4712032 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -8,8 +8,9 @@ use crate::util::{ units::Second, }; +/// Performance metrics and exit codes for each run #[derive(Debug, Default, Clone, Serialize, PartialEq)] -pub struct BenchmarkRun { +pub struct Run { /// Wall clock time measurement pub wall_clock_time: Second, @@ -26,15 +27,14 @@ pub struct BenchmarkRun { pub exit_code: Option, } +/// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Parameter { pub value: String, pub is_unused: bool, } -/// Set of values that will be exported. -// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the -// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate. +/// Meta data and performance metrics for a single benchmark #[derive(Debug, Default, Clone, Serialize, PartialEq)] #[serde(transparent)] pub struct Parameter { @@ -49,8 +49,8 @@ pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, - /// All run time measurements - pub runs: Vec, + /// Performance metrics and exit codes for each run + pub runs: Vec, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index e450d40..afc8405 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -144,7 +144,17 @@ impl Executor for RawExecutor<'_> { &self.options.command_input_policy, output_policy, &command.get_command_line(), - ) + )?; + + Ok(( + TimingResult { + time_wall_clock: result.time_wall_clock, + time_user: result.time_user, + time_system: result.time_system, + memory_usage_byte: result.memory_usage_byte, + }, + result.status, + )) } fn calibrate(&mut self) -> Result<()> { @@ -204,12 +214,20 @@ impl Executor for ShellExecutor<'_> { // Subtract shell spawning time if let Some(spawning_time) = self.shell_spawning_time { result.time_wall_clock = - (result.time_wall_clock - spawning_time.time_wall_clock).max(Time::zero()); - result.time_user = (result.time_user - spawning_time.time_user).max(Time::zero()); - result.time_system = (result.time_system - spawning_time.time_system).max(Time::zero()); + (result.time_wall_clock - spawning_time.time_wall_clock).max(0.0); + result.time_user = (result.time_user - spawning_time.time_user).max(0.0); + result.time_system = (result.time_system - spawning_time.time_system).max(0.0); } - Ok(result) + Ok(( + TimingResult { + time_wall_clock: result.time_wall_clock, + time_user: result.time_user, + time_system: result.time_system, + memory_usage_byte: result.memory_usage_byte, + }, + result.status, + )) } /// Measure the average shell spawning time @@ -225,7 +243,9 @@ impl Executor for ShellExecutor<'_> { None }; - let mut measurements = Measurements::default(); + let mut times_wall_clock: Vec = vec![]; + let mut times_user: Vec = vec![]; + let mut times_system: Vec = vec![]; for _ in 0..COUNT { // Just run the shell without any command @@ -249,8 +269,10 @@ impl Executor for ShellExecutor<'_> { shell_cmd ); } - Ok(r) => { - measurements.push(r); + Ok((r, _)) => { + times_wall_clock.push(r.time_wall_clock); + times_user.push(r.time_user); + times_system.push(r.time_system); } } @@ -263,18 +285,17 @@ impl Executor for ShellExecutor<'_> { bar.finish_and_clear() } - self.shell_spawning_time = Some(Measurement { - time_wall_clock: measurements.time_wall_clock_mean(), - time_user: measurements.time_user_mean(), - time_system: measurements.time_system_mean(), - peak_memory_usage: measurements.peak_memory_usage_mean(), - exit_status: ExitStatus::default(), + self.shell_spawning_time = Some(TimingResult { + time_wall_clock: mean(×_wall_clock), + time_user: mean(×_user), + time_system: mean(×_system), + memory_usage_byte: 0, }); Ok(()) } - fn time_overhead(&self) -> Time { + fn time_overhead(&self) -> Second { self.shell_spawning_time.unwrap().time_wall_clock } } @@ -321,13 +342,15 @@ impl Executor for MockExecutor { ExitStatus::from_raw(0) }; - Ok(Measurement { - time_wall_clock: Self::extract_time(command.get_command_line()), - time_user: Time::zero(), - time_system: Time::zero(), - peak_memory_usage: Information::zero(), - exit_status, - }) + Ok(( + TimingResult { + time_wall_clock: Self::extract_time(command.get_command_line()), + time_user: 0.0, + time_system: 0.0, + memory_usage_byte: 0, + }, + status, + )) } fn calibrate(&mut self) -> Result<()> { diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index db9a550..88347ec 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,7 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; -use crate::benchmark::benchmark_result::{BenchmarkRun, Parameter}; +use crate::benchmark::benchmark_result::{Parameter, Run}; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ @@ -168,7 +168,11 @@ impl<'a> Benchmark<'a> { )?; } - let mut measurements = Measurements::default(); + let mut times_wall_clock: Vec = vec![]; + let mut times_user: Vec = vec![]; + let mut times_system: Vec = vec![]; + let mut memory_usage_byte: Vec = vec![]; + let mut exit_codes: Vec> = vec![]; let mut all_succeeded = true; let output_policy = &self.options.command_output_policies[self.number]; @@ -260,9 +264,8 @@ impl<'a> Benchmark<'a> { None }; - let benchmark_iteration = BenchmarkIteration::Benchmark(0); - let preparation_result = run_preparation_command(benchmark_iteration)?; - let preparation_overhead = preparation_result.map_or(Time::zero(), |res| { + let preparation_result = run_preparation_command()?; + let preparation_overhead = preparation_result.map_or(0.0, |res| { res.time_wall_clock + self.executor.time_overhead() }); @@ -290,8 +293,8 @@ impl<'a> Benchmark<'a> { ); } - let conclusion_result = run_conclusion_command(benchmark_iteration)?; - let conclusion_overhead = conclusion_result.map_or(Time::zero(), |res| { + let conclusion_result = run_conclusion_command()?; + let conclusion_overhead = conclusion_result.map_or(0.0, |res| { res.time_wall_clock + self.executor.time_overhead() }); @@ -317,7 +320,11 @@ impl<'a> Benchmark<'a> { let count_remaining = count - 1; // Save the first result - measurements.push(res); + times_wall_clock.push(res.time_wall_clock); + times_user.push(res.time_user); + times_system.push(res.time_system); + memory_usage_byte.push(res.memory_usage_byte); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -332,12 +339,7 @@ impl<'a> Benchmark<'a> { let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1); let msg = { - let t_wall_clock_mean = measurements.time_wall_clock_mean(); - let time_unit = self - .options - .time_unit - .unwrap_or(t_wall_clock_mean.suitable_unit()); - let mean = t_wall_clock_mean.format(time_unit); + let mean = format_duration(mean(×_wall_clock), self.options.time_unit); format!("Current estimate: {}", mean.to_string().green()) }; @@ -355,7 +357,11 @@ impl<'a> Benchmark<'a> { )?; let success = res.exit_status.success(); - measurements.push(res); + times_wall_clock.push(res.time_wall_clock); + times_user.push(res.time_user); + times_system.push(res.time_system); + memory_usage_byte.push(res.memory_usage_byte); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -371,15 +377,15 @@ impl<'a> Benchmark<'a> { } // Compute statistical quantities - let t_num = times_real.len(); - let t_mean = mean(×_real); - let t_stddev = if times_real.len() > 1 { - Some(standard_deviation(×_real, Some(t_mean))) + let t_num = times_wall_clock.len(); + let t_mean = mean(×_wall_clock); + let t_stddev = if times_wall_clock.len() > 1 { + Some(standard_deviation(×_wall_clock, Some(t_mean))) } else { None }; - let t_min = min(×_real); - let t_max = max(×_real); + let t_min = min(×_wall_clock); + let t_max = max(×_wall_clock); let user_mean = measurements.time_user_mean(); let system_mean = measurements.time_system_mean(); @@ -395,10 +401,8 @@ impl<'a> Benchmark<'a> { let system_str = system_mean.format(time_unit); if self.options.output_style != OutputStyleOption::Disabled { - let mut stdout = io::stdout().lock(); - if measurements.len() == 1 { - console_writeln!( - stdout, + if times_wall_clock.len() == 1 { + println!( " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", "abs".green().bold(), mean_str.green().bold(), @@ -437,9 +441,7 @@ impl<'a> Benchmark<'a> { // Check execution time if matches!(self.options.executor_kind, ExecutorKind::Shell(_)) - && measurements - .wall_clock_times() - .any(|t| t < MIN_EXECUTION_TIME) + && times_wall_clock.iter().any(|&t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } @@ -450,7 +452,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = measurements.modified_zscores(); + let scores = modified_zscores(×_wall_clock); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, @@ -465,7 +467,7 @@ impl<'a> Benchmark<'a> { if scores[0] > OUTLIER_THRESHOLD { warnings.push(Warnings::SlowInitialRun( - measurements.wall_clock_times().next().unwrap(), + times_wall_clock[0], outlier_warning_options, )); } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) { @@ -489,7 +491,7 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), - runs: times_real + runs: times_wall_clock .iter() .zip(times_user.iter()) .zip(times_system.iter()) @@ -500,7 +502,7 @@ impl<'a> Benchmark<'a> { (((wall_clock_time, user_time), system_time), memory_usage_byte), exit_code, )| { - BenchmarkRun { + Run { wall_clock_time: *wall_clock_time, user_time: *user_time, system_time: *system_time, diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 71d63da..2dc300c 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -147,11 +147,11 @@ pub fn compute<'a>( fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; - use crate::benchmark::benchmark_result::BenchmarkRun; + use crate::benchmark::benchmark_result::Run; BenchmarkResult { command: name.into(), - runs: vec![BenchmarkRun { + runs: vec![Run { wall_clock_time: mean, user_time: mean, system_time: 0., diff --git a/src/export/csv.rs b/src/export/csv.rs index fcd782d..4c961ce 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -68,7 +68,7 @@ impl Exporter for CsvExporter { } #[cfg(test)] -use crate::benchmark::benchmark_result::{BenchmarkRun, Parameter}; +use crate::benchmark::benchmark_result::{Parameter, Run}; #[test] fn test_csv() { @@ -85,21 +85,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_a"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 7.0, user_time: 7.0, system_time: 0.0, memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 8.0, user_time: 8.0, system_time: 0.0, memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 12.0, user_time: 12.0, system_time: 0.0, @@ -129,21 +129,21 @@ fn test_csv() { BenchmarkResult { command: String::from("command_b"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 17.0, user_time: 17.0, system_time: 0.0, memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 18.0, user_time: 18.0, system_time: 0.0, memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 19.0, user_time: 19.0, system_time: 0.0, diff --git a/src/export/tests.rs b/src/export/tests.rs index 2c303d2..0f3ed15 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -1,5 +1,5 @@ use super::Exporter; -use crate::benchmark::benchmark_result::{BenchmarkResult, BenchmarkRun}; +use crate::benchmark::benchmark_result::{BenchmarkResult, Run}; use crate::export::asciidoc::AsciidocExporter; use crate::export::orgmode::OrgmodeExporter; use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero}; @@ -28,21 +28,21 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 0.1"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 0.09, user_time: 0.09, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.10, user_time: 0.10, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.14, user_time: 0.14, system_time: 0., @@ -55,21 +55,21 @@ fn test_markup_export_auto_ms() { BenchmarkResult { command: String::from("sleep 2"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 2.0, user_time: 2.0, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 3.0, user_time: 3.0, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 4.0, user_time: 4.0, system_time: 0., @@ -127,21 +127,21 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 2"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., @@ -154,21 +154,21 @@ fn test_markup_export_auto_s() { BenchmarkResult { command: String::from("sleep 0.1"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., @@ -226,21 +226,21 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 2"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 2.1, user_time: 2.1, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.2, user_time: 2.2, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.3, user_time: 2.3, system_time: 0., @@ -253,21 +253,21 @@ fn test_markup_export_manual_ms() { BenchmarkResult { command: String::from("sleep 0.1"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 0.1, user_time: 0.1, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.2, user_time: 0.2, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.3, user_time: 0.3, system_time: 0., @@ -324,21 +324,21 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 2"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 2.01, user_time: 2.01, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.02, user_time: 2.02, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 2.03, user_time: 2.03, system_time: 0., @@ -351,21 +351,21 @@ fn test_markup_export_manual_s() { BenchmarkResult { command: String::from("sleep 0.1"), runs: vec![ - BenchmarkRun { + Run { wall_clock_time: 0.11, user_time: 0.11, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.12, user_time: 0.12, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }, - BenchmarkRun { + Run { wall_clock_time: 0.13, user_time: 0.13, system_time: 0., diff --git a/src/timer/mod.rs b/src/timer/mod.rs index 9b7df54..15dbc8b 100644 --- a/src/timer/mod.rs +++ b/src/timer/mod.rs @@ -27,7 +27,7 @@ use anyhow::Result; /// Used to indicate the result of running a command #[derive(Debug, Copy, Clone)] pub struct TimerResult { - pub time_real: Second, + pub time_wall_clock: Second, pub time_user: Second, pub time_system: Second, pub memory_usage_byte: u64, @@ -94,9 +94,9 @@ pub fn execute_and_measure(mut command: Command) -> Result { } let (status, time_user, time_system, memory_usage_byte) = cpu_timer.stop(child)?; - let time_real = wallclock_timer.stop(); + let time_wall_clock = wallclock_timer.stop(); - Ok(Measurement { + Ok(TimerResult { time_wall_clock, time_user, time_system, From f7e6d7286fd8810695ad988cc6c7cee8ed0c6a26 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:15:02 +0200 Subject: [PATCH 06/55] Simplify run creation --- src/benchmark/benchmark_result.rs | 126 +++++++++++++++++++++--------- src/benchmark/mod.rs | 105 +++++++++---------------- src/benchmark/relative_speed.rs | 18 +++-- src/export/csv.rs | 10 +-- src/export/tests.rs | 34 ++++---- 5 files changed, 159 insertions(+), 134 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 4712032..13d954a 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -27,6 +27,87 @@ pub struct Run { pub exit_code: Option, } +#[derive(Debug, Default, Clone, Serialize, PartialEq)] +pub struct Runs { + pub runs: Vec, +} + +impl Runs { + pub fn new(runs: Vec) -> Self { + Self { runs } + } + + pub fn len(&self) -> usize { + self.runs.len() + } + + pub fn is_empty(&self) -> bool { + self.runs.is_empty() + } + + pub fn push(&mut self, run: Run) { + self.runs.push(run); + } + + pub fn wall_clock_times(&self) -> Vec { + self.runs.iter().map(|run| run.wall_clock_time).collect() + } + + /// The average wall clock time + pub fn mean(&self) -> Second { + mean(&self.wall_clock_times()) + } + + /// The standard deviation of all wall clock times. Not available if only one run has been performed + pub fn stddev(&self) -> Option { + let times = self.wall_clock_times(); + + let t_mean = mean(×); + if times.len() > 1 { + Some(standard_deviation(×, Some(t_mean))) + } else { + None + } + } + + /// The median wall clock time + pub fn median(&self) -> Second { + median(&self.wall_clock_times()) + } + + /// The minimum wall clock time + pub fn min(&self) -> Second { + min(&self.wall_clock_times()) + } + + /// The maximum wall clock time + pub fn max(&self) -> Second { + max(&self.wall_clock_times()) + } + + /// The average user time + pub fn user_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.user_time) + .collect::>(), + ) + } + + /// The average system time + pub fn system_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.system_time) + .collect::>(), + ) + } +} + /// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Parameter { @@ -50,7 +131,8 @@ pub struct BenchmarkResult { pub command: String, /// Performance metrics and exit codes for each run - pub runs: Vec, + #[serde(flatten)] + pub runs: Runs, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -58,60 +140,32 @@ pub struct BenchmarkResult { } impl BenchmarkResult { - fn wall_clock_times(&self) -> Vec { - self.runs.iter().map(|run| run.wall_clock_time).collect() - } - - /// The average run time pub fn mean(&self) -> Second { - mean(&self.wall_clock_times()) + self.runs.mean() } - /// The standard deviation of all run times. Not available if only one run has been performed pub fn stddev(&self) -> Option { - let times = self.wall_clock_times(); - - let t_mean = mean(×); - if times.len() > 1 { - Some(standard_deviation(×, Some(t_mean))) - } else { - None - } + self.runs.stddev() } - /// The median run time pub fn median(&self) -> Second { - median(&self.wall_clock_times()) + self.runs.median() } - /// The minimum run time pub fn min(&self) -> Second { - min(&self.wall_clock_times()) + self.runs.min() } - /// The maximum run time pub fn max(&self) -> Second { - max(&self.wall_clock_times()) + self.runs.max() } pub fn user_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.user_time) - .collect::>(), - ) + self.runs.user_mean() } pub fn system_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.system_time) - .collect::>(), - ) + self.runs.system_mean() } /// The full command line of the program that is being benchmarked, possibly including a list of diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 88347ec..4478b58 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,7 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; -use crate::benchmark::benchmark_result::{Parameter, Run}; +use crate::benchmark::benchmark_result::{Parameter, Run, Runs}; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ @@ -21,13 +21,14 @@ use crate::output::progress_bar::{ }; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; -use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero}; -use benchmark_result::{BenchmarkResult, Parameter}; +use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero}; +use crate::util::exit_code::extract_exit_code; +use crate::util::units::Second; +use benchmark_result::BenchmarkResult; use measurement::{Measurement, Measurements}; use anyhow::{anyhow, Result}; use colored::*; -use statistical::{mean, standard_deviation}; use self::executor::Executor; @@ -168,11 +169,7 @@ impl<'a> Benchmark<'a> { )?; } - let mut times_wall_clock: Vec = vec![]; - let mut times_user: Vec = vec![]; - let mut times_system: Vec = vec![]; - let mut memory_usage_byte: Vec = vec![]; - let mut exit_codes: Vec> = vec![]; + let mut runs = Runs::default(); let mut all_succeeded = true; let output_policy = &self.options.command_output_policies[self.number]; @@ -320,11 +317,13 @@ impl<'a> Benchmark<'a> { let count_remaining = count - 1; // Save the first result - times_wall_clock.push(res.time_wall_clock); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + runs.push(Run { + wall_clock_time: res.time_wall_clock, + user_time: res.time_user, + system_time: res.time_system, + memory_usage_byte: res.memory_usage_byte, + exit_code: extract_exit_code(status), + }); all_succeeded = all_succeeded && success; @@ -339,7 +338,7 @@ impl<'a> Benchmark<'a> { let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1); let msg = { - let mean = format_duration(mean(×_wall_clock), self.options.time_unit); + let mean = format_duration(runs.mean(), self.options.time_unit); format!("Current estimate: {}", mean.to_string().green()) }; @@ -357,11 +356,13 @@ impl<'a> Benchmark<'a> { )?; let success = res.exit_status.success(); - times_wall_clock.push(res.time_wall_clock); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + runs.push(Run { + wall_clock_time: res.time_wall_clock, + user_time: res.time_user, + system_time: res.time_system, + memory_usage_byte: res.memory_usage_byte, + exit_code: extract_exit_code(status), + }); all_succeeded = all_succeeded && success; @@ -376,32 +377,17 @@ impl<'a> Benchmark<'a> { bar.finish_and_clear() } - // Compute statistical quantities - let t_num = times_wall_clock.len(); - let t_mean = mean(×_wall_clock); - let t_stddev = if times_wall_clock.len() > 1 { - Some(standard_deviation(×_wall_clock, Some(t_mean))) - } else { - None - }; - let t_min = min(×_wall_clock); - let t_max = max(×_wall_clock); - - let user_mean = measurements.time_user_mean(); - let system_mean = measurements.time_system_mean(); - // Formatting and console output - let time_unit = self.options.time_unit.unwrap_or(t_mean.suitable_unit()); - let mean_str = t_mean.format(time_unit); - let min_str = t_min.format(time_unit); - let max_str = t_max.format(time_unit); - let num_str = format!("{t_num} runs"); + let (mean_str, time_unit) = format_duration_unit(runs.mean(), self.options.time_unit); + let min_str = format_duration(runs.min(), Some(time_unit)); + let max_str = format_duration(runs.max(), Some(time_unit)); + let num_str = format!("{num_runs} runs", num_runs = runs.len()); - let user_str = user_mean.format(time_unit); - let system_str = system_mean.format(time_unit); + let user_str = format_duration(runs.user_mean(), Some(time_unit)); + let system_str = format_duration(runs.system_mean(), Some(time_unit)); if self.options.output_style != OutputStyleOption::Disabled { - if times_wall_clock.len() == 1 { + if runs.len() == 1 { println!( " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", "abs".green().bold(), @@ -411,7 +397,7 @@ impl<'a> Benchmark<'a> { system_str.blue() )?; } else { - let stddev_str = t_stddev.unwrap().format(time_unit); + let stddev_str = format_duration(runs.stddev().unwrap(), Some(time_unit)); console_writeln!( stdout, @@ -441,7 +427,10 @@ impl<'a> Benchmark<'a> { // Check execution time if matches!(self.options.executor_kind, ExecutorKind::Shell(_)) - && times_wall_clock.iter().any(|&t| t < MIN_EXECUTION_TIME) + && runs + .wall_clock_times() + .iter() + .any(|&t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } @@ -452,7 +441,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = modified_zscores(×_wall_clock); + let scores = modified_zscores(&runs.wall_clock_times()); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, @@ -467,7 +456,7 @@ impl<'a> Benchmark<'a> { if scores[0] > OUTLIER_THRESHOLD { warnings.push(Warnings::SlowInitialRun( - times_wall_clock[0], + runs.wall_clock_times()[0], outlier_warning_options, )); } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) { @@ -491,27 +480,7 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), - runs: times_wall_clock - .iter() - .zip(times_user.iter()) - .zip(times_system.iter()) - .zip(memory_usage_byte.iter()) - .zip(exit_codes.iter()) - .map( - |( - (((wall_clock_time, user_time), system_time), memory_usage_byte), - exit_code, - )| { - Run { - wall_clock_time: *wall_clock_time, - user_time: *user_time, - system_time: *system_time, - memory_usage_byte: *memory_usage_byte, - exit_code: *exit_code, - } - }, - ) - .collect(), + runs, parameters: self .command .get_parameters() @@ -521,7 +490,7 @@ impl<'a> Benchmark<'a> { name.to_string(), Parameter { value: value.to_string(), - is_unused: self.command.is_parameter_unused(&name), + is_unused: self.command.is_parameter_unused(name), }, ) }) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 2dc300c..7260295 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -147,17 +147,19 @@ pub fn compute<'a>( fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; - use crate::benchmark::benchmark_result::Run; + use crate::benchmark::benchmark_result::{Run, Runs}; BenchmarkResult { command: name.into(), - runs: vec![Run { - wall_clock_time: mean, - user_time: mean, - system_time: 0., - memory_usage_byte: 1024, - exit_code: Some(0), - }], + runs: Runs { + runs: vec![Run { + wall_clock_time: mean, + user_time: mean, + system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), + }], + }, parameters: BTreeMap::new(), } } diff --git a/src/export/csv.rs b/src/export/csv.rs index 4c961ce..aa8806d 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -68,7 +68,7 @@ impl Exporter for CsvExporter { } #[cfg(test)] -use crate::benchmark::benchmark_result::{Parameter, Run}; +use crate::benchmark::benchmark_result::{Parameter, Run, Runs}; #[test] fn test_csv() { @@ -84,7 +84,7 @@ fn test_csv() { let results = vec![ BenchmarkResult { command: String::from("command_a"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 7.0, user_time: 7.0, @@ -106,7 +106,7 @@ fn test_csv() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: { let mut params = BTreeMap::new(); params.insert( @@ -128,7 +128,7 @@ fn test_csv() { }, BenchmarkResult { command: String::from("command_b"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 17.0, user_time: 17.0, @@ -150,7 +150,7 @@ fn test_csv() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: { let mut params = BTreeMap::new(); params.insert( diff --git a/src/export/tests.rs b/src/export/tests.rs index 0f3ed15..e0c6c0e 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -1,5 +1,5 @@ use super::Exporter; -use crate::benchmark::benchmark_result::{BenchmarkResult, Run}; +use crate::benchmark::benchmark_result::{BenchmarkResult, Run, Runs}; use crate::export::asciidoc::AsciidocExporter; use crate::export::orgmode::OrgmodeExporter; use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero}; @@ -27,7 +27,7 @@ fn test_markup_export_auto_ms() { let results = [ BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.09, user_time: 0.09, @@ -49,12 +49,12 @@ fn test_markup_export_auto_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.0, user_time: 2.0, @@ -76,7 +76,7 @@ fn test_markup_export_auto_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -126,7 +126,7 @@ fn test_markup_export_auto_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.1, user_time: 2.1, @@ -148,12 +148,12 @@ fn test_markup_export_auto_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.1, user_time: 0.1, @@ -175,7 +175,7 @@ fn test_markup_export_auto_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -225,7 +225,7 @@ fn test_markup_export_manual_ms() { let timing_results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.1, user_time: 2.1, @@ -247,12 +247,12 @@ fn test_markup_export_manual_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.1, user_time: 0.1, @@ -274,7 +274,7 @@ fn test_markup_export_manual_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -323,7 +323,7 @@ fn test_markup_export_manual_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.01, user_time: 2.01, @@ -345,12 +345,12 @@ fn test_markup_export_manual_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.11, user_time: 0.11, @@ -372,7 +372,7 @@ fn test_markup_export_manual_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; From 8ba632c95492ca1bc00cd18e71e7b3d905f8603a Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:15:17 +0200 Subject: [PATCH 07/55] More simplifications --- src/benchmark/benchmark_result.rs | 40 ++++++++++--------------------- src/benchmark/mod.rs | 3 ++- src/benchmark/relative_speed.rs | 15 ++++++++---- src/export/csv.rs | 14 +++++------ src/export/markup.rs | 18 +++++++------- 5 files changed, 39 insertions(+), 51 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 13d954a..e1191d7 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -3,9 +3,12 @@ use std::collections::BTreeMap; use serde::Serialize; use statistical::{mean, median, standard_deviation}; -use crate::util::{ - min_max::{max, min}, - units::Second, +use crate::{ + outlier_detection::modified_zscores, + util::{ + min_max::{max, min}, + units::Second, + }, }; /// Performance metrics and exit codes for each run @@ -106,6 +109,10 @@ impl Runs { .collect::>(), ) } + + pub fn modified_zscores(&self) -> Vec { + modified_zscores(&self.wall_clock_times()) + } } /// Parameter value and whether it was used in the command line template @@ -140,34 +147,11 @@ pub struct BenchmarkResult { } impl BenchmarkResult { - pub fn mean(&self) -> Second { + /// The average wall clock time + pub fn mean_wall_clock_time(&self) -> Second { self.runs.mean() } - pub fn stddev(&self) -> Option { - self.runs.stddev() - } - - pub fn median(&self) -> Second { - self.runs.median() - } - - pub fn min(&self) -> Second { - self.runs.min() - } - - pub fn max(&self) -> Second { - self.runs.max() - } - - pub fn user_mean(&self) -> Second { - self.runs.user_mean() - } - - pub fn system_mean(&self) -> Second { - self.runs.system_mean() - } - /// The full command line of the program that is being benchmarked, possibly including a list of /// parameters that were not used in the command line template. pub fn command_with_unused_parameters(&self) -> String { diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 4478b58..b1023d8 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -15,6 +15,7 @@ use crate::options::{ CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, }; use crate::outlier_detection::OUTLIER_THRESHOLD; +use crate::output::format::{format_duration, format_duration_unit}; use crate::output::console_writeln; use crate::output::progress_bar::{ finish_initial_measurement, get_progress_bar, start_initial_measurement, @@ -441,7 +442,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = modified_zscores(&runs.wall_clock_times()); + let scores = runs.modified_zscores(); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 7260295..6628970 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -31,7 +31,9 @@ impl BenchmarkResultWithRelativeSpeed<'_> { } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { - l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal) + l.mean_wall_clock_time() + .partial_cmp(&r.mean_wall_clock_time()) + .unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { @@ -66,9 +68,11 @@ fn compute_relative_speeds<'a>( } let ratio = match relative_ordering { - Ordering::Less => reference.mean() / result.mean(), + Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(), Ordering::Equal => 1.0, - Ordering::Greater => result.mean() / reference.mean(), + Ordering::Greater => { + result.mean_wall_clock_time() / reference.mean_wall_clock_time() + } }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas @@ -113,7 +117,8 @@ pub fn compute_with_check_from_reference<'a>( reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { - if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 { + if fastest_of(results).mean_wall_clock_time() == 0.0 || reference.mean_wall_clock_time() == 0.0 + { return None; } @@ -126,7 +131,7 @@ pub fn compute_with_check( ) -> Option>> { let fastest = fastest_of(results); - if fastest.mean() == 0.0 { + if fastest.mean_wall_clock_time() == 0.0 { return None; } diff --git a/src/export/csv.rs b/src/export/csv.rs index aa8806d..11eb877 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -44,13 +44,13 @@ impl Exporter for CsvExporter { for res in results { let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; for f in &[ - res.mean(), - res.stddev().unwrap_or(0.0), - res.median(), - res.user_mean(), - res.system_mean(), - res.min(), - res.max(), + res.mean_wall_clock_time(), + res.runs.stddev().unwrap_or(0.0), + res.runs.median(), + res.runs.user_mean(), + res.runs.system_mean(), + res.runs.min(), + res.runs.max(), ] { fields.push(Cow::Owned( f.format_with_precision(CSV_UNIT, CSV_PRECISION) diff --git a/src/export/markup.rs b/src/export/markup.rs index e201a2e..9c1caee 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -48,25 +48,23 @@ pub trait MarkupExporter { table.push_str(&self.table_divider(&cells_alignment)); for entry in entries { - let measurement = &entry.result; + let result = &entry.result; // prepare data row strings - let cmd_str = measurement - .command_with_unused_parameters() - .replace('|', "\\|"); - let mean_str = format_duration_value(measurement.mean(), Some(unit)).0; - let stddev_str = if let Some(stddev) = measurement.stddev() { + let cmd_str = result.command_with_unused_parameters().replace('|', "\\|"); + let mean_str = format_duration_value(result.mean_wall_clock_time(), Some(unit)).0; + let stddev_str = if let Some(stddev) = result.runs.stddev() { format!(" ± {}", format_duration_value(stddev, Some(unit)).0) } else { "".into() }; - let min_str = measurement.measurements.min().format_value(unit); - let max_str = measurement.measurements.max().format_value(unit); + let min_str = result.measurements.min().format_value(unit); + let max_str = result.measurements.max().format_value(unit); // The ratio of two zero times is undefined, even if they compare equal. let relative_unavailable = reference_pending || (explicit_reference && !entry.is_reference && entry.relative_ordering == Ordering::Equal - && measurement.mean_wall_clock_time() == Time::zero()); + && result.mean_wall_clock_time() == Time::zero()); let relative = if relative_unavailable { "N/A".to_string() } else { @@ -116,7 +114,7 @@ pub trait MarkupExporter { fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit { if let Some(first_result) = results.first() { // Use the first BenchmarkResult entry to determine the unit for all entries. - format_duration_value(first_result.mean(), None).1 + format_duration_value(first_result.mean_wall_clock_time(), None).1 } else { // Default to `Second`. TimeUnit::Second From d4c6b53e404e7c8390c31356a35186ce5d06e5b4 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:15:34 +0200 Subject: [PATCH 08/55] Runs => Measurements --- src/benchmark/benchmark_result.rs | 119 +----------------------------- src/benchmark/measurement.rs | 111 ++++++++++++++++------------ src/benchmark/mod.rs | 36 ++++----- src/benchmark/relative_speed.rs | 6 +- src/benchmark/scheduler.rs | 4 +- src/export/csv.rs | 33 +++++---- src/export/markup.rs | 2 +- src/export/tests.rs | 67 ++++++++--------- 8 files changed, 146 insertions(+), 232 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index e1191d7..cead751 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,119 +1,8 @@ use std::collections::BTreeMap; use serde::Serialize; -use statistical::{mean, median, standard_deviation}; -use crate::{ - outlier_detection::modified_zscores, - util::{ - min_max::{max, min}, - units::Second, - }, -}; - -/// Performance metrics and exit codes for each run -#[derive(Debug, Default, Clone, Serialize, PartialEq)] -pub struct Run { - /// Wall clock time measurement - pub wall_clock_time: Second, - - /// Time spent in user mode - pub user_time: Second, - - /// Time spent in kernel mode - pub system_time: Second, - - /// Maximum memory usage of the process, in bytes - pub memory_usage_byte: u64, - - /// Exit codes of the process - pub exit_code: Option, -} - -#[derive(Debug, Default, Clone, Serialize, PartialEq)] -pub struct Runs { - pub runs: Vec, -} - -impl Runs { - pub fn new(runs: Vec) -> Self { - Self { runs } - } - - pub fn len(&self) -> usize { - self.runs.len() - } - - pub fn is_empty(&self) -> bool { - self.runs.is_empty() - } - - pub fn push(&mut self, run: Run) { - self.runs.push(run); - } - - pub fn wall_clock_times(&self) -> Vec { - self.runs.iter().map(|run| run.wall_clock_time).collect() - } - - /// The average wall clock time - pub fn mean(&self) -> Second { - mean(&self.wall_clock_times()) - } - - /// The standard deviation of all wall clock times. Not available if only one run has been performed - pub fn stddev(&self) -> Option { - let times = self.wall_clock_times(); - - let t_mean = mean(×); - if times.len() > 1 { - Some(standard_deviation(×, Some(t_mean))) - } else { - None - } - } - - /// The median wall clock time - pub fn median(&self) -> Second { - median(&self.wall_clock_times()) - } - - /// The minimum wall clock time - pub fn min(&self) -> Second { - min(&self.wall_clock_times()) - } - - /// The maximum wall clock time - pub fn max(&self) -> Second { - max(&self.wall_clock_times()) - } - - /// The average user time - pub fn user_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.user_time) - .collect::>(), - ) - } - - /// The average system time - pub fn system_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.system_time) - .collect::>(), - ) - } - - pub fn modified_zscores(&self) -> Vec { - modified_zscores(&self.wall_clock_times()) - } -} +use crate::{benchmark::measurement::Measurements, util::units::Second}; /// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] @@ -137,9 +26,9 @@ pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, - /// Performance metrics and exit codes for each run + /// Performance metric measurements and exit codes for each run #[serde(flatten)] - pub runs: Runs, + pub measurements: Measurements, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -149,7 +38,7 @@ pub struct BenchmarkResult { impl BenchmarkResult { /// The average wall clock time pub fn mean_wall_clock_time(&self) -> Second { - self.runs.mean() + self.measurements.mean() } /// The full command line of the program that is being benchmarked, possibly including a list of diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index da35e9b..2955220 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -1,29 +1,33 @@ -use std::process::ExitStatus; +use serde::Serialize; -use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation}; -use crate::quantity::{Information, Time}; +use statistical::{mean, median, standard_deviation}; -/// Performance measurements and exit status from running a single command -#[derive(Debug, Default, Copy, Clone, PartialEq)] +use crate::util::units::Second; +use crate::{ + outlier_detection::modified_zscores, + util::min_max::{max, min}, +}; + +/// Performance metric measurements and exit code for a single run +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Measurement { - /// Wall clock time - pub time_wall_clock: Time, + /// Elapsed wall clock time (real time) + pub wall_clock_time: Second, /// Time spent in user mode - pub time_user: Time, + pub user_time: Second, /// Time spent in kernel mode - pub time_system: Time, + pub system_time: Second, - /// Maximum amount of memory used - pub peak_memory_usage: Information, + /// Maximum memory usage of the process, in bytes + pub memory_usage_byte: u64, - /// The exit status of the process - pub exit_status: ExitStatus, + /// Exit codes of the process + pub exit_code: Option, } -/// Performance measurements and exit statuses for all runs of a command. -#[derive(Debug, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Measurements { pub measurements: Vec, } @@ -45,54 +49,69 @@ impl Measurements { self.measurements.push(measurement); } - pub fn wall_clock_times(&self) -> impl Iterator + '_ { - self.measurements.iter().map(|m| m.time_wall_clock) + pub fn wall_clock_times(&self) -> Vec { + self.measurements + .iter() + .map(|m| m.wall_clock_time) + .collect() } - /// The average wall clock time. - pub fn time_wall_clock_mean(&self) -> Time { - mean(self.wall_clock_times()) + /// The average wall clock time + pub fn mean(&self) -> Second { + mean(&self.wall_clock_times()) } /// The standard deviation of all wall clock times. Not available if only one run has been performed - pub fn stddev(&self) -> Option