From f7e6d7286fd8810695ad988cc6c7cee8ed0c6a26 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 14:15:02 +0200 Subject: [PATCH] Simplify run creation --- src/benchmark/benchmark_result.rs | 126 +++++++++++++++++++++--------- src/benchmark/mod.rs | 105 +++++++++---------------- src/benchmark/relative_speed.rs | 18 +++-- src/export/csv.rs | 10 +-- src/export/tests.rs | 34 ++++---- 5 files changed, 159 insertions(+), 134 deletions(-) diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 4712032..13d954a 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -27,6 +27,87 @@ pub struct Run { pub exit_code: Option, } +#[derive(Debug, Default, Clone, Serialize, PartialEq)] +pub struct Runs { + pub runs: Vec, +} + +impl Runs { + pub fn new(runs: Vec) -> Self { + Self { runs } + } + + pub fn len(&self) -> usize { + self.runs.len() + } + + pub fn is_empty(&self) -> bool { + self.runs.is_empty() + } + + pub fn push(&mut self, run: Run) { + self.runs.push(run); + } + + pub fn wall_clock_times(&self) -> Vec { + self.runs.iter().map(|run| run.wall_clock_time).collect() + } + + /// The average wall clock time + pub fn mean(&self) -> Second { + mean(&self.wall_clock_times()) + } + + /// The standard deviation of all wall clock times. Not available if only one run has been performed + pub fn stddev(&self) -> Option { + let times = self.wall_clock_times(); + + let t_mean = mean(×); + if times.len() > 1 { + Some(standard_deviation(×, Some(t_mean))) + } else { + None + } + } + + /// The median wall clock time + pub fn median(&self) -> Second { + median(&self.wall_clock_times()) + } + + /// The minimum wall clock time + pub fn min(&self) -> Second { + min(&self.wall_clock_times()) + } + + /// The maximum wall clock time + pub fn max(&self) -> Second { + max(&self.wall_clock_times()) + } + + /// The average user time + pub fn user_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.user_time) + .collect::>(), + ) + } + + /// The average system time + pub fn system_mean(&self) -> Second { + mean( + &self + .runs + .iter() + .map(|run| run.system_time) + .collect::>(), + ) + } +} + /// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Parameter { @@ -50,7 +131,8 @@ pub struct BenchmarkResult { pub command: String, /// Performance metrics and exit codes for each run - pub runs: Vec, + #[serde(flatten)] + pub runs: Runs, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -58,60 +140,32 @@ pub struct BenchmarkResult { } impl BenchmarkResult { - fn wall_clock_times(&self) -> Vec { - self.runs.iter().map(|run| run.wall_clock_time).collect() - } - - /// The average run time pub fn mean(&self) -> Second { - mean(&self.wall_clock_times()) + self.runs.mean() } - /// The standard deviation of all run times. Not available if only one run has been performed pub fn stddev(&self) -> Option { - let times = self.wall_clock_times(); - - let t_mean = mean(×); - if times.len() > 1 { - Some(standard_deviation(×, Some(t_mean))) - } else { - None - } + self.runs.stddev() } - /// The median run time pub fn median(&self) -> Second { - median(&self.wall_clock_times()) + self.runs.median() } - /// The minimum run time pub fn min(&self) -> Second { - min(&self.wall_clock_times()) + self.runs.min() } - /// The maximum run time pub fn max(&self) -> Second { - max(&self.wall_clock_times()) + self.runs.max() } pub fn user_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.user_time) - .collect::>(), - ) + self.runs.user_mean() } pub fn system_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.system_time) - .collect::>(), - ) + self.runs.system_mean() } /// The full command line of the program that is being benchmarked, possibly including a list of diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 88347ec..4478b58 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,7 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; -use crate::benchmark::benchmark_result::{Parameter, Run}; +use crate::benchmark::benchmark_result::{Parameter, Run, Runs}; use crate::benchmark::executor::BenchmarkIteration; use crate::command::Command; use crate::options::{ @@ -21,13 +21,14 @@ use crate::output::progress_bar::{ }; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; -use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero}; -use benchmark_result::{BenchmarkResult, Parameter}; +use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero}; +use crate::util::exit_code::extract_exit_code; +use crate::util::units::Second; +use benchmark_result::BenchmarkResult; use measurement::{Measurement, Measurements}; use anyhow::{anyhow, Result}; use colored::*; -use statistical::{mean, standard_deviation}; use self::executor::Executor; @@ -168,11 +169,7 @@ impl<'a> Benchmark<'a> { )?; } - let mut times_wall_clock: Vec = vec![]; - let mut times_user: Vec = vec![]; - let mut times_system: Vec = vec![]; - let mut memory_usage_byte: Vec = vec![]; - let mut exit_codes: Vec> = vec![]; + let mut runs = Runs::default(); let mut all_succeeded = true; let output_policy = &self.options.command_output_policies[self.number]; @@ -320,11 +317,13 @@ impl<'a> Benchmark<'a> { let count_remaining = count - 1; // Save the first result - times_wall_clock.push(res.time_wall_clock); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + runs.push(Run { + wall_clock_time: res.time_wall_clock, + user_time: res.time_user, + system_time: res.time_system, + memory_usage_byte: res.memory_usage_byte, + exit_code: extract_exit_code(status), + }); all_succeeded = all_succeeded && success; @@ -339,7 +338,7 @@ impl<'a> Benchmark<'a> { let benchmark_iteration = BenchmarkIteration::Benchmark(i + 1); let msg = { - let mean = format_duration(mean(×_wall_clock), self.options.time_unit); + let mean = format_duration(runs.mean(), self.options.time_unit); format!("Current estimate: {}", mean.to_string().green()) }; @@ -357,11 +356,13 @@ impl<'a> Benchmark<'a> { )?; let success = res.exit_status.success(); - times_wall_clock.push(res.time_wall_clock); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + runs.push(Run { + wall_clock_time: res.time_wall_clock, + user_time: res.time_user, + system_time: res.time_system, + memory_usage_byte: res.memory_usage_byte, + exit_code: extract_exit_code(status), + }); all_succeeded = all_succeeded && success; @@ -376,32 +377,17 @@ impl<'a> Benchmark<'a> { bar.finish_and_clear() } - // Compute statistical quantities - let t_num = times_wall_clock.len(); - let t_mean = mean(×_wall_clock); - let t_stddev = if times_wall_clock.len() > 1 { - Some(standard_deviation(×_wall_clock, Some(t_mean))) - } else { - None - }; - let t_min = min(×_wall_clock); - let t_max = max(×_wall_clock); - - let user_mean = measurements.time_user_mean(); - let system_mean = measurements.time_system_mean(); - // Formatting and console output - let time_unit = self.options.time_unit.unwrap_or(t_mean.suitable_unit()); - let mean_str = t_mean.format(time_unit); - let min_str = t_min.format(time_unit); - let max_str = t_max.format(time_unit); - let num_str = format!("{t_num} runs"); + let (mean_str, time_unit) = format_duration_unit(runs.mean(), self.options.time_unit); + let min_str = format_duration(runs.min(), Some(time_unit)); + let max_str = format_duration(runs.max(), Some(time_unit)); + let num_str = format!("{num_runs} runs", num_runs = runs.len()); - let user_str = user_mean.format(time_unit); - let system_str = system_mean.format(time_unit); + let user_str = format_duration(runs.user_mean(), Some(time_unit)); + let system_str = format_duration(runs.system_mean(), Some(time_unit)); if self.options.output_style != OutputStyleOption::Disabled { - if times_wall_clock.len() == 1 { + if runs.len() == 1 { println!( " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", "abs".green().bold(), @@ -411,7 +397,7 @@ impl<'a> Benchmark<'a> { system_str.blue() )?; } else { - let stddev_str = t_stddev.unwrap().format(time_unit); + let stddev_str = format_duration(runs.stddev().unwrap(), Some(time_unit)); console_writeln!( stdout, @@ -441,7 +427,10 @@ impl<'a> Benchmark<'a> { // Check execution time if matches!(self.options.executor_kind, ExecutorKind::Shell(_)) - && times_wall_clock.iter().any(|&t| t < MIN_EXECUTION_TIME) + && runs + .wall_clock_times() + .iter() + .any(|&t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } @@ -452,7 +441,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = modified_zscores(×_wall_clock); + let scores = modified_zscores(&runs.wall_clock_times()); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, @@ -467,7 +456,7 @@ impl<'a> Benchmark<'a> { if scores[0] > OUTLIER_THRESHOLD { warnings.push(Warnings::SlowInitialRun( - times_wall_clock[0], + runs.wall_clock_times()[0], outlier_warning_options, )); } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) { @@ -491,27 +480,7 @@ impl<'a> Benchmark<'a> { Ok(BenchmarkResult { command: self.command.get_name(), - runs: times_wall_clock - .iter() - .zip(times_user.iter()) - .zip(times_system.iter()) - .zip(memory_usage_byte.iter()) - .zip(exit_codes.iter()) - .map( - |( - (((wall_clock_time, user_time), system_time), memory_usage_byte), - exit_code, - )| { - Run { - wall_clock_time: *wall_clock_time, - user_time: *user_time, - system_time: *system_time, - memory_usage_byte: *memory_usage_byte, - exit_code: *exit_code, - } - }, - ) - .collect(), + runs, parameters: self .command .get_parameters() @@ -521,7 +490,7 @@ impl<'a> Benchmark<'a> { name.to_string(), Parameter { value: value.to_string(), - is_unused: self.command.is_parameter_unused(&name), + is_unused: self.command.is_parameter_unused(name), }, ) }) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 2dc300c..7260295 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -147,17 +147,19 @@ pub fn compute<'a>( fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; - use crate::benchmark::benchmark_result::Run; + use crate::benchmark::benchmark_result::{Run, Runs}; BenchmarkResult { command: name.into(), - runs: vec![Run { - wall_clock_time: mean, - user_time: mean, - system_time: 0., - memory_usage_byte: 1024, - exit_code: Some(0), - }], + runs: Runs { + runs: vec![Run { + wall_clock_time: mean, + user_time: mean, + system_time: 0., + memory_usage_byte: 1024, + exit_code: Some(0), + }], + }, parameters: BTreeMap::new(), } } diff --git a/src/export/csv.rs b/src/export/csv.rs index 4c961ce..aa8806d 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -68,7 +68,7 @@ impl Exporter for CsvExporter { } #[cfg(test)] -use crate::benchmark::benchmark_result::{Parameter, Run}; +use crate::benchmark::benchmark_result::{Parameter, Run, Runs}; #[test] fn test_csv() { @@ -84,7 +84,7 @@ fn test_csv() { let results = vec![ BenchmarkResult { command: String::from("command_a"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 7.0, user_time: 7.0, @@ -106,7 +106,7 @@ fn test_csv() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: { let mut params = BTreeMap::new(); params.insert( @@ -128,7 +128,7 @@ fn test_csv() { }, BenchmarkResult { command: String::from("command_b"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 17.0, user_time: 17.0, @@ -150,7 +150,7 @@ fn test_csv() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: { let mut params = BTreeMap::new(); params.insert( diff --git a/src/export/tests.rs b/src/export/tests.rs index 0f3ed15..e0c6c0e 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -1,5 +1,5 @@ use super::Exporter; -use crate::benchmark::benchmark_result::{BenchmarkResult, Run}; +use crate::benchmark::benchmark_result::{BenchmarkResult, Run, Runs}; use crate::export::asciidoc::AsciidocExporter; use crate::export::orgmode::OrgmodeExporter; use crate::quantity::{byte, second, Information, Time, TimeUnit, Zero}; @@ -27,7 +27,7 @@ fn test_markup_export_auto_ms() { let results = [ BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.09, user_time: 0.09, @@ -49,12 +49,12 @@ fn test_markup_export_auto_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.0, user_time: 2.0, @@ -76,7 +76,7 @@ fn test_markup_export_auto_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -126,7 +126,7 @@ fn test_markup_export_auto_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.1, user_time: 2.1, @@ -148,12 +148,12 @@ fn test_markup_export_auto_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.1, user_time: 0.1, @@ -175,7 +175,7 @@ fn test_markup_export_auto_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -225,7 +225,7 @@ fn test_markup_export_manual_ms() { let timing_results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.1, user_time: 2.1, @@ -247,12 +247,12 @@ fn test_markup_export_manual_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.1, user_time: 0.1, @@ -274,7 +274,7 @@ fn test_markup_export_manual_ms() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ]; @@ -323,7 +323,7 @@ fn test_markup_export_manual_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 2.01, user_time: 2.01, @@ -345,12 +345,12 @@ fn test_markup_export_manual_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, BenchmarkResult { command: String::from("sleep 0.1"), - runs: vec![ + runs: Runs::new(vec![ Run { wall_clock_time: 0.11, user_time: 0.11, @@ -372,7 +372,7 @@ fn test_markup_export_manual_s() { memory_usage_byte: 1024, exit_code: Some(0), }, - ], + ]), parameters: BTreeMap::new(), }, ];