diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index e1191d7..cead751 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,119 +1,8 @@ use std::collections::BTreeMap; use serde::Serialize; -use statistical::{mean, median, standard_deviation}; -use crate::{ - outlier_detection::modified_zscores, - util::{ - min_max::{max, min}, - units::Second, - }, -}; - -/// Performance metrics and exit codes for each run -#[derive(Debug, Default, Clone, Serialize, PartialEq)] -pub struct Run { - /// Wall clock time measurement - pub wall_clock_time: Second, - - /// Time spent in user mode - pub user_time: Second, - - /// Time spent in kernel mode - pub system_time: Second, - - /// Maximum memory usage of the process, in bytes - pub memory_usage_byte: u64, - - /// Exit codes of the process - pub exit_code: Option, -} - -#[derive(Debug, Default, Clone, Serialize, PartialEq)] -pub struct Runs { - pub runs: Vec, -} - -impl Runs { - pub fn new(runs: Vec) -> Self { - Self { runs } - } - - pub fn len(&self) -> usize { - self.runs.len() - } - - pub fn is_empty(&self) -> bool { - self.runs.is_empty() - } - - pub fn push(&mut self, run: Run) { - self.runs.push(run); - } - - pub fn wall_clock_times(&self) -> Vec { - self.runs.iter().map(|run| run.wall_clock_time).collect() - } - - /// The average wall clock time - pub fn mean(&self) -> Second { - mean(&self.wall_clock_times()) - } - - /// The standard deviation of all wall clock times. Not available if only one run has been performed - pub fn stddev(&self) -> Option { - let times = self.wall_clock_times(); - - let t_mean = mean(×); - if times.len() > 1 { - Some(standard_deviation(×, Some(t_mean))) - } else { - None - } - } - - /// The median wall clock time - pub fn median(&self) -> Second { - median(&self.wall_clock_times()) - } - - /// The minimum wall clock time - pub fn min(&self) -> Second { - min(&self.wall_clock_times()) - } - - /// The maximum wall clock time - pub fn max(&self) -> Second { - max(&self.wall_clock_times()) - } - - /// The average user time - pub fn user_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.user_time) - .collect::>(), - ) - } - - /// The average system time - pub fn system_mean(&self) -> Second { - mean( - &self - .runs - .iter() - .map(|run| run.system_time) - .collect::>(), - ) - } - - pub fn modified_zscores(&self) -> Vec { - modified_zscores(&self.wall_clock_times()) - } -} +use crate::{benchmark::measurement::Measurements, util::units::Second}; /// Parameter value and whether it was used in the command line template #[derive(Debug, Default, Clone, Serialize, PartialEq)] @@ -137,9 +26,9 @@ pub struct BenchmarkResult { /// The full command line of the program that is being benchmarked pub command: String, - /// Performance metrics and exit codes for each run + /// Performance metric measurements and exit codes for each run #[serde(flatten)] - pub runs: Runs, + pub measurements: Measurements, /// Parameter values for this benchmark #[serde(skip_serializing_if = "BTreeMap::is_empty")] @@ -149,7 +38,7 @@ pub struct BenchmarkResult { impl BenchmarkResult { /// The average wall clock time pub fn mean_wall_clock_time(&self) -> Second { - self.runs.mean() + self.measurements.mean() } /// The full command line of the program that is being benchmarked, possibly including a list of diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index da35e9b..2955220 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -1,29 +1,33 @@ -use std::process::ExitStatus; +use serde::Serialize; -use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation}; -use crate::quantity::{Information, Time}; +use statistical::{mean, median, standard_deviation}; -/// Performance measurements and exit status from running a single command -#[derive(Debug, Default, Copy, Clone, PartialEq)] +use crate::util::units::Second; +use crate::{ + outlier_detection::modified_zscores, + util::min_max::{max, min}, +}; + +/// Performance metric measurements and exit code for a single run +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Measurement { - /// Wall clock time - pub time_wall_clock: Time, + /// Elapsed wall clock time (real time) + pub wall_clock_time: Second, /// Time spent in user mode - pub time_user: Time, + pub user_time: Second, /// Time spent in kernel mode - pub time_system: Time, + pub system_time: Second, - /// Maximum amount of memory used - pub peak_memory_usage: Information, + /// Maximum memory usage of the process, in bytes + pub memory_usage_byte: u64, - /// The exit status of the process - pub exit_status: ExitStatus, + /// Exit codes of the process + pub exit_code: Option, } -/// Performance measurements and exit statuses for all runs of a command. -#[derive(Debug, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Measurements { pub measurements: Vec, } @@ -45,54 +49,69 @@ impl Measurements { self.measurements.push(measurement); } - pub fn wall_clock_times(&self) -> impl Iterator + '_ { - self.measurements.iter().map(|m| m.time_wall_clock) + pub fn wall_clock_times(&self) -> Vec { + self.measurements + .iter() + .map(|m| m.wall_clock_time) + .collect() } - /// The average wall clock time. - pub fn time_wall_clock_mean(&self) -> Time { - mean(self.wall_clock_times()) + /// The average wall clock time + pub fn mean(&self) -> Second { + mean(&self.wall_clock_times()) } /// The standard deviation of all wall clock times. Not available if only one run has been performed - pub fn stddev(&self) -> Option