diff --git a/Cargo.lock b/Cargo.lock index 3b5c95e..8cc7a8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,27 +97,12 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "autocfg" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78" -dependencies = [ - "autocfg 1.5.1", -] - [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.2" @@ -231,15 +216,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -352,12 +328,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "getrandom" version = "0.2.17" @@ -422,7 +392,6 @@ dependencies = [ "serde", "serde_json", "shell-words", - "statistical", "tempfile", "thiserror", "windows-sys 0.61.2", @@ -522,7 +491,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -534,79 +503,13 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.5.1", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg 1.5.1", - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg 1.5.1", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "autocfg 1.5.1", + "autocfg", ] [[package]] @@ -711,25 +614,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.8", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.3", - "rand_hc", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - [[package]] name = "rand" version = "0.8.8" @@ -762,16 +646,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.3.2", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -792,21 +666,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96f815e01bbd9678b50d927f79aa1cf3ffdfdb1b9787317c1284dadb894ad0e8" -dependencies = [ - "rand_core 0.4.3", -] - -[[package]] -name = "rand_core" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e5937858e6fd18cd595d558f90bb5de3b72ae23f9e3763af0e805949b04ef60" - [[package]] name = "rand_core" version = "0.6.4" @@ -831,77 +690,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.2", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.2", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.3", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.3", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.8", - "rand_core 0.4.3", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.2", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.2", -] - [[package]] name = "regex" version = "1.13.1" @@ -954,7 +742,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -1028,16 +816,6 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "statistical" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49d57902bb128e5e38b5218d3681215ae3e322d99f65d5420e9849730d2ea372" -dependencies = [ - "num", - "rand 0.6.5", -] - [[package]] name = "strip-ansi-escapes" version = "0.2.1" @@ -1257,28 +1035,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 0ba6a49..e2ab23a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ windows_process_extensions_main_thread_handle = [] colored = "3.1" # Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580 indicatif = "=0.17.4" -statistical = "1.0" csv = "1.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index 915b735..52284a4 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -7,14 +7,13 @@ use crate::options::{ CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, }; use crate::output::progress_bar::get_progress_bar; -use crate::timer::{execute_and_measure, TimerResult}; +use crate::timer::execute_and_measure; use crate::util::randomized_environment_offset; use crate::util::units::Second; -use super::timing_result::TimingResult; +use super::measurement::{Measurement, Measurements}; use anyhow::{bail, Context, Result}; -use statistical::mean; pub enum BenchmarkIteration { NonBenchmarkRun, @@ -40,7 +39,7 @@ pub trait Executor { iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, - ) -> Result<(TimingResult, ExitStatus)>; + ) -> Result; /// Perform a calibration of this executor. For example, /// when running commands through a shell, we need to @@ -62,7 +61,7 @@ fn run_command_and_measure_common( command_input_policy: &CommandInputPolicy, command_output_policy: &CommandOutputPolicy, command_name: &str, -) -> Result { +) -> Result { let stdin = command_input_policy.get_stdin()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; command.stdin(stdin).stdout(stdout).stderr(stderr); @@ -79,7 +78,7 @@ fn run_command_and_measure_common( let result = execute_and_measure(command) .with_context(|| format!("Failed to run command '{command_name}'"))?; - if !result.status.success() { + if !result.exit_status.success() { use crate::util::exit_code::extract_exit_code; let should_fail = match command_failure_action { @@ -87,7 +86,7 @@ fn run_command_and_measure_common( CmdFailureAction::IgnoreAllFailures => false, CmdFailureAction::IgnoreSpecificFailures(ref codes) => { // Only fail if the exit code is not in the list of codes to ignore - if let Some(exit_code) = extract_exit_code(result.status) { + if let Some(exit_code) = extract_exit_code(result.exit_status) { !codes.contains(&exit_code) } else { // If we can't extract an exit code, treat it as a failure @@ -107,7 +106,7 @@ fn run_command_and_measure_common( bail!( "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ Alternatively, use the '--show-output' option to debug what went wrong.", - cause=result.status.code().map_or( + cause=result.exit_status.code().map_or( "The process has been terminated by a signal".into(), |c| format!("Command terminated with non-zero exit code {c}") @@ -136,25 +135,15 @@ impl Executor for RawExecutor<'_> { iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, - ) -> Result<(TimingResult, ExitStatus)> { - let result = run_command_and_measure_common( + ) -> Result { + run_command_and_measure_common( command.get_command()?, iteration, command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), &self.options.command_input_policy, output_policy, &command.get_command_line(), - )?; - - Ok(( - TimingResult { - time_real: result.time_real, - time_user: result.time_user, - time_system: result.time_system, - memory_usage_byte: result.memory_usage_byte, - }, - result.status, - )) + ) } fn calibrate(&mut self) -> Result<()> { @@ -169,7 +158,7 @@ impl Executor for RawExecutor<'_> { pub struct ShellExecutor<'a> { options: &'a Options, shell: &'a Shell, - shell_spawning_time: Option, + shell_spawning_time: Option, } impl<'a> ShellExecutor<'a> { @@ -189,7 +178,7 @@ impl Executor for ShellExecutor<'_> { iteration: BenchmarkIteration, command_failure_action: Option, output_policy: &CommandOutputPolicy, - ) -> Result<(TimingResult, ExitStatus)> { + ) -> Result { let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe"); let mut command_builder = self.shell.command(); command_builder.arg(if on_windows_cmd { "/C" } else { "-c" }); @@ -213,20 +202,13 @@ impl Executor for ShellExecutor<'_> { // Subtract shell spawning time if let Some(spawning_time) = self.shell_spawning_time { - result.time_real = (result.time_real - spawning_time.time_real).max(0.0); + result.time_wall_clock = + (result.time_wall_clock - spawning_time.time_wall_clock).max(0.0); result.time_user = (result.time_user - spawning_time.time_user).max(0.0); result.time_system = (result.time_system - spawning_time.time_system).max(0.0); } - Ok(( - TimingResult { - time_real: result.time_real, - time_user: result.time_user, - time_system: result.time_system, - memory_usage_byte: result.memory_usage_byte, - }, - result.status, - )) + Ok(result) } /// Measure the average shell spawning time @@ -242,9 +224,7 @@ impl Executor for ShellExecutor<'_> { None }; - let mut times_real: Vec = vec![]; - let mut times_user: Vec = vec![]; - let mut times_system: Vec = vec![]; + let mut measurements = Measurements::default(); for _ in 0..COUNT { // Just run the shell without any command @@ -268,10 +248,8 @@ impl Executor for ShellExecutor<'_> { shell_cmd ); } - Ok((r, _)) => { - times_real.push(r.time_real); - times_user.push(r.time_user); - times_system.push(r.time_system); + Ok(r) => { + measurements.push(r); } } @@ -284,18 +262,19 @@ impl Executor for ShellExecutor<'_> { bar.finish_and_clear() } - self.shell_spawning_time = Some(TimingResult { - time_real: mean(×_real), - time_user: mean(×_user), - time_system: mean(×_system), - memory_usage_byte: 0, + self.shell_spawning_time = Some(Measurement { + time_wall_clock: measurements.time_wall_clock_mean(), + time_user: measurements.time_user_mean(), + time_system: measurements.time_system_mean(), + peak_memory_usage: 0, + exit_status: ExitStatus::default(), }); Ok(()) } fn time_overhead(&self) -> Second { - self.shell_spawning_time.unwrap().time_real + self.shell_spawning_time.unwrap().time_wall_clock } } @@ -326,28 +305,26 @@ impl Executor for MockExecutor { _iteration: BenchmarkIteration, _command_failure_action: Option, _output_policy: &CommandOutputPolicy, - ) -> Result<(TimingResult, ExitStatus)> { + ) -> Result { #[cfg(unix)] - let status = { + let exit_status = { use std::os::unix::process::ExitStatusExt; ExitStatus::from_raw(0) }; #[cfg(windows)] - let status = { + let exit_status = { use std::os::windows::process::ExitStatusExt; ExitStatus::from_raw(0) }; - Ok(( - TimingResult { - time_real: Self::extract_time(command.get_command_line()), - time_user: 0.0, - time_system: 0.0, - memory_usage_byte: 0, - }, - status, - )) + Ok(Measurement { + time_wall_clock: Self::extract_time(command.get_command_line()), + time_user: 0.0, + time_system: 0.0, + peak_memory_usage: 0, + exit_status, + }) } fn calibrate(&mut self) -> Result<()> { diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs new file mode 100644 index 0000000..4e4efef --- /dev/null +++ b/src/benchmark/measurement.rs @@ -0,0 +1,93 @@ +use std::process::ExitStatus; + +use crate::outlier_detection::modified_zscores; +use crate::util::statistics::{max, mean, median, min, standard_deviation}; +use crate::util::units::Second; + +/// Performance measurements and exit status from running a single command +#[derive(Debug, Default, Copy, Clone)] +pub struct Measurement { + /// Wall clock time + pub time_wall_clock: Second, + + /// Time spent in user mode + pub time_user: Second, + + /// Time spent in kernel mode + pub time_system: Second, + + /// Maximum amount of memory used, in bytes + pub peak_memory_usage: u64, + + /// The exit status of the process + pub exit_status: ExitStatus, +} + +/// Performance measurements and exit statuses for all runs of a command. +#[derive(Debug, Default, Clone)] +pub struct Measurements { + pub measurements: Vec, +} + +impl Measurements { + pub fn len(&self) -> usize { + self.measurements.len() + } + + pub fn is_empty(&self) -> bool { + self.measurements.is_empty() + } + + pub fn push(&mut self, measurement: Measurement) { + self.measurements.push(measurement); + } + + pub fn wall_clock_times(&self) -> impl Iterator + '_ { + self.measurements.iter().map(|m| m.time_wall_clock) + } + + /// The average wall clock time. + pub fn time_wall_clock_mean(&self) -> Second { + mean(self.wall_clock_times()) + } + + /// The standard deviation of wall clock times, if at least two runs were measured. + pub fn stddev(&self) -> Option { + if self.len() < 2 { + None + } else { + let times: Vec<_> = self.wall_clock_times().collect(); + Some(standard_deviation(×, self.time_wall_clock_mean())) + } + } + + /// The median wall clock time. + pub fn median(&self) -> Second { + median(&self.wall_clock_times().collect::>()) + } + + /// The minimum wall clock time. + pub fn min(&self) -> Second { + min(self.wall_clock_times()) + } + + /// The maximum wall clock time. + pub fn max(&self) -> Second { + max(self.wall_clock_times()) + } + + /// Compute modified Z-scores for the wall clock times. + pub fn modified_zscores(&self) -> Vec { + modified_zscores(&self.wall_clock_times().collect::>()) + } + + /// The average user time. + pub fn time_user_mean(&self) -> Second { + mean(self.measurements.iter().map(|m| m.time_user)) + } + + /// The average system time. + pub fn time_system_mean(&self) -> Second { + mean(self.measurements.iter().map(|m| m.time_system)) + } +} diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index b6c615d..8f4c1e1 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -1,8 +1,8 @@ pub mod benchmark_result; pub mod executor; +pub mod measurement; pub mod relative_speed; pub mod scheduler; -pub mod timing_result; use std::cmp; use std::io::{self, Write}; @@ -12,21 +12,19 @@ use crate::command::Command; use crate::options::{ CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, }; -use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; +use crate::outlier_detection::OUTLIER_THRESHOLD; use crate::output::console_writeln; use crate::output::format::{format_duration, format_duration_unit}; use crate::output::progress_bar::get_progress_bar; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; use crate::util::exit_code::extract_exit_code; -use crate::util::min_max::{max, min}; use crate::util::units::Second; use benchmark_result::BenchmarkResult; -use timing_result::TimingResult; +use measurement::{Measurement, Measurements}; use anyhow::{anyhow, Result}; use colored::*; -use statistical::{mean, median, standard_deviation}; use self::executor::Executor; @@ -61,7 +59,7 @@ impl<'a> Benchmark<'a> { command: &Command<'_>, error_output: &'static str, output_policy: &CommandOutputPolicy, - ) -> Result { + ) -> Result { self.executor .run_command_and_measure( command, @@ -69,7 +67,6 @@ impl<'a> Benchmark<'a> { Some(CmdFailureAction::RaiseError), output_policy, ) - .map(|r| r.0) .map_err(|_| anyhow!(error_output)) } @@ -78,7 +75,7 @@ impl<'a> Benchmark<'a> { &self, parameters: impl IntoIterator>, output_policy: &CommandOutputPolicy, - ) -> Result { + ) -> Result { let command = self .options .setup_command @@ -99,7 +96,7 @@ impl<'a> Benchmark<'a> { &self, parameters: impl IntoIterator>, output_policy: &CommandOutputPolicy, - ) -> Result { + ) -> Result { let command = self .options .cleanup_command @@ -120,7 +117,7 @@ impl<'a> Benchmark<'a> { &self, command: &Command<'_>, output_policy: &CommandOutputPolicy, - ) -> Result { + ) -> Result { let error_output = "The preparation command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; @@ -132,7 +129,7 @@ impl<'a> Benchmark<'a> { &self, command: &Command<'_>, output_policy: &CommandOutputPolicy, - ) -> Result { + ) -> Result { let error_output = "The conclusion command terminated with a non-zero exit code. \ Append ' || true' to the command if you are sure that this can be ignored."; @@ -151,11 +148,7 @@ impl<'a> Benchmark<'a> { )?; } - let mut times_real: Vec = vec![]; - let mut times_user: Vec = vec![]; - let mut times_system: Vec = vec![]; - let mut memory_usage_byte: Vec = vec![]; - let mut exit_codes: Vec> = vec![]; + let mut measurements = Measurements::default(); let mut all_succeeded = true; let output_policy = &self.options.command_output_policies[self.number]; @@ -243,25 +236,27 @@ impl<'a> Benchmark<'a> { }; let preparation_result = run_preparation_command()?; - let preparation_overhead = - preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); + let preparation_overhead = preparation_result.map_or(0.0, |res| { + res.time_wall_clock + self.executor.time_overhead() + }); // Initial timing run - let (res, status) = self.executor.run_command_and_measure( + let res = self.executor.run_command_and_measure( self.command, BenchmarkIteration::Benchmark(0), None, output_policy, )?; - let success = status.success(); + let success = res.exit_status.success(); let conclusion_result = run_conclusion_command()?; - let conclusion_overhead = - conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead()); + let conclusion_overhead = conclusion_result.map_or(0.0, |res| { + res.time_wall_clock + self.executor.time_overhead() + }); // Determine number of benchmark runs let runs_in_min_time = (self.options.min_benchmarking_time - / (res.time_real + / (res.time_wall_clock + self.executor.time_overhead() + preparation_overhead + conclusion_overhead)) as u64; @@ -280,11 +275,7 @@ impl<'a> Benchmark<'a> { let count_remaining = count - 1; // Save the first result - times_real.push(res.time_real); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + measurements.push(res); all_succeeded = all_succeeded && success; @@ -301,7 +292,8 @@ impl<'a> Benchmark<'a> { run_preparation_command()?; let msg = { - let mean = format_duration(mean(×_real), self.options.time_unit); + let mean = + format_duration(measurements.time_wall_clock_mean(), self.options.time_unit); format!("Current estimate: {}", mean.to_string().green()) }; @@ -309,19 +301,15 @@ impl<'a> Benchmark<'a> { bar.set_message(msg.to_owned()) } - let (res, status) = self.executor.run_command_and_measure( + let res = self.executor.run_command_and_measure( self.command, BenchmarkIteration::Benchmark(i + 1), None, output_policy, )?; - let success = status.success(); + let success = res.exit_status.success(); - times_real.push(res.time_real); - times_user.push(res.time_user); - times_system.push(res.time_system); - memory_usage_byte.push(res.memory_usage_byte); - exit_codes.push(extract_exit_code(status)); + measurements.push(res); all_succeeded = all_succeeded && success; @@ -337,19 +325,15 @@ impl<'a> Benchmark<'a> { } // Compute statistical quantities - let t_num = times_real.len(); - let t_mean = mean(×_real); - let t_stddev = if times_real.len() > 1 { - Some(standard_deviation(×_real, Some(t_mean))) - } else { - None - }; - let t_median = median(×_real); - let t_min = min(×_real); - let t_max = max(×_real); + let t_num = measurements.len(); + let t_mean = measurements.time_wall_clock_mean(); + let t_stddev = measurements.stddev(); + let t_median = measurements.median(); + let t_min = measurements.min(); + let t_max = measurements.max(); - let user_mean = mean(×_user); - let system_mean = mean(×_system); + let user_mean = measurements.time_user_mean(); + let system_mean = measurements.time_system_mean(); // Formatting and console output let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit); @@ -362,7 +346,7 @@ impl<'a> Benchmark<'a> { if self.options.output_style != OutputStyleOption::Disabled { let mut stdout = io::stdout().lock(); - if times_real.len() == 1 { + if measurements.len() == 1 { console_writeln!( stdout, " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", @@ -403,7 +387,9 @@ impl<'a> Benchmark<'a> { // Check execution time if matches!(self.options.executor_kind, ExecutorKind::Shell(_)) - && times_real.iter().any(|&t| t < MIN_EXECUTION_TIME) + && measurements + .wall_clock_times() + .any(|t| t < MIN_EXECUTION_TIME) { warnings.push(Warnings::FastExecutionTime); } @@ -414,7 +400,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = modified_zscores(×_real); + let scores = measurements.modified_zscores(); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, @@ -429,7 +415,7 @@ impl<'a> Benchmark<'a> { if scores[0] > OUTLIER_THRESHOLD { warnings.push(Warnings::SlowInitialRun( - times_real[0], + measurements.wall_clock_times().next().unwrap(), outlier_warning_options, )); } else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) { @@ -461,9 +447,19 @@ impl<'a> Benchmark<'a> { system: system_mean, min: t_min, max: t_max, - times: Some(times_real), - memory_usage_byte: Some(memory_usage_byte), - exit_codes, + times: Some(measurements.wall_clock_times().collect()), + memory_usage_byte: Some( + measurements + .measurements + .iter() + .map(|m| m.peak_memory_usage) + .collect(), + ), + exit_codes: measurements + .measurements + .iter() + .map(|m| extract_exit_code(m.exit_status)) + .collect(), parameters: self .command .get_parameters() diff --git a/src/benchmark/timing_result.rs b/src/benchmark/timing_result.rs deleted file mode 100644 index f7aac92..0000000 --- a/src/benchmark/timing_result.rs +++ /dev/null @@ -1,17 +0,0 @@ -use crate::util::units::Second; - -/// Results from timing a single command -#[derive(Debug, Default, Copy, Clone)] -pub struct TimingResult { - /// Wall clock time - pub time_real: Second, - - /// Time spent in user mode - pub time_user: Second, - - /// Time spent in kernel mode - pub time_system: Second, - - /// Maximum amount of memory used, in bytes - pub memory_usage_byte: u64, -} diff --git a/src/outlier_detection.rs b/src/outlier_detection.rs index 270e522..c4c3a24 100644 --- a/src/outlier_detection.rs +++ b/src/outlier_detection.rs @@ -5,7 +5,7 @@ //! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka, //! Ph.D., Editor. -use statistical::median; +use crate::util::statistics::median; /// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that /// converts the MAD to an estimator for the standard deviation. The second factor is the number diff --git a/src/timer/mod.rs b/src/timer/mod.rs index b2cd265..94357d1 100644 --- a/src/timer/mod.rs +++ b/src/timer/mod.rs @@ -16,11 +16,11 @@ use std::os::fd::AsFd; #[cfg(target_os = "windows")] use windows_sys::Win32::System::Threading::CREATE_SUSPENDED; -use crate::util::units::Second; +use crate::benchmark::measurement::Measurement; use wall_clock_timer::WallClockTimer; use std::io::Read; -use std::process::{ChildStdout, Command, ExitStatus}; +use std::process::{ChildStdout, Command}; use anyhow::Result; @@ -37,17 +37,6 @@ struct CPUTimes { pub memory_usage_byte: u64, } -/// Used to indicate the result of running a command -#[derive(Debug, Copy, Clone)] -pub struct TimerResult { - pub time_real: Second, - pub time_user: Second, - pub time_system: Second, - pub memory_usage_byte: u64, - /// The exit status of the process - pub status: ExitStatus, -} - /// Discard the output of a child process. fn discard(output: ChildStdout) { const CHUNK_SIZE: usize = 64 << 10; @@ -80,7 +69,7 @@ fn discard(output: ChildStdout) { } /// Execute the given command and return a timing summary -pub fn execute_and_measure(mut command: Command) -> Result { +pub fn execute_and_measure(mut command: Command) -> Result { #[cfg(not(windows))] let cpu_timer = self::unix_timer::CPUTimer::start(); @@ -106,16 +95,16 @@ pub fn execute_and_measure(mut command: Command) -> Result { discard(output); } - let status = child.wait()?; + let exit_status = child.wait()?; - let time_real = wallclock_timer.stop(); - let (time_user, time_system, memory_usage_byte) = cpu_timer.stop(); + let time_wall_clock = wallclock_timer.stop(); + let (time_user, time_system, peak_memory_usage) = cpu_timer.stop(); - Ok(TimerResult { - time_real, + Ok(Measurement { + time_wall_clock, time_user, time_system, - memory_usage_byte, - status, + peak_memory_usage, + exit_status, }) } diff --git a/src/util/min_max.rs b/src/util/min_max.rs deleted file mode 100644 index d4ee07e..0000000 --- a/src/util/min_max.rs +++ /dev/null @@ -1,28 +0,0 @@ -/// A max function for f64's without NaNs -pub fn max(vals: &[f64]) -> f64 { - *vals - .iter() - .max_by(|a, b| a.partial_cmp(b).unwrap()) - .unwrap() -} - -/// A min function for f64's without NaNs -pub fn min(vals: &[f64]) -> f64 { - *vals - .iter() - .min_by(|a, b| a.partial_cmp(b).unwrap()) - .unwrap() -} - -#[test] -fn test_max() { - let assert_float_eq = |a: f64, b: f64| { - assert!((a - b).abs() < f64::EPSILON); - }; - - assert_float_eq(1.0, max(&[1.0])); - assert_float_eq(-1.0, max(&[-1.0])); - assert_float_eq(-1.0, max(&[-2.0, -1.0])); - assert_float_eq(1.0, max(&[-1.0, 1.0])); - assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0])); -} diff --git a/src/util/mod.rs b/src/util/mod.rs index de9e168..e5cdaba 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,5 @@ pub mod exit_code; -pub mod min_max; pub mod number; pub mod randomized_environment_offset; +pub mod statistics; pub mod units; diff --git a/src/util/statistics.rs b/src/util/statistics.rs new file mode 100644 index 0000000..443e866 --- /dev/null +++ b/src/util/statistics.rs @@ -0,0 +1,119 @@ +//! Statistics for non-empty samples of finite measurements. + +/// A min function that assumes no NaNs and at least one element +pub fn min(values: impl IntoIterator) -> Q { + values + .into_iter() + .min_by(|a, b| a.partial_cmp(b).expect("No NaN values")) + .expect("'min' requires at least one element") +} + +/// A max function that assumes no NaNs and at least one element +pub fn max(values: impl IntoIterator) -> Q { + values + .into_iter() + .max_by(|a, b| a.partial_cmp(b).expect("No NaN values")) + .expect("'max' requires at least one element") +} + +/// Arithmetic mean, summing measurements in their original order. +pub fn mean(values: impl IntoIterator) -> f64 { + let mut sum = 0.0; + let mut count = 0usize; + for value in values { + sum += value; + count += 1; + } + sum / count as f64 +} + +/// Median without modifying the input sample. +pub fn median(values: &[f64]) -> f64 { + assert!( + !values.is_empty(), + "median requires at least one measurement" + ); + let mut sorted = values.to_vec(); + sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements")); + + let mid = sorted.len() / 2; + if sorted.len() % 2 == 1 { + sorted[mid] + } else { + (sorted[mid] + sorted[mid - 1]) / 2.0 + } +} + +/// Sample standard deviation, using an already computed mean. +pub fn standard_deviation(values: &[f64], mean: f64) -> f64 { + assert!( + values.len() > 1, + "standard deviation requires at least two measurements" + ); + let sum = values + .iter() + .map(|value| (value - mean) * (value - mean)) + .fold(0.0, |sum, deviation| sum + deviation); + assert!(sum >= 0.0, "invalid sum of squared deviations"); + + // Preserve the operation order of statistical 1.0: dividing by n - 1 + // before taking the square root also preserves exported floating-point values. + (sum / (values.len() as f64 - 1.0)).sqrt() +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_relative_eq; + + #[test] + fn test_min() { + assert_eq!(1.0, min([1.0])); + assert_eq!(-1.0, min([-1.0])); + assert_eq!(-2.0, min([-2.0, -1.0])); + assert_eq!(-1.0, min([-1.0, 1.0])); + assert_eq!(-1.0, min([1.0, -1.0, 0.0])); + + let values = vec![1024.0, 2.0, 3.0]; + assert_eq!(*min(&values), 2.0); + } + + #[test] + fn test_max() { + assert_eq!(1.0, max([1.0])); + assert_eq!(-1.0, max([-1.0])); + assert_eq!(-1.0, max([-2.0, -1.0])); + assert_eq!(1.0, max([-1.0, 1.0])); + assert_eq!(1.0, max([-1.0, 1.0, 0.0])); + + let values = vec![1.0, 2048.0, 3.0]; + assert_eq!(*max(&values), 2048.0); + } + + #[test] + fn test_mean() { + assert_eq!(1.0, mean([1.0])); + assert_relative_eq!(2.0, mean([1.0, 3.0])); + + let values = [0.1, 0.2, 0.6]; + assert_relative_eq!(mean(values), 0.3); + } + + #[test] + fn test_median() { + assert_eq!(1.0, median(&[1.0])); + assert_relative_eq!(2.0, median(&[1.0, 3.0])); + + let values = [0.1, 0.2, 0.6]; + assert_relative_eq!(median(&values), 0.2); + + let values = [0.1, 0.2, 0.3, 0.6]; + assert_relative_eq!(median(&values), 0.25); + } + + #[test] + fn test_standard_deviation() { + let values = [0.1, 0.2, 0.3]; + assert_relative_eq!(standard_deviation(&values, mean(values)), 0.1); + } +}