Merge master and resolve dependency lockfile conflict

This commit is contained in:
David Peter committed 2026-10-05 10:18:47 +02:00
commit 0bccf959b9
11 files changed
+313 -429

No files matched your search

Generated
+3 -247
View File
@@ -97,27 +97,12 @@ dependencies = [
"wait-timeout",
]
[[package]]
name = "autocfg"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
dependencies = [
"autocfg 1.5.1",
]
[[package]]
name = "autocfg"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.13.2"
@@ -231,15 +216,6 @@ version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -352,12 +328,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -422,7 +392,6 @@ dependencies = [
"serde",
"serde_json",
"shell-words",
"statistical",
"tempfile",
"thiserror",
"windows-sys 0.61.2",
@@ -522,7 +491,7 @@ version = "0.31.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
dependencies = [
"bitflags 2.13.2",
"bitflags",
"cfg-if",
"cfg_aliases",
"libc",
@@ -534,79 +503,13 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
[[package]]
name = "num"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.5.1",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.5.1",
"num-traits",
]
[[package]]
name = "num-integer"
version = "0.1.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
dependencies = [
"autocfg 1.5.1",
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg 1.5.1",
"autocfg",
]
[[package]]
@@ -711,25 +614,6 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.8",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.3",
"rand_hc",
"rand_isaac",
"rand_jitter",
"rand_os",
"rand_pcg",
"rand_xorshift",
"winapi",
]
[[package]]
name = "rand"
version = "0.8.8"
@@ -762,16 +646,6 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "rand_chacha"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.8",
"rand_core 0.3.2",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
@@ -792,21 +666,6 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rand_core"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96f815e01bbd9678b50d927f79aa1cf3ffdfdb1b9787317c1284dadb894ad0e8"
dependencies = [
"rand_core 0.4.3",
]
[[package]]
name = "rand_core"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e5937858e6fd18cd595d558f90bb5de3b72ae23f9e3763af0e805949b04ef60"
[[package]]
name = "rand_core"
version = "0.6.4"
@@ -831,77 +690,6 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rand_jitter"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
dependencies = [
"libc",
"rand_core 0.4.3",
"winapi",
]
[[package]]
name = "rand_os"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
dependencies = [
"cloudabi",
"fuchsia-cprng",
"libc",
"rand_core 0.4.3",
"rdrand",
"winapi",
]
[[package]]
name = "rand_pcg"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.8",
"rand_core 0.4.3",
]
[[package]]
name = "rand_xorshift"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.2",
]
[[package]]
name = "regex"
version = "1.13.1"
@@ -954,7 +742,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
dependencies = [
"bitflags 2.13.2",
"bitflags",
"errno",
"libc",
"linux-raw-sys",
@@ -1028,16 +816,6 @@ version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "statistical"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49d57902bb128e5e38b5218d3681215ae3e322d99f65d5420e9849730d2ea372"
dependencies = [
"num",
"rand 0.6.5",
]
[[package]]
name = "strip-ansi-escapes"
version = "0.2.1"
@@ -1257,28 +1035,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-link"
version = "0.2.1"
-1
View File
@@ -20,7 +20,6 @@ windows_process_extensions_main_thread_handle = []
colored = "3.1"
# Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580
indicatif = "=0.17.4"
statistical = "1.0"
csv = "1.4"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
+35 -58
View File
@@ -7,14 +7,13 @@ use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
};
use crate::output::progress_bar::get_progress_bar;
use crate::timer::{execute_and_measure, TimerResult};
use crate::timer::execute_and_measure;
use crate::util::randomized_environment_offset;
use crate::util::units::Second;
use super::timing_result::TimingResult;
use super::measurement::{Measurement, Measurements};
use anyhow::{bail, Context, Result};
use statistical::mean;
pub enum BenchmarkIteration {
NonBenchmarkRun,
@@ -40,7 +39,7 @@ pub trait Executor {
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy,
) -> Result<(TimingResult, ExitStatus)>;
) -> Result<Measurement>;
/// Perform a calibration of this executor. For example,
/// when running commands through a shell, we need to
@@ -62,7 +61,7 @@ fn run_command_and_measure_common(
command_input_policy: &CommandInputPolicy,
command_output_policy: &CommandOutputPolicy,
command_name: &str,
) -> Result<TimerResult> {
) -> Result<Measurement> {
let stdin = command_input_policy.get_stdin()?;
let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -79,7 +78,7 @@ fn run_command_and_measure_common(
let result = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{command_name}'"))?;
if !result.status.success() {
if !result.exit_status.success() {
use crate::util::exit_code::extract_exit_code;
let should_fail = match command_failure_action {
@@ -87,7 +86,7 @@ fn run_command_and_measure_common(
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(result.status) {
if let Some(exit_code) = extract_exit_code(result.exit_status) {
!codes.contains(&exit_code)
} else {
// If we can't extract an exit code, treat it as a failure
@@ -107,7 +106,7 @@ fn run_command_and_measure_common(
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.status.code().map_or(
cause=result.exit_status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
@@ -136,25 +135,15 @@ impl Executor for RawExecutor<'_> {
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy,
) -> Result<(TimingResult, ExitStatus)> {
let result = run_command_and_measure_common(
) -> Result<Measurement> {
run_command_and_measure_common(
command.get_command()?,
iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
&self.options.command_input_policy,
output_policy,
&command.get_command_line(),
)?;
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
)
}
fn calibrate(&mut self) -> Result<()> {
@@ -169,7 +158,7 @@ impl Executor for RawExecutor<'_> {
pub struct ShellExecutor<'a> {
options: &'a Options,
shell: &'a Shell,
shell_spawning_time: Option<TimingResult>,
shell_spawning_time: Option<Measurement>,
}
impl<'a> ShellExecutor<'a> {
@@ -189,7 +178,7 @@ impl Executor for ShellExecutor<'_> {
iteration: BenchmarkIteration,
command_failure_action: Option<CmdFailureAction>,
output_policy: &CommandOutputPolicy,
) -> Result<(TimingResult, ExitStatus)> {
) -> Result<Measurement> {
let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
let mut command_builder = self.shell.command();
command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
@@ -213,20 +202,13 @@ impl Executor for ShellExecutor<'_> {
// Subtract shell spawning time
if let Some(spawning_time) = self.shell_spawning_time {
result.time_real = (result.time_real - spawning_time.time_real).max(0.0);
result.time_wall_clock =
(result.time_wall_clock - spawning_time.time_wall_clock).max(0.0);
result.time_user = (result.time_user - spawning_time.time_user).max(0.0);
result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
}
Ok((
TimingResult {
time_real: result.time_real,
time_user: result.time_user,
time_system: result.time_system,
memory_usage_byte: result.memory_usage_byte,
},
result.status,
))
Ok(result)
}
/// Measure the average shell spawning time
@@ -242,9 +224,7 @@ impl Executor for ShellExecutor<'_> {
None
};
let mut times_real: Vec<Second> = vec![];
let mut times_user: Vec<Second> = vec![];
let mut times_system: Vec<Second> = vec![];
let mut measurements = Measurements::default();
for _ in 0..COUNT {
// Just run the shell without any command
@@ -268,10 +248,8 @@ impl Executor for ShellExecutor<'_> {
shell_cmd
);
}
Ok((r, _)) => {
times_real.push(r.time_real);
times_user.push(r.time_user);
times_system.push(r.time_system);
Ok(r) => {
measurements.push(r);
}
}
@@ -284,18 +262,19 @@ impl Executor for ShellExecutor<'_> {
bar.finish_and_clear()
}
self.shell_spawning_time = Some(TimingResult {
time_real: mean(&times_real),
time_user: mean(&times_user),
time_system: mean(&times_system),
memory_usage_byte: 0,
self.shell_spawning_time = Some(Measurement {
time_wall_clock: measurements.time_wall_clock_mean(),
time_user: measurements.time_user_mean(),
time_system: measurements.time_system_mean(),
peak_memory_usage: 0,
exit_status: ExitStatus::default(),
});
Ok(())
}
fn time_overhead(&self) -> Second {
self.shell_spawning_time.unwrap().time_real
self.shell_spawning_time.unwrap().time_wall_clock
}
}
@@ -326,28 +305,26 @@ impl Executor for MockExecutor {
_iteration: BenchmarkIteration,
_command_failure_action: Option<CmdFailureAction>,
_output_policy: &CommandOutputPolicy,
) -> Result<(TimingResult, ExitStatus)> {
) -> Result<Measurement> {
#[cfg(unix)]
let status = {
let exit_status = {
use std::os::unix::process::ExitStatusExt;
ExitStatus::from_raw(0)
};
#[cfg(windows)]
let status = {
let exit_status = {
use std::os::windows::process::ExitStatusExt;
ExitStatus::from_raw(0)
};
Ok((
TimingResult {
time_real: Self::extract_time(command.get_command_line()),
time_user: 0.0,
time_system: 0.0,
memory_usage_byte: 0,
},
status,
))
Ok(Measurement {
time_wall_clock: Self::extract_time(command.get_command_line()),
time_user: 0.0,
time_system: 0.0,
peak_memory_usage: 0,
exit_status,
})
}
fn calibrate(&mut self) -> Result<()> {
+93
View File
@@ -0,0 +1,93 @@
use std::process::ExitStatus;
use crate::outlier_detection::modified_zscores;
use crate::util::statistics::{max, mean, median, min, standard_deviation};
use crate::util::units::Second;
/// Performance measurements and exit status from running a single command
#[derive(Debug, Default, Copy, Clone)]
pub struct Measurement {
/// Wall clock time
pub time_wall_clock: Second,
/// Time spent in user mode
pub time_user: Second,
/// Time spent in kernel mode
pub time_system: Second,
/// Maximum amount of memory used, in bytes
pub peak_memory_usage: u64,
/// The exit status of the process
pub exit_status: ExitStatus,
}
/// Performance measurements and exit statuses for all runs of a command.
#[derive(Debug, Default, Clone)]
pub struct Measurements {
pub measurements: Vec<Measurement>,
}
impl Measurements {
pub fn len(&self) -> usize {
self.measurements.len()
}
pub fn is_empty(&self) -> bool {
self.measurements.is_empty()
}
pub fn push(&mut self, measurement: Measurement) {
self.measurements.push(measurement);
}
pub fn wall_clock_times(&self) -> impl Iterator<Item = Second> + '_ {
self.measurements.iter().map(|m| m.time_wall_clock)
}
/// The average wall clock time.
pub fn time_wall_clock_mean(&self) -> Second {
mean(self.wall_clock_times())
}
/// The standard deviation of wall clock times, if at least two runs were measured.
pub fn stddev(&self) -> Option<Second> {
if self.len() < 2 {
None
} else {
let times: Vec<_> = self.wall_clock_times().collect();
Some(standard_deviation(&times, self.time_wall_clock_mean()))
}
}
/// The median wall clock time.
pub fn median(&self) -> Second {
median(&self.wall_clock_times().collect::<Vec<_>>())
}
/// The minimum wall clock time.
pub fn min(&self) -> Second {
min(self.wall_clock_times())
}
/// The maximum wall clock time.
pub fn max(&self) -> Second {
max(self.wall_clock_times())
}
/// Compute modified Z-scores for the wall clock times.
pub fn modified_zscores(&self) -> Vec<f64> {
modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
}
/// The average user time.
pub fn time_user_mean(&self) -> Second {
mean(self.measurements.iter().map(|m| m.time_user))
}
/// The average system time.
pub fn time_system_mean(&self) -> Second {
mean(self.measurements.iter().map(|m| m.time_system))
}
}
+51 -55
View File
@@ -1,8 +1,8 @@
pub mod benchmark_result;
pub mod executor;
pub mod measurement;
pub mod relative_speed;
pub mod scheduler;
pub mod timing_result;
use std::cmp;
use std::io::{self, Write};
@@ -12,21 +12,19 @@ use crate::command::Command;
use crate::options::{
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
};
use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
use crate::outlier_detection::OUTLIER_THRESHOLD;
use crate::output::console_writeln;
use crate::output::format::{format_duration, format_duration_unit};
use crate::output::progress_bar::get_progress_bar;
use crate::output::warnings::{OutlierWarningOptions, Warnings};
use crate::parameter::ParameterNameAndValue;
use crate::util::exit_code::extract_exit_code;
use crate::util::min_max::{max, min};
use crate::util::units::Second;
use benchmark_result::BenchmarkResult;
use timing_result::TimingResult;
use measurement::{Measurement, Measurements};
use anyhow::{anyhow, Result};
use colored::*;
use statistical::{mean, median, standard_deviation};
use self::executor::Executor;
@@ -61,7 +59,7 @@ impl<'a> Benchmark<'a> {
command: &Command<'_>,
error_output: &'static str,
output_policy: &CommandOutputPolicy,
) -> Result<TimingResult> {
) -> Result<Measurement> {
self.executor
.run_command_and_measure(
command,
@@ -69,7 +67,6 @@ impl<'a> Benchmark<'a> {
Some(CmdFailureAction::RaiseError),
output_policy,
)
.map(|r| r.0)
.map_err(|_| anyhow!(error_output))
}
@@ -78,7 +75,7 @@ impl<'a> Benchmark<'a> {
&self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy,
) -> Result<TimingResult> {
) -> Result<Measurement> {
let command = self
.options
.setup_command
@@ -99,7 +96,7 @@ impl<'a> Benchmark<'a> {
&self,
parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
output_policy: &CommandOutputPolicy,
) -> Result<TimingResult> {
) -> Result<Measurement> {
let command = self
.options
.cleanup_command
@@ -120,7 +117,7 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
) -> Result<TimingResult> {
) -> Result<Measurement> {
let error_output = "The preparation command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
@@ -132,7 +129,7 @@ impl<'a> Benchmark<'a> {
&self,
command: &Command<'_>,
output_policy: &CommandOutputPolicy,
) -> Result<TimingResult> {
) -> Result<Measurement> {
let error_output = "The conclusion command terminated with a non-zero exit code. \
Append ' || true' to the command if you are sure that this can be ignored.";
@@ -151,11 +148,7 @@ impl<'a> Benchmark<'a> {
)?;
}
let mut times_real: Vec<Second> = vec![];
let mut times_user: Vec<Second> = vec![];
let mut times_system: Vec<Second> = vec![];
let mut memory_usage_byte: Vec<u64> = vec![];
let mut exit_codes: Vec<Option<i32>> = vec![];
let mut measurements = Measurements::default();
let mut all_succeeded = true;
let output_policy = &self.options.command_output_policies[self.number];
@@ -243,25 +236,27 @@ impl<'a> Benchmark<'a> {
};
let preparation_result = run_preparation_command()?;
let preparation_overhead =
preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
let preparation_overhead = preparation_result.map_or(0.0, |res| {
res.time_wall_clock + self.executor.time_overhead()
});
// Initial timing run
let (res, status) = self.executor.run_command_and_measure(
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(0),
None,
output_policy,
)?;
let success = status.success();
let success = res.exit_status.success();
let conclusion_result = run_conclusion_command()?;
let conclusion_overhead =
conclusion_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
let conclusion_overhead = conclusion_result.map_or(0.0, |res| {
res.time_wall_clock + self.executor.time_overhead()
});
// Determine number of benchmark runs
let runs_in_min_time = (self.options.min_benchmarking_time
/ (res.time_real
/ (res.time_wall_clock
+ self.executor.time_overhead()
+ preparation_overhead
+ conclusion_overhead)) as u64;
@@ -280,11 +275,7 @@ impl<'a> Benchmark<'a> {
let count_remaining = count - 1;
// Save the first result
times_real.push(res.time_real);
times_user.push(res.time_user);
times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(status));
measurements.push(res);
all_succeeded = all_succeeded && success;
@@ -301,7 +292,8 @@ impl<'a> Benchmark<'a> {
run_preparation_command()?;
let msg = {
let mean = format_duration(mean(&times_real), self.options.time_unit);
let mean =
format_duration(measurements.time_wall_clock_mean(), self.options.time_unit);
format!("Current estimate: {}", mean.to_string().green())
};
@@ -309,19 +301,15 @@ impl<'a> Benchmark<'a> {
bar.set_message(msg.to_owned())
}
let (res, status) = self.executor.run_command_and_measure(
let res = self.executor.run_command_and_measure(
self.command,
BenchmarkIteration::Benchmark(i + 1),
None,
output_policy,
)?;
let success = status.success();
let success = res.exit_status.success();
times_real.push(res.time_real);
times_user.push(res.time_user);
times_system.push(res.time_system);
memory_usage_byte.push(res.memory_usage_byte);
exit_codes.push(extract_exit_code(status));
measurements.push(res);
all_succeeded = all_succeeded && success;
@@ -337,19 +325,15 @@ impl<'a> Benchmark<'a> {
}
// Compute statistical quantities
let t_num = times_real.len();
let t_mean = mean(&times_real);
let t_stddev = if times_real.len() > 1 {
Some(standard_deviation(&times_real, Some(t_mean)))
} else {
None
};
let t_median = median(&times_real);
let t_min = min(&times_real);
let t_max = max(&times_real);
let t_num = measurements.len();
let t_mean = measurements.time_wall_clock_mean();
let t_stddev = measurements.stddev();
let t_median = measurements.median();
let t_min = measurements.min();
let t_max = measurements.max();
let user_mean = mean(&times_user);
let system_mean = mean(&times_system);
let user_mean = measurements.time_user_mean();
let system_mean = measurements.time_system_mean();
// Formatting and console output
let (mean_str, time_unit) = format_duration_unit(t_mean, self.options.time_unit);
@@ -362,7 +346,7 @@ impl<'a> Benchmark<'a> {
if self.options.output_style != OutputStyleOption::Disabled {
let mut stdout = io::stdout().lock();
if times_real.len() == 1 {
if measurements.len() == 1 {
console_writeln!(
stdout,
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
@@ -403,7 +387,9 @@ impl<'a> Benchmark<'a> {
// Check execution time
if matches!(self.options.executor_kind, ExecutorKind::Shell(_))
&& times_real.iter().any(|&t| t < MIN_EXECUTION_TIME)
&& measurements
.wall_clock_times()
.any(|t| t < MIN_EXECUTION_TIME)
{
warnings.push(Warnings::FastExecutionTime);
}
@@ -414,7 +400,7 @@ impl<'a> Benchmark<'a> {
}
// Run outlier detection
let scores = modified_zscores(&times_real);
let scores = measurements.modified_zscores();
let outlier_warning_options = OutlierWarningOptions {
warmup_in_use: self.options.warmup_count > 0,
@@ -429,7 +415,7 @@ impl<'a> Benchmark<'a> {
if scores[0] > OUTLIER_THRESHOLD {
warnings.push(Warnings::SlowInitialRun(
times_real[0],
measurements.wall_clock_times().next().unwrap(),
outlier_warning_options,
));
} else if scores.iter().any(|&s| s.abs() > OUTLIER_THRESHOLD) {
@@ -461,9 +447,19 @@ impl<'a> Benchmark<'a> {
system: system_mean,
min: t_min,
max: t_max,
times: Some(times_real),
memory_usage_byte: Some(memory_usage_byte),
exit_codes,
times: Some(measurements.wall_clock_times().collect()),
memory_usage_byte: Some(
measurements
.measurements
.iter()
.map(|m| m.peak_memory_usage)
.collect(),
),
exit_codes: measurements
.measurements
.iter()
.map(|m| extract_exit_code(m.exit_status))
.collect(),
parameters: self
.command
.get_parameters()
-17
View File
@@ -1,17 +0,0 @@
use crate::util::units::Second;
/// Results from timing a single command
#[derive(Debug, Default, Copy, Clone)]
pub struct TimingResult {
/// Wall clock time
pub time_real: Second,
/// Time spent in user mode
pub time_user: Second,
/// Time spent in kernel mode
pub time_system: Second,
/// Maximum amount of memory used, in bytes
pub memory_usage_byte: u64,
}
+1 -1
View File
@@ -5,7 +5,7 @@
//! The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
//! Ph.D., Editor.
use statistical::median;
use crate::util::statistics::median;
/// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
/// converts the MAD to an estimator for the standard deviation. The second factor is the number
+10 -21
View File
@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
#[cfg(target_os = "windows")]
use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
use crate::util::units::Second;
use crate::benchmark::measurement::Measurement;
use wall_clock_timer::WallClockTimer;
use std::io::Read;
use std::process::{ChildStdout, Command, ExitStatus};
use std::process::{ChildStdout, Command};
use anyhow::Result;
@@ -37,17 +37,6 @@ struct CPUTimes {
pub memory_usage_byte: u64,
}
/// Used to indicate the result of running a command
#[derive(Debug, Copy, Clone)]
pub struct TimerResult {
pub time_real: Second,
pub time_user: Second,
pub time_system: Second,
pub memory_usage_byte: u64,
/// The exit status of the process
pub status: ExitStatus,
}
/// Discard the output of a child process.
fn discard(output: ChildStdout) {
const CHUNK_SIZE: usize = 64 << 10;
@@ -80,7 +69,7 @@ fn discard(output: ChildStdout) {
}
/// Execute the given command and return a timing summary
pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
#[cfg(not(windows))]
let cpu_timer = self::unix_timer::CPUTimer::start();
@@ -106,16 +95,16 @@ pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
discard(output);
}
let status = child.wait()?;
let exit_status = child.wait()?;
let time_real = wallclock_timer.stop();
let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
let time_wall_clock = wallclock_timer.stop();
let (time_user, time_system, peak_memory_usage) = cpu_timer.stop();
Ok(TimerResult {
time_real,
Ok(Measurement {
time_wall_clock,
time_user,
time_system,
memory_usage_byte,
status,
peak_memory_usage,
exit_status,
})
}
-28
View File
@@ -1,28 +0,0 @@
/// A max function for f64's without NaNs
pub fn max(vals: &[f64]) -> f64 {
*vals
.iter()
.max_by(|a, b| a.partial_cmp(b).unwrap())
.unwrap()
}
/// A min function for f64's without NaNs
pub fn min(vals: &[f64]) -> f64 {
*vals
.iter()
.min_by(|a, b| a.partial_cmp(b).unwrap())
.unwrap()
}
#[test]
fn test_max() {
let assert_float_eq = |a: f64, b: f64| {
assert!((a - b).abs() < f64::EPSILON);
};
assert_float_eq(1.0, max(&[1.0]));
assert_float_eq(-1.0, max(&[-1.0]));
assert_float_eq(-1.0, max(&[-2.0, -1.0]));
assert_float_eq(1.0, max(&[-1.0, 1.0]));
assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0]));
}
+1 -1
View File
@@ -1,5 +1,5 @@
pub mod exit_code;
pub mod min_max;
pub mod number;
pub mod randomized_environment_offset;
pub mod statistics;
pub mod units;
+119
View File
@@ -0,0 +1,119 @@
//! Statistics for non-empty samples of finite measurements.
/// A min function that assumes no NaNs and at least one element
pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
values
.into_iter()
.min_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
.expect("'min' requires at least one element")
}
/// A max function that assumes no NaNs and at least one element
pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
values
.into_iter()
.max_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
.expect("'max' requires at least one element")
}
/// Arithmetic mean, summing measurements in their original order.
pub fn mean(values: impl IntoIterator<Item = f64>) -> f64 {
let mut sum = 0.0;
let mut count = 0usize;
for value in values {
sum += value;
count += 1;
}
sum / count as f64
}
/// Median without modifying the input sample.
pub fn median(values: &[f64]) -> f64 {
assert!(
!values.is_empty(),
"median requires at least one measurement"
);
let mut sorted = values.to_vec();
sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
let mid = sorted.len() / 2;
if sorted.len() % 2 == 1 {
sorted[mid]
} else {
(sorted[mid] + sorted[mid - 1]) / 2.0
}
}
/// Sample standard deviation, using an already computed mean.
pub fn standard_deviation(values: &[f64], mean: f64) -> f64 {
assert!(
values.len() > 1,
"standard deviation requires at least two measurements"
);
let sum = values
.iter()
.map(|value| (value - mean) * (value - mean))
.fold(0.0, |sum, deviation| sum + deviation);
assert!(sum >= 0.0, "invalid sum of squared deviations");
// Preserve the operation order of statistical 1.0: dividing by n - 1
// before taking the square root also preserves exported floating-point values.
(sum / (values.len() as f64 - 1.0)).sqrt()
}
#[cfg(test)]
mod tests {
use super::*;
use approx::assert_relative_eq;
#[test]
fn test_min() {
assert_eq!(1.0, min([1.0]));
assert_eq!(-1.0, min([-1.0]));
assert_eq!(-2.0, min([-2.0, -1.0]));
assert_eq!(-1.0, min([-1.0, 1.0]));
assert_eq!(-1.0, min([1.0, -1.0, 0.0]));
let values = vec![1024.0, 2.0, 3.0];
assert_eq!(*min(&values), 2.0);
}
#[test]
fn test_max() {
assert_eq!(1.0, max([1.0]));
assert_eq!(-1.0, max([-1.0]));
assert_eq!(-1.0, max([-2.0, -1.0]));
assert_eq!(1.0, max([-1.0, 1.0]));
assert_eq!(1.0, max([-1.0, 1.0, 0.0]));
let values = vec![1.0, 2048.0, 3.0];
assert_eq!(*max(&values), 2048.0);
}
#[test]
fn test_mean() {
assert_eq!(1.0, mean([1.0]));
assert_relative_eq!(2.0, mean([1.0, 3.0]));
let values = [0.1, 0.2, 0.6];
assert_relative_eq!(mean(values), 0.3);
}
#[test]
fn test_median() {
assert_eq!(1.0, median(&[1.0]));
assert_relative_eq!(2.0, median(&[1.0, 3.0]));
let values = [0.1, 0.2, 0.6];
assert_relative_eq!(median(&values), 0.2);
let values = [0.1, 0.2, 0.3, 0.6];
assert_relative_eq!(median(&values), 0.25);
}
#[test]
fn test_standard_deviation() {
let values = [0.1, 0.2, 0.3];
assert_relative_eq!(standard_deviation(&values, mean(values)), 0.1);
}
}