diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 13d954a..e1191d7 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -3,9 +3,12 @@ use std::collections::BTreeMap; use serde::Serialize; use statistical::{mean, median, standard_deviation}; -use crate::util::{ - min_max::{max, min}, - units::Second, +use crate::{ + outlier_detection::modified_zscores, + util::{ + min_max::{max, min}, + units::Second, + }, }; /// Performance metrics and exit codes for each run @@ -106,6 +109,10 @@ impl Runs { .collect::>(), ) } + + pub fn modified_zscores(&self) -> Vec { + modified_zscores(&self.wall_clock_times()) + } } /// Parameter value and whether it was used in the command line template @@ -140,34 +147,11 @@ pub struct BenchmarkResult { } impl BenchmarkResult { - pub fn mean(&self) -> Second { + /// The average wall clock time + pub fn mean_wall_clock_time(&self) -> Second { self.runs.mean() } - pub fn stddev(&self) -> Option { - self.runs.stddev() - } - - pub fn median(&self) -> Second { - self.runs.median() - } - - pub fn min(&self) -> Second { - self.runs.min() - } - - pub fn max(&self) -> Second { - self.runs.max() - } - - pub fn user_mean(&self) -> Second { - self.runs.user_mean() - } - - pub fn system_mean(&self) -> Second { - self.runs.system_mean() - } - /// The full command line of the program that is being benchmarked, possibly including a list of /// parameters that were not used in the command line template. pub fn command_with_unused_parameters(&self) -> String { diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 4478b58..b1023d8 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -15,6 +15,7 @@ use crate::options::{ CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, }; use crate::outlier_detection::OUTLIER_THRESHOLD; +use crate::output::format::{format_duration, format_duration_unit}; use crate::output::console_writeln; use crate::output::progress_bar::{ finish_initial_measurement, get_progress_bar, start_initial_measurement, @@ -441,7 +442,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = modified_zscores(&runs.wall_clock_times()); + let scores = runs.modified_zscores(); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 7260295..6628970 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -31,7 +31,9 @@ impl BenchmarkResultWithRelativeSpeed<'_> { } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { - l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal) + l.mean_wall_clock_time() + .partial_cmp(&r.mean_wall_clock_time()) + .unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { @@ -66,9 +68,11 @@ fn compute_relative_speeds<'a>( } let ratio = match relative_ordering { - Ordering::Less => reference.mean() / result.mean(), + Ordering::Less => reference.mean_wall_clock_time() / result.mean_wall_clock_time(), Ordering::Equal => 1.0, - Ordering::Greater => result.mean() / reference.mean(), + Ordering::Greater => { + result.mean_wall_clock_time() / reference.mean_wall_clock_time() + } }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas @@ -113,7 +117,8 @@ pub fn compute_with_check_from_reference<'a>( reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { - if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 { + if fastest_of(results).mean_wall_clock_time() == 0.0 || reference.mean_wall_clock_time() == 0.0 + { return None; } @@ -126,7 +131,7 @@ pub fn compute_with_check( ) -> Option>> { let fastest = fastest_of(results); - if fastest.mean() == 0.0 { + if fastest.mean_wall_clock_time() == 0.0 { return None; } diff --git a/src/export/csv.rs b/src/export/csv.rs index aa8806d..11eb877 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -44,13 +44,13 @@ impl Exporter for CsvExporter { for res in results { let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; for f in &[ - res.mean(), - res.stddev().unwrap_or(0.0), - res.median(), - res.user_mean(), - res.system_mean(), - res.min(), - res.max(), + res.mean_wall_clock_time(), + res.runs.stddev().unwrap_or(0.0), + res.runs.median(), + res.runs.user_mean(), + res.runs.system_mean(), + res.runs.min(), + res.runs.max(), ] { fields.push(Cow::Owned( f.format_with_precision(CSV_UNIT, CSV_PRECISION) diff --git a/src/export/markup.rs b/src/export/markup.rs index e201a2e..9c1caee 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -48,25 +48,23 @@ pub trait MarkupExporter { table.push_str(&self.table_divider(&cells_alignment)); for entry in entries { - let measurement = &entry.result; + let result = &entry.result; // prepare data row strings - let cmd_str = measurement - .command_with_unused_parameters() - .replace('|', "\\|"); - let mean_str = format_duration_value(measurement.mean(), Some(unit)).0; - let stddev_str = if let Some(stddev) = measurement.stddev() { + let cmd_str = result.command_with_unused_parameters().replace('|', "\\|"); + let mean_str = format_duration_value(result.mean_wall_clock_time(), Some(unit)).0; + let stddev_str = if let Some(stddev) = result.runs.stddev() { format!(" ± {}", format_duration_value(stddev, Some(unit)).0) } else { "".into() }; - let min_str = measurement.measurements.min().format_value(unit); - let max_str = measurement.measurements.max().format_value(unit); + let min_str = result.measurements.min().format_value(unit); + let max_str = result.measurements.max().format_value(unit); // The ratio of two zero times is undefined, even if they compare equal. let relative_unavailable = reference_pending || (explicit_reference && !entry.is_reference && entry.relative_ordering == Ordering::Equal - && measurement.mean_wall_clock_time() == Time::zero()); + && result.mean_wall_clock_time() == Time::zero()); let relative = if relative_unavailable { "N/A".to_string() } else { @@ -116,7 +114,7 @@ pub trait MarkupExporter { fn determine_unit_from_results(results: &[BenchmarkResult]) -> TimeUnit { if let Some(first_result) = results.first() { // Use the first BenchmarkResult entry to determine the unit for all entries. - format_duration_value(first_result.mean(), None).1 + format_duration_value(first_result.mean_wall_clock_time(), None).1 } else { // Default to `Second`. TimeUnit::Second