From 715994847fe090b82687ef01b9b71dba09105521 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 6 Oct 2026 13:20:18 +0200 Subject: [PATCH] Simplifications --- scripts/README.md | 20 ------------------ src/benchmark/measurement.rs | 39 ++++++++++++++++++++++++++++++++---- src/benchmark/mod.rs | 15 +++++--------- src/output/warnings.rs | 16 ++++++++++----- src/quantity/statistics.rs | 5 ----- 5 files changed, 51 insertions(+), 44 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 03c0ad5..fcf9c7d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,31 +8,11 @@ uv run plot_whisker.py sleep.json ``` All plotting and analysis scripts use the JSON export's `primary_metric` by default. -Pass `--metric` to select a different metric. Supported metrics are `time_wall_clock`, -`time_cpu`, `time_user`, `time_system`, `memory_peak_resident`, `cpu_cycles`, -`instructions`, `cache_references`, `cache_misses`, and `branch_misses`, provided the -selected metric is available in every benchmark being analyzed. For example: ```bash uv run plot_histogram.py benchmark.json --metric instructions ``` -Time is displayed in seconds, memory in MiB, and counters as unscaled counts. -Multi-file plots require matching primary metrics unless `--metric` explicitly selects -one metric across all files. Exports without `primary_metric` fall back to wall-clock -time; the per-run `measurements` JSON schema is still required. - -For descriptive statistics or a Welch's t-test comparing exactly two benchmark results: - -```bash -uv run advanced_statistics.py benchmark.json --metric time_cpu --time-unit millisecond -uv run welch_ttest.py comparison.json --metric memory_peak_resident -``` - -`advanced_statistics.py` accepts `--time-unit second` or `--time-unit millisecond` -for time metrics only. Both analysis scripts also accept the default primary metric -without `--metric`. - ### Prerequisites Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index de19efc..8911f2d 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -2,7 +2,8 @@ use std::process::ExitStatus; use serde::Serialize; -use crate::quantity::statistics::{mean, modified_zscores}; +use crate::metric::Metric; +use crate::quantity::statistics::{mean, modified_zscores_f64}; use crate::quantity::{serialize_information, serialize_time, Information, Time}; use crate::util::exit_code::extract_exit_code; @@ -157,9 +158,14 @@ impl Measurements { mean(self.wall_clock_times()) } - /// Compute modified Z-scores for the wall clock times - pub fn modified_zscores(&self) -> Vec { - modified_zscores(&self.wall_clock_times().collect::>()) + /// Compute modified Z-scores for a metric validated to be available in every run. + pub fn modified_zscores(&self, metric: Metric) -> Vec { + let values: Vec<_> = self + .measurements + .iter() + .map(|m| metric.value(m).expect("Validated metric is available")) + .collect(); + modified_zscores_f64(&values) } /// The average user time @@ -172,3 +178,28 @@ impl Measurements { mean(self.measurements.iter().map(|m| m.time_system)) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::outlier_detection::OUTLIER_THRESHOLD; + use crate::quantity::{byte, second}; + + #[test] + fn outliers_follow_the_selected_metric() { + let measurements = Measurements::new( + [(1.0, 100.0), (1.0, 1.0), (1.0, 1.0), (100.0, 1.0)] + .iter() + .copied() + .map(|(time, memory)| Measurement { + time_wall_clock: Time::new::(time), + memory_peak_resident: Some(Information::new::(memory)), + ..Measurement::default() + }) + .collect(), + ); + + assert!(measurements.modified_zscores(Metric::TimeWallClock)[0] < OUTLIER_THRESHOLD); + assert!(measurements.modified_zscores(Metric::MemoryPeakResident)[0] > OUTLIER_THRESHOLD); + } +} diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index fdd9e01..d0dce9d 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -412,7 +412,7 @@ impl<'a> Benchmark<'a> { } // Run outlier detection - let scores = measurements.modified_zscores(); + let scores = measurements.modified_zscores(primary.metric); let outlier_warning_options = OutlierWarningOptions { warmup_in_use: self.options.warmup_count > 0, @@ -425,15 +425,10 @@ impl<'a> Benchmark<'a> { > 0, }; - if self - .options - .metrics - .iter() - .any(|m| m.metric == crate::metric::Metric::TimeWallClock) - && scores[0] > OUTLIER_THRESHOLD - { - warnings.push(Warnings::SlowInitialRun( - measurements.wall_clock_times().next().unwrap(), + if scores[0] > OUTLIER_THRESHOLD { + warnings.push(Warnings::InitialRunOutlier( + primary, + primary.metric.value(&measurements.measurements[0]).unwrap(), outlier_warning_options, )); } diff --git a/src/output/warnings.rs b/src/output/warnings.rs index 1f4a8ee..1a70231 100644 --- a/src/output/warnings.rs +++ b/src/output/warnings.rs @@ -1,8 +1,8 @@ use std::fmt; use crate::benchmark::MIN_EXECUTION_TIME; -use crate::metric::MetricSelection; -use crate::quantity::{millisecond, second, Time}; +use crate::metric::{Metric, MetricSelection}; +use crate::quantity::millisecond; pub struct OutlierWarningOptions { pub warmup_in_use: bool, @@ -13,7 +13,7 @@ pub struct OutlierWarningOptions { pub enum Warnings { FastExecutionTime, NonZeroExitCode, - SlowInitialRun(Time, OutlierWarningOptions), + InitialRunOutlier(MetricSelection, f64, OutlierWarningOptions), } impl fmt::Display for Warnings { @@ -28,12 +28,18 @@ impl fmt::Display for Warnings { MIN_EXECUTION_TIME.get::() ), Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."), - Warnings::SlowInitialRun(time_first_run, ref options) => write!( + Warnings::InitialRunOutlier(selection, value, _) if selection.metric != Metric::TimeWallClock => write!( + f, + "The first benchmarking run for this command had an unusually high value for the primary '{}' metric ({}) compared with the rest.", + selection.metric.name(), + selection.display_unit(value).format(value), + ), + Warnings::InitialRunOutlier(selection, value, ref options) => write!( f, "The first benchmarking run for this command was significantly slower than the \ rest ({time}). This could be caused by (filesystem) caches that were not filled until \ after the first run. {hints}", - time=MetricSelection::default().display_unit(time_first_run.get::()).format(time_first_run.get::()), + time=selection.display_unit(value).format(value), hints=match (options.warmup_in_use, options.prepare_in_use) { (true, true) => "You are already using both the '--warmup' option as well \ as the '--prepare' option. Consider re-running the benchmark on a quiet system. \ diff --git a/src/quantity/statistics.rs b/src/quantity/statistics.rs index 695ea14..6bef3a0 100644 --- a/src/quantity/statistics.rs +++ b/src/quantity/statistics.rs @@ -146,11 +146,6 @@ pub fn modified_zscores_f64(xs: &[f64]) -> Vec { xs.iter().map(|&x| (x - x_median) / mad).collect() } -pub fn modified_zscores(values: &[Q]) -> Vec { - let values: Vec<_> = values.iter().map(|q| q.unsafe_raw_value()).collect(); - modified_zscores_f64(&values) -} - #[cfg(test)] mod tests { use super::*;