Simplifications

src/benchmark/measurement.rs

@@ -2,7 +2,8 @@ use std::process::ExitStatus;
 
 use serde::Serialize;
 
-use crate::quantity::statistics::{mean, modified_zscores};
+use crate::metric::Metric;
+use crate::quantity::statistics::{mean, modified_zscores_f64};
 use crate::quantity::{serialize_information, serialize_time, Information, Time};
 use crate::util::exit_code::extract_exit_code;
 
@@ -157,9 +158,14 @@ impl Measurements {
         mean(self.wall_clock_times())
     }
 
-    /// Compute modified Z-scores for the wall clock times
-    pub fn modified_zscores(&self) -> Vec<f64> {
-        modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
+    /// Compute modified Z-scores for a metric validated to be available in every run.
+    pub fn modified_zscores(&self, metric: Metric) -> Vec<f64> {
+        let values: Vec<_> = self
+            .measurements
+            .iter()
+            .map(|m| metric.value(m).expect("Validated metric is available"))
+            .collect();
+        modified_zscores_f64(&values)
     }
 
     /// The average user time
@@ -172,3 +178,28 @@ impl Measurements {
         mean(self.measurements.iter().map(|m| m.time_system))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::outlier_detection::OUTLIER_THRESHOLD;
+    use crate::quantity::{byte, second};
+
+    #[test]
+    fn outliers_follow_the_selected_metric() {
+        let measurements = Measurements::new(
+            [(1.0, 100.0), (1.0, 1.0), (1.0, 1.0), (100.0, 1.0)]
+                .iter()
+                .copied()
+                .map(|(time, memory)| Measurement {
+                    time_wall_clock: Time::new::<second>(time),
+                    memory_peak_resident: Some(Information::new::<byte>(memory)),
+                    ..Measurement::default()
+                })
+                .collect(),
+        );
+
+        assert!(measurements.modified_zscores(Metric::TimeWallClock)[0] < OUTLIER_THRESHOLD);
+        assert!(measurements.modified_zscores(Metric::MemoryPeakResident)[0] > OUTLIER_THRESHOLD);
+    }
+}

src/benchmark/mod.rs

@@ -412,7 +412,7 @@ impl<'a> Benchmark<'a> {
         }
 
         // Run outlier detection
-        let scores = measurements.modified_zscores();
+        let scores = measurements.modified_zscores(primary.metric);
 
         let outlier_warning_options = OutlierWarningOptions {
             warmup_in_use: self.options.warmup_count > 0,
@@ -425,15 +425,10 @@ impl<'a> Benchmark<'a> {
                 > 0,
         };
 
-        if self
-            .options
-            .metrics
-            .iter()
-            .any(|m| m.metric == crate::metric::Metric::TimeWallClock)
-            && scores[0] > OUTLIER_THRESHOLD
-        {
-            warnings.push(Warnings::SlowInitialRun(
-                measurements.wall_clock_times().next().unwrap(),
+        if scores[0] > OUTLIER_THRESHOLD {
+            warnings.push(Warnings::InitialRunOutlier(
+                primary,
+                primary.metric.value(&measurements.measurements[0]).unwrap(),
                 outlier_warning_options,
             ));
         }

src/output/warnings.rs

@@ -1,8 +1,8 @@
 use std::fmt;
 
 use crate::benchmark::MIN_EXECUTION_TIME;
-use crate::metric::MetricSelection;
-use crate::quantity::{millisecond, second, Time};
+use crate::metric::{Metric, MetricSelection};
+use crate::quantity::millisecond;
 
 pub struct OutlierWarningOptions {
     pub warmup_in_use: bool,
@@ -13,7 +13,7 @@ pub struct OutlierWarningOptions {
 pub enum Warnings {
     FastExecutionTime,
     NonZeroExitCode,
-    SlowInitialRun(Time, OutlierWarningOptions),
+    InitialRunOutlier(MetricSelection, f64, OutlierWarningOptions),
 }
 
 impl fmt::Display for Warnings {
@@ -28,12 +28,18 @@ impl fmt::Display for Warnings {
                 MIN_EXECUTION_TIME.get::<millisecond>()
             ),
             Warnings::NonZeroExitCode => write!(f, "Ignoring non-zero exit code."),
-            Warnings::SlowInitialRun(time_first_run, ref options) => write!(
+            Warnings::InitialRunOutlier(selection, value, _) if selection.metric != Metric::TimeWallClock => write!(
+                f,
+                "The first benchmarking run for this command had an unusually high value for the primary '{}' metric ({}) compared with the rest.",
+                selection.metric.name(),
+                selection.display_unit(value).format(value),
+            ),
+            Warnings::InitialRunOutlier(selection, value, ref options) => write!(
                 f,
                 "The first benchmarking run for this command was significantly slower than the \
                  rest ({time}). This could be caused by (filesystem) caches that were not filled until \
                  after the first run. {hints}",
-                time=MetricSelection::default().display_unit(time_first_run.get::<second>()).format(time_first_run.get::<second>()),
+                time=selection.display_unit(value).format(value),
                 hints=match (options.warmup_in_use, options.prepare_in_use) {
                     (true, true) => "You are already using both the '--warmup' option as well \
                     as the '--prepare' option. Consider re-running the benchmark on a quiet system. \

src/quantity/statistics.rs

@@ -146,11 +146,6 @@ pub fn modified_zscores_f64(xs: &[f64]) -> Vec<f64> {
     xs.iter().map(|&x| (x - x_median) / mad).collect()
 }
 
-pub fn modified_zscores<Q: UnsafeRawValue>(values: &[Q]) -> Vec<f64> {
-    let values: Vec<_> = values.iter().map(|q| q.unsafe_raw_value()).collect();
-    modified_zscores_f64(&values)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;