Merge pull request #954 from sharkdp/codex/remove-statistical

Replace statistical with local floating-point helpers

src/benchmark/executor.rs

@@ -7,14 +7,14 @@ use crate::options::{
     CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
 };
 use crate::output::progress_bar::get_progress_bar;
-use crate::timer::{execute_and_measure, TimerResult};
+use crate::timer::execute_and_measure;
 use crate::util::randomized_environment_offset;
+use crate::util::statistics::mean;
 use crate::util::units::Second;
 
-use super::timing_result::TimingResult;
+use super::measurement::Measurement;
 
 use anyhow::{bail, Context, Result};
-use statistical::mean;
 
 pub enum BenchmarkIteration {
     NonBenchmarkRun,
@@ -40,7 +40,7 @@ pub trait Executor {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)>;
+    ) -> Result<Measurement>;
 
     /// Perform a calibration of this executor. For example,
     /// when running commands through a shell, we need to
@@ -62,7 +62,7 @@ fn run_command_and_measure_common(
     command_input_policy: &CommandInputPolicy,
     command_output_policy: &CommandOutputPolicy,
     command_name: &str,
-) -> Result<TimerResult> {
+) -> Result<Measurement> {
     let stdin = command_input_policy.get_stdin()?;
     let (stdout, stderr) = command_output_policy.get_stdout_stderr()?;
     command.stdin(stdin).stdout(stdout).stderr(stderr);
@@ -136,25 +136,15 @@ impl Executor for RawExecutor<'_> {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
-        let result = run_command_and_measure_common(
+    ) -> Result<Measurement> {
+        run_command_and_measure_common(
             command.get_command()?,
             iteration,
             command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
             &self.options.command_input_policy,
             output_policy,
             &command.get_command_line(),
-        )?;
-
-        Ok((
-            TimingResult {
-                time_real: result.time_real,
-                time_user: result.time_user,
-                time_system: result.time_system,
-                memory_usage_byte: result.memory_usage_byte,
-            },
-            result.status,
-        ))
+        )
     }
 
     fn calibrate(&mut self) -> Result<()> {
@@ -169,7 +159,7 @@ impl Executor for RawExecutor<'_> {
 pub struct ShellExecutor<'a> {
     options: &'a Options,
     shell: &'a Shell,
-    shell_spawning_time: Option<TimingResult>,
+    shell_spawning_time: Option<Measurement>,
 }
 
 impl<'a> ShellExecutor<'a> {
@@ -189,7 +179,7 @@ impl Executor for ShellExecutor<'_> {
         iteration: BenchmarkIteration,
         command_failure_action: Option<CmdFailureAction>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
+    ) -> Result<Measurement> {
         let on_windows_cmd = cfg!(windows) && *self.shell == Shell::Default("cmd.exe");
         let mut command_builder = self.shell.command();
         command_builder.arg(if on_windows_cmd { "/C" } else { "-c" });
@@ -218,15 +208,7 @@ impl Executor for ShellExecutor<'_> {
             result.time_system = (result.time_system - spawning_time.time_system).max(0.0);
         }
 
-        Ok((
-            TimingResult {
-                time_real: result.time_real,
-                time_user: result.time_user,
-                time_system: result.time_system,
-                memory_usage_byte: result.memory_usage_byte,
-            },
-            result.status,
-        ))
+        Ok(result)
     }
 
     /// Measure the average shell spawning time
@@ -268,7 +250,7 @@ impl Executor for ShellExecutor<'_> {
                         shell_cmd
                     );
                 }
-                Ok((r, _)) => {
+                Ok(r) => {
                     times_real.push(r.time_real);
                     times_user.push(r.time_user);
                     times_system.push(r.time_system);
@@ -284,11 +266,12 @@ impl Executor for ShellExecutor<'_> {
             bar.finish_and_clear()
         }
 
-        self.shell_spawning_time = Some(TimingResult {
+        self.shell_spawning_time = Some(Measurement {
             time_real: mean(&times_real),
             time_user: mean(&times_user),
             time_system: mean(&times_system),
             memory_usage_byte: 0,
+            status: ExitStatus::default(),
         });
 
         Ok(())
@@ -326,7 +309,7 @@ impl Executor for MockExecutor {
         _iteration: BenchmarkIteration,
         _command_failure_action: Option<CmdFailureAction>,
         _output_policy: &CommandOutputPolicy,
-    ) -> Result<(TimingResult, ExitStatus)> {
+    ) -> Result<Measurement> {
         #[cfg(unix)]
         let status = {
             use std::os::unix::process::ExitStatusExt;
@@ -339,15 +322,13 @@ impl Executor for MockExecutor {
             ExitStatus::from_raw(0)
         };
 
-        Ok((
-            TimingResult {
-                time_real: Self::extract_time(command.get_command_line()),
-                time_user: 0.0,
-                time_system: 0.0,
-                memory_usage_byte: 0,
-            },
+        Ok(Measurement {
+            time_real: Self::extract_time(command.get_command_line()),
+            time_user: 0.0,
+            time_system: 0.0,
+            memory_usage_byte: 0,
             status,
-        ))
+        })
     }
 
     fn calibrate(&mut self) -> Result<()> {

src/benchmark/measurement.rs

@@ -1,8 +1,10 @@
+use std::process::ExitStatus;
+
 use crate::util::units::Second;
 
-/// Results from timing a single command
+/// Performance measurements and exit status from running a single command
 #[derive(Debug, Default, Copy, Clone)]
-pub struct TimingResult {
+pub struct Measurement {
     /// Wall clock time
     pub time_real: Second,
 
@@ -14,4 +16,7 @@ pub struct TimingResult {
 
     /// Maximum amount of memory used, in bytes
     pub memory_usage_byte: u64,
+
+    /// The exit status of the process
+    pub status: ExitStatus,
 }

src/benchmark/mod.rs

@@ -1,8 +1,8 @@
 pub mod benchmark_result;
 pub mod executor;
+pub mod measurement;
 pub mod relative_speed;
 pub mod scheduler;
-pub mod timing_result;
 
 use std::cmp;
 use std::io::{self, Write};
@@ -19,14 +19,13 @@ use crate::output::progress_bar::get_progress_bar;
 use crate::output::warnings::{OutlierWarningOptions, Warnings};
 use crate::parameter::ParameterNameAndValue;
 use crate::util::exit_code::extract_exit_code;
-use crate::util::min_max::{max, min};
+use crate::util::statistics::{max, mean, median, min, standard_deviation};
 use crate::util::units::Second;
 use benchmark_result::BenchmarkResult;
-use timing_result::TimingResult;
+use measurement::Measurement;
 
 use anyhow::{anyhow, Result};
 use colored::*;
-use statistical::{mean, median, standard_deviation};
 
 use self::executor::Executor;
 
@@ -61,7 +60,7 @@ impl<'a> Benchmark<'a> {
         command: &Command<'_>,
         error_output: &'static str,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         self.executor
             .run_command_and_measure(
                 command,
@@ -69,7 +68,6 @@ impl<'a> Benchmark<'a> {
                 Some(CmdFailureAction::RaiseError),
                 output_policy,
             )
-            .map(|r| r.0)
             .map_err(|_| anyhow!(error_output))
     }
 
@@ -78,7 +76,7 @@ impl<'a> Benchmark<'a> {
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let command = self
             .options
             .setup_command
@@ -99,7 +97,7 @@ impl<'a> Benchmark<'a> {
         &self,
         parameters: impl IntoIterator<Item = ParameterNameAndValue<'a>>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let command = self
             .options
             .cleanup_command
@@ -120,7 +118,7 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let error_output = "The preparation command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
@@ -132,7 +130,7 @@ impl<'a> Benchmark<'a> {
         &self,
         command: &Command<'_>,
         output_policy: &CommandOutputPolicy,
-    ) -> Result<TimingResult> {
+    ) -> Result<Measurement> {
         let error_output = "The conclusion command terminated with a non-zero exit code. \
                             Append ' || true' to the command if you are sure that this can be ignored.";
 
@@ -247,13 +245,13 @@ impl<'a> Benchmark<'a> {
             preparation_result.map_or(0.0, |res| res.time_real + self.executor.time_overhead());
 
         // Initial timing run
-        let (res, status) = self.executor.run_command_and_measure(
+        let res = self.executor.run_command_and_measure(
             self.command,
             BenchmarkIteration::Benchmark(0),
             None,
             output_policy,
         )?;
-        let success = status.success();
+        let success = res.status.success();
 
         let conclusion_result = run_conclusion_command()?;
         let conclusion_overhead =
@@ -284,7 +282,7 @@ impl<'a> Benchmark<'a> {
         times_user.push(res.time_user);
         times_system.push(res.time_system);
         memory_usage_byte.push(res.memory_usage_byte);
-        exit_codes.push(extract_exit_code(status));
+        exit_codes.push(extract_exit_code(res.status));
 
         all_succeeded = all_succeeded && success;
 
@@ -309,19 +307,19 @@ impl<'a> Benchmark<'a> {
                 bar.set_message(msg.to_owned())
             }
 
-            let (res, status) = self.executor.run_command_and_measure(
+            let res = self.executor.run_command_and_measure(
                 self.command,
                 BenchmarkIteration::Benchmark(i + 1),
                 None,
                 output_policy,
             )?;
-            let success = status.success();
+            let success = res.status.success();
 
             times_real.push(res.time_real);
             times_user.push(res.time_user);
             times_system.push(res.time_system);
             memory_usage_byte.push(res.memory_usage_byte);
-            exit_codes.push(extract_exit_code(status));
+            exit_codes.push(extract_exit_code(res.status));
 
             all_succeeded = all_succeeded && success;
 
@@ -340,13 +338,13 @@ impl<'a> Benchmark<'a> {
         let t_num = times_real.len();
         let t_mean = mean(&times_real);
         let t_stddev = if times_real.len() > 1 {
-            Some(standard_deviation(&times_real, Some(t_mean)))
+            Some(standard_deviation(&times_real, t_mean))
         } else {
             None
         };
         let t_median = median(&times_real);
-        let t_min = min(&times_real);
-        let t_max = max(&times_real);
+        let t_min = min(times_real.iter().copied());
+        let t_max = max(times_real.iter().copied());
 
         let user_mean = mean(&times_user);
         let system_mean = mean(&times_system);

src/outlier_detection.rs

@@ -5,7 +5,7 @@
 //!   The ASQC Basic References in Quality Control: Statistical Techniques, Edward F. Mykytka,
 //!   Ph.D., Editor.
 
-use statistical::median;
+use crate::util::statistics::median;
 
 /// Minimum modified Z-score for a datapoint to be an outlier. Here, 1.4826 is a factor that
 /// converts the MAD to an estimator for the standard deviation. The second factor is the number

src/timer/mod.rs

@@ -16,11 +16,11 @@ use std::os::fd::AsFd;
 #[cfg(target_os = "windows")]
 use windows_sys::Win32::System::Threading::CREATE_SUSPENDED;
 
-use crate::util::units::Second;
+use crate::benchmark::measurement::Measurement;
 use wall_clock_timer::WallClockTimer;
 
 use std::io::Read;
-use std::process::{ChildStdout, Command, ExitStatus};
+use std::process::{ChildStdout, Command};
 
 use anyhow::Result;
 
@@ -37,17 +37,6 @@ struct CPUTimes {
     pub memory_usage_byte: u64,
 }
 
-/// Used to indicate the result of running a command
-#[derive(Debug, Copy, Clone)]
-pub struct TimerResult {
-    pub time_real: Second,
-    pub time_user: Second,
-    pub time_system: Second,
-    pub memory_usage_byte: u64,
-    /// The exit status of the process
-    pub status: ExitStatus,
-}
-
 /// Discard the output of a child process.
 fn discard(output: ChildStdout) {
     const CHUNK_SIZE: usize = 64 << 10;
@@ -80,7 +69,7 @@ fn discard(output: ChildStdout) {
 }
 
 /// Execute the given command and return a timing summary
-pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
+pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
     #[cfg(not(windows))]
     let cpu_timer = self::unix_timer::CPUTimer::start();
 
@@ -111,7 +100,7 @@ pub fn execute_and_measure(mut command: Command) -> Result<TimerResult> {
     let time_real = wallclock_timer.stop();
     let (time_user, time_system, memory_usage_byte) = cpu_timer.stop();
 
-    Ok(TimerResult {
+    Ok(Measurement {
         time_real,
         time_user,
         time_system,

src/util/min_max.rs

@@ -1,28 +0,0 @@
-/// A max function for f64's without NaNs
-pub fn max(vals: &[f64]) -> f64 {
-    *vals
-        .iter()
-        .max_by(|a, b| a.partial_cmp(b).unwrap())
-        .unwrap()
-}
-
-/// A min function for f64's without NaNs
-pub fn min(vals: &[f64]) -> f64 {
-    *vals
-        .iter()
-        .min_by(|a, b| a.partial_cmp(b).unwrap())
-        .unwrap()
-}
-
-#[test]
-fn test_max() {
-    let assert_float_eq = |a: f64, b: f64| {
-        assert!((a - b).abs() < f64::EPSILON);
-    };
-
-    assert_float_eq(1.0, max(&[1.0]));
-    assert_float_eq(-1.0, max(&[-1.0]));
-    assert_float_eq(-1.0, max(&[-2.0, -1.0]));
-    assert_float_eq(1.0, max(&[-1.0, 1.0]));
-    assert_float_eq(1.0, max(&[-1.0, 1.0, 0.0]));
-}

src/util/mod.rs

@@ -1,5 +1,5 @@
 pub mod exit_code;
-pub mod min_max;
 pub mod number;
 pub mod randomized_environment_offset;
+pub mod statistics;
 pub mod units;

src/util/statistics.rs

@@ -0,0 +1,113 @@
+//! Statistics for non-empty samples of finite measurements.
+
+/// A min function that assumes no NaNs and at least one element
+pub fn min<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
+    values
+        .into_iter()
+        .min_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
+        .expect("'min' requires at least one element")
+}
+
+/// A max function that assumes no NaNs and at least one element
+pub fn max<Q: PartialOrd>(values: impl IntoIterator<Item = Q>) -> Q {
+    values
+        .into_iter()
+        .max_by(|a, b| a.partial_cmp(b).expect("No NaN values"))
+        .expect("'max' requires at least one element")
+}
+
+/// Arithmetic mean, summing measurements in their original order.
+pub fn mean(values: &[f64]) -> f64 {
+    values.iter().fold(0.0, |sum, value| sum + value) / values.len() as f64
+}
+
+/// Median without modifying the input sample.
+pub fn median(values: &[f64]) -> f64 {
+    assert!(
+        !values.is_empty(),
+        "median requires at least one measurement"
+    );
+    let mut sorted = values.to_vec();
+    sorted.sort_unstable_by(|a, b| a.partial_cmp(b).expect("No NaN measurements"));
+
+    let mid = sorted.len() / 2;
+    if sorted.len() % 2 == 1 {
+        sorted[mid]
+    } else {
+        (sorted[mid] + sorted[mid - 1]) / 2.0
+    }
+}
+
+/// Sample standard deviation, using an already computed mean.
+pub fn standard_deviation(values: &[f64], mean: f64) -> f64 {
+    assert!(
+        values.len() > 1,
+        "standard deviation requires at least two measurements"
+    );
+    let sum = values
+        .iter()
+        .map(|value| (value - mean) * (value - mean))
+        .fold(0.0, |sum, deviation| sum + deviation);
+    assert!(sum >= 0.0, "invalid sum of squared deviations");
+
+    // Preserve the operation order of statistical 1.0: dividing by n - 1
+    // before taking the square root also preserves exported floating-point values.
+    (sum / (values.len() as f64 - 1.0)).sqrt()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use approx::assert_relative_eq;
+
+    #[test]
+    fn test_min() {
+        assert_eq!(1.0, min([1.0]));
+        assert_eq!(-1.0, min([-1.0]));
+        assert_eq!(-2.0, min([-2.0, -1.0]));
+        assert_eq!(-1.0, min([-1.0, 1.0]));
+        assert_eq!(-1.0, min([1.0, -1.0, 0.0]));
+
+        let values = vec![1024.0, 2.0, 3.0];
+        assert_eq!(*min(&values), 2.0);
+    }
+
+    #[test]
+    fn test_max() {
+        assert_eq!(1.0, max([1.0]));
+        assert_eq!(-1.0, max([-1.0]));
+        assert_eq!(-1.0, max([-2.0, -1.0]));
+        assert_eq!(1.0, max([-1.0, 1.0]));
+        assert_eq!(1.0, max([-1.0, 1.0, 0.0]));
+
+        let values = vec![1.0, 2048.0, 3.0];
+        assert_eq!(*max(&values), 2048.0);
+    }
+
+    #[test]
+    fn test_mean() {
+        assert_eq!(1.0, mean(&[1.0]));
+        assert_relative_eq!(2.0, mean(&[1.0, 3.0]));
+
+        let values = [0.1, 0.2, 0.6];
+        assert_relative_eq!(mean(&values), 0.3);
+    }
+
+    #[test]
+    fn test_median() {
+        assert_eq!(1.0, median(&[1.0]));
+        assert_relative_eq!(2.0, median(&[1.0, 3.0]));
+
+        let values = [0.1, 0.2, 0.6];
+        assert_relative_eq!(median(&values), 0.2);
+
+        let values = [0.1, 0.2, 0.3, 0.6];
+        assert_relative_eq!(median(&values), 0.25);
+    }
+
+    #[test]
+    fn test_standard_deviation() {
+        let values = [0.1, 0.2, 0.3];
+        assert_relative_eq!(standard_deviation(&values, mean(&values)), 0.1);
+    }
+}