Use statistical crate

Cargo.lock

@@ -88,6 +88,7 @@ dependencies = [
  "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indicatif 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -133,6 +134,72 @@ dependencies = [
  "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "num"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-bigint"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-iter"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.1.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "owning_ref"
 version = "0.3.3"
@@ -200,6 +267,11 @@ name = "regex-syntax"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "rustc-serialize"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "smallvec"
 version = "0.6.0"
@@ -210,6 +282,15 @@ name = "stable_deref_trait"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "statistical"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "strsim"
 version = "0.6.0"
@@ -329,6 +410,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
 "checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121"
 "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
+"checksum num 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cc4083e14b542ea3eb9b5f33ff48bd373a92d78687e74f4cc0a30caeb754f0ca"
+"checksum num-bigint 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "bdc1494b5912f088f260b775799468d9b9209ac60885d8186a547a0476289e23"
+"checksum num-complex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "58de7b4bf7cf5dbecb635a5797d489864eadd03b107930cbccf9e0fd7428b47c"
+"checksum num-integer 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "d1452e8b06e448a07f0e6ebb0bb1d92b8890eea63288c0b627331d53514d0fba"
+"checksum num-iter 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)" = "7485fcc84f85b4ecd0ea527b14189281cf27d60e583ae65ebc9c088b13dffe01"
+"checksum num-rational 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "0c7cb72a95250d8a370105c828f388932373e0e94414919891a0f945222310fe"
+"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070"
 "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
 "checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412"
 "checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8"
@@ -337,8 +425,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
 "checksum regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "744554e01ccbd98fff8c457c3b092cd67af62a555a43bfe97ae8a0451f7799fa"
 "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
+"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
 "checksum smallvec 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44db0ecb22921ef790d17ae13a3f6d15784183ff5f2a01aa32098c7498d2b4b9"
 "checksum stable_deref_trait 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "15132e0e364248108c5e2c02e3ab539be8d6f5d52a01ca9bbf27ed657316f02b"
+"checksum statistical 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c139942f46d96c53b28420a2cdfb374629f122656bd9daef7fc221ed4d8ec228"
 "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
 "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327"
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"

Cargo.toml

@@ -12,6 +12,7 @@ version = "0.2.0"
 [dependencies]
 ansi_term = "0.10"
 indicatif = "0.8"
+statistical = "0.1"
 
 [dependencies.clap]
 version = "2"

src/hyperfine/benchmark.rs

@@ -3,31 +3,35 @@ use std::process::{Command, Stdio};
 use std::time::Instant;
 
 use ansi_term::Colour::{Cyan, Green, White, Yellow};
+use statistical::{mean, standard_deviation};
 
 use hyperfine::internal::{get_progress_bar, HyperfineOptions, Second, Warnings, MIN_EXECUTION_TIME};
 use hyperfine::format::{format_duration, format_duration_unit, Unit};
-use hyperfine::statistics::mean;
 
 /// Results from timing a single shell command
 pub struct TimingResult {
     /// Execution time in seconds
-    pub execution_time_sec: Second,
+    pub execution_time: Second,
 
     /// True if the command finished with exit code zero
     pub success: bool,
 }
 
 impl TimingResult {
-    fn new(execution_time_sec: Second, success: bool) -> TimingResult {
+    fn new(execution_time: Second, success: bool) -> TimingResult {
         TimingResult {
-            execution_time_sec,
+            execution_time,
             success,
         }
     }
 }
 
 /// Run the given shell command and measure the execution time
-pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result<TimingResult> {
+pub fn time_shell_command(
+    shell_cmd: &str,
+    ignore_failure: bool,
+    shell_spawning_time: Option<Second>,
+) -> io::Result<TimingResult> {
     let start = Instant::now();
 
     let status = Command::new("sh")
@@ -48,9 +52,23 @@ pub fn time_shell_command(shell_cmd: &str, ignore_failure: bool) -> io::Result<T
 
     let duration = start.elapsed();
 
-    let execution_time_sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 * 1e-9;
+    let execution_time = duration.as_secs() as f64 + duration.subsec_nanos() as f64 * 1e-9;
 
-    Ok(TimingResult::new(execution_time_sec, status.success()))
+    // Correct for shell spawning time
+    let execution_time_corrected = if let Some(spawning_time) = shell_spawning_time {
+        if execution_time < spawning_time {
+            0.0
+        } else {
+            execution_time - spawning_time
+        }
+    } else {
+        execution_time
+    };
+
+    Ok(TimingResult::new(
+        execution_time_corrected,
+        status.success(),
+    ))
 }
 
 /// Measure the average shell spawning time
@@ -60,7 +78,8 @@ pub fn mean_shell_spawning_time() -> io::Result<Second> {
 
     let mut times: Vec<Second> = vec![];
     for _ in 0..COUNT {
-        let res = time_shell_command("", false);
+        // Just run the shell without any command
+        let res = time_shell_command("", false, None);
 
         match res {
             Err(_) => {
@@ -71,15 +90,14 @@ pub fn mean_shell_spawning_time() -> io::Result<Second> {
                 ))
             }
             Ok(r) => {
-                times.push(r.execution_time_sec);
+                times.push(r.execution_time);
             }
         }
         bar.inc(1);
     }
     bar.finish_and_clear();
 
-    let mean: f64 = mean(times.iter().cloned()); // TODO: get rid of .cloned()
+    Ok(mean(&times))
-    Ok(mean)
 }
 
 /// Run the benchmark for a single shell command
@@ -89,15 +107,6 @@ pub fn run_benchmark(
     shell_spawning_time: Second,
     options: &HyperfineOptions,
 ) -> io::Result<()> {
-    // Helper function to compute corrected execution times
-    let get_execution_time = |r: &TimingResult| -> Second {
-        if r.execution_time_sec < shell_spawning_time {
-            0.0
-        } else {
-            r.execution_time_sec - shell_spawning_time
-        }
-    };
-
     println!(
         "{}{}: {}",
         White.bold().paint("Benchmark #"),
@@ -106,14 +115,15 @@ pub fn run_benchmark(
     );
     println!();
 
-    let mut results = vec![];
+    let mut execution_times: Vec<Second> = vec![];
+    let mut all_succeeded = true;
 
     // Warmup phase
     if options.warmup_count > 0 {
         let bar = get_progress_bar(options.warmup_count, "Performing warmup runs");
 
         for _ in 0..options.warmup_count {
-            let _ = time_shell_command(cmd, options.ignore_failure)?;
+            let _ = time_shell_command(cmd, options.ignore_failure, None)?;
             bar.inc(1);
         }
         bar.finish_and_clear();
@@ -123,10 +133,11 @@ pub fn run_benchmark(
     let bar = get_progress_bar(options.min_runs, "Initial time measurement");
 
     // Initial timing run
-    let res = time_shell_command(cmd, options.ignore_failure)?;
+    let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?;
 
     // Determine number of benchmark runs
-    let runs_in_min_time = (options.min_time_sec / res.execution_time_sec) as u64;
+    let runs_in_min_time =
+        (options.min_time_sec / (res.execution_time + shell_spawning_time)) as u64;
 
     let count = if runs_in_min_time >= options.min_runs {
         runs_in_min_time
@@ -137,7 +148,8 @@ pub fn run_benchmark(
     let count_remaining = count - 1;
 
     // Save the first result
-    results.push(res);
+    execution_times.push(res.execution_time);
+    all_succeeded = all_succeeded && res.success;
 
     // Re-configure the progress bar
     bar.set_length(count_remaining);
@@ -145,14 +157,16 @@ pub fn run_benchmark(
     // Gather statistics
     for _ in 0..count_remaining {
         let msg = {
-            let execution_times = results.iter().map(&get_execution_time);
+            let mean = format_duration(mean(&execution_times), Unit::Auto);
-            let mean = format_duration(mean(execution_times), Unit::Auto);
             format!("Current estimate: {}", Green.paint(mean))
         };
         bar.set_message(&msg);
 
-        let res = time_shell_command(cmd, options.ignore_failure)?;
+        let res = time_shell_command(cmd, options.ignore_failure, Some(shell_spawning_time))?;
-        results.push(res);
+
+        execution_times.push(res.execution_time);
+        all_succeeded = all_succeeded && res.success;
+
         bar.inc(1);
     }
     bar.finish_and_clear();
@@ -160,15 +174,11 @@ pub fn run_benchmark(
     // Compute statistical quantities
 
     // Corrected execution times
-    let execution_times = results.iter().map(&get_execution_time);
+    let mean = mean(&execution_times);
-
+    let stddev = standard_deviation(&execution_times, Some(mean));
-    let t_mean = mean(execution_times.clone());
-    let t2_mean = mean(execution_times.clone().map(|t| t.powi(2)));
-
-    let stddev = (t2_mean - t_mean.powi(2)).sqrt();
 
     // Formatting and console output
-    let (mean_str, unit_mean) = format_duration_unit(t_mean, Unit::Auto);
+    let (mean_str, unit_mean) = format_duration_unit(mean, Unit::Auto);
     let stddev_str = format_duration(stddev, unit_mean);
     let time_fmt = format!("{} ± {}", mean_str, stddev_str);
 
@@ -178,12 +188,12 @@ pub fn run_benchmark(
     let mut warnings = vec![];
 
     // Check execution time
-    if execution_times.clone().any(|t| t < MIN_EXECUTION_TIME) {
+    if execution_times.iter().any(|&t| t < MIN_EXECUTION_TIME) {
         warnings.push(Warnings::FastExecutionTime);
     }
 
     // Check programm exit codes
-    if results.iter().any(|r| !r.success) {
+    if !all_succeeded {
         warnings.push(Warnings::NonZeroExitCode);
     }
 

src/hyperfine/mod.rs

@@ -1,4 +1,3 @@
 pub mod benchmark;
 pub mod format;
 pub mod internal;
-pub mod statistics;

src/hyperfine/statistics.rs

@@ -1,13 +0,0 @@
-/// Calculate statistical average
-pub fn mean<I>(values: I) -> f64
-where
-    I: IntoIterator<Item = f64>,
-{
-    let mut sum: f64 = 0.0;
-    let mut len: u64 = 0;
-    for v in values {
-        sum += v;
-        len += 1;
-    }
-    sum / (len as f64)
-}

src/main.rs

@@ -2,6 +2,7 @@ extern crate ansi_term;
 #[macro_use]
 extern crate clap;
 extern crate indicatif;
+extern crate statistical;
 
 use std::cmp;
 use std::error::Error;
@@ -81,7 +82,8 @@ fn main() {
         .unwrap_or(0);
 
     if let Some(min_runs) = matches.value_of("min-runs").and_then(&str_to_u64) {
-        options.min_runs = cmp::max(1, min_runs);
+        // we need at least two runs to compute a variance
+        options.min_runs = cmp::max(2, min_runs);
     }
 
     options.ignore_failure = matches.is_present("ignore-failure");