diff --git a/src/benchmark.rs b/src/benchmark.rs index 7f0d1e9..301443c 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -391,10 +391,11 @@ pub fn run_benchmark( // Compute statistical quantities let t_num = times_real.len(); let t_mean = mean(×_real); - let mut t_stddev = 0.0; - if times_real.len() != 1 { - t_stddev = standard_deviation(×_real, Some(t_mean)); - } + let t_stddev = if times_real.len() > 1 { + Some(standard_deviation(×_real, Some(t_mean))) + } else { + None + }; let t_median = median(×_real); let t_min = min(×_real); let t_max = max(×_real); @@ -404,7 +405,6 @@ pub fn run_benchmark( // Formatting and console output let (mean_str, time_unit) = format_duration_unit(t_mean, options.time_unit); - let stddev_str = format_duration(t_stddev, Some(time_unit)); let min_str = format_duration(t_min, Some(time_unit)); let max_str = format_duration(t_max, Some(time_unit)); let num_str = format!("{} runs", t_num); @@ -412,34 +412,38 @@ pub fn run_benchmark( let user_str = format_duration(user_mean, Some(time_unit)); let system_str = format_duration(system_mean, Some(time_unit)); - if options.output_style != OutputStyleOption::Disabled && times_real.len() == 1 { - println!( - " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", - "abs".green().bold(), - mean_str.green().bold(), - " ".to_string(), // alignment - user_str.blue(), - system_str.blue() - ); - } else if options.output_style != OutputStyleOption::Disabled { - println!( - " Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]", - "mean".green().bold(), - "σ".green(), - mean_str.green().bold(), - stddev_str.green(), - user_str.blue(), - system_str.blue() - ); + if options.output_style != OutputStyleOption::Disabled { + if times_real.len() == 1 { + println!( + " Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]", + "abs".green().bold(), + mean_str.green().bold(), + " ".to_string(), // alignment + user_str.blue(), + system_str.blue() + ); + } else { + let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit)); - println!( - " Range ({} … {}): {:>8} … {:>8} {}", - "min".cyan(), - "max".purple(), - min_str.cyan(), - max_str.purple(), - num_str.dimmed() - ); + println!( + " Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]", + "mean".green().bold(), + "σ".green(), + mean_str.green().bold(), + stddev_str.green(), + user_str.blue(), + system_str.blue() + ); + + println!( + " Range ({} … {}): {:>8} … {:>8} {}", + "min".cyan(), + "max".purple(), + min_str.cyan(), + max_str.purple(), + num_str.dimmed() + ); + } } // Warnings diff --git a/src/benchmark_result.rs b/src/benchmark_result.rs index b47bbfd..72f8483 100644 --- a/src/benchmark_result.rs +++ b/src/benchmark_result.rs @@ -15,8 +15,8 @@ pub struct BenchmarkResult { /// The mean run time pub mean: Second, - /// The standard deviation of all run times - pub stddev: Second, + /// The standard deviation of all run times. Not available if only one run has been performed + pub stddev: Option, /// The median run time pub median: Second, @@ -51,7 +51,7 @@ impl BenchmarkResult { pub fn new( command: String, mean: Second, - stddev: Second, + stddev: Option, median: Second, user: Second, system: Second, diff --git a/src/export/asciidoc.rs b/src/export/asciidoc.rs index 8afafa9..8ce14b5 100644 --- a/src/export/asciidoc.rs +++ b/src/export/asciidoc.rs @@ -60,7 +60,7 @@ fn table_row(entry: &BenchmarkResult, unit: Unit) -> Vec { | {}…{}\n", entry.command.replace("|", "\\|"), form(entry.mean).0, - form(entry.stddev).0, + entry.stddev.map(|s| form(s).0).unwrap_or("?".into()), form(entry.min).0, form(entry.max).0 ) @@ -84,14 +84,14 @@ fn test_asciidoc_header() { fn test_asciidoc_table_row() { use std::collections::BTreeMap; let result = BenchmarkResult::new( - String::from("sleep 1"), // command - 0.10491992406666667, // mean - 0.00397851689425097, // stddev - 0.10491992406666667, // median - 0.005182013333333333, // user - 0.0, // system - 0.1003342584, // min - 0.10745223440000001, // max + String::from("sleep 1"), // command + 0.10491992406666667, // mean + Some(0.00397851689425097), // stddev + 0.10491992406666667, // median + 0.005182013333333333, // user + 0.0, // system + 0.1003342584, // min + 0.10745223440000001, // max vec![ // times 0.1003342584, @@ -108,7 +108,7 @@ fn test_asciidoc_table_row() { | {}…{}\n", result.command, Unit::MilliSecond.format(result.mean), - Unit::MilliSecond.format(result.stddev), + Unit::MilliSecond.format(result.stddev.unwrap()), Unit::MilliSecond.format(result.min), Unit::MilliSecond.format(result.max) ) @@ -119,7 +119,7 @@ fn test_asciidoc_table_row() { | {}…{}\n", result.command, Unit::Second.format(result.mean), - Unit::Second.format(result.stddev), + Unit::Second.format(result.stddev.unwrap()), Unit::Second.format(result.min), Unit::Second.format(result.max) ) @@ -137,14 +137,14 @@ fn test_asciidoc_table_row() { fn test_asciidoc_table_row_command_escape() { use std::collections::BTreeMap; let result = BenchmarkResult::new( - String::from("sleep 1|"), // command - 0.10491992406666667, // mean - 0.00397851689425097, // stddev - 0.10491992406666667, // median - 0.005182013333333333, // user - 0.0, // system - 0.1003342584, // min - 0.10745223440000001, // max + String::from("sleep 1|"), // command + 0.10491992406666667, // mean + Some(0.00397851689425097), // stddev + 0.10491992406666667, // median + 0.005182013333333333, // user + 0.0, // system + 0.1003342584, // min + 0.10745223440000001, // max vec![ // times 0.1003342584, @@ -159,7 +159,7 @@ fn test_asciidoc_table_row_command_escape() { | {} ± {}\n\ | {}…{}\n", Unit::Second.format(result.mean), - Unit::Second.format(result.stddev), + Unit::Second.format(result.stddev.unwrap()), Unit::Second.format(result.min), Unit::Second.format(result.max) ) @@ -179,7 +179,7 @@ fn test_asciidoc() { BenchmarkResult::new( String::from("FOO=1 BAR=2 command | 1"), 1.0, - 2.0, + Some(2.0), 1.0, 3.0, 4.0, @@ -197,7 +197,7 @@ fn test_asciidoc() { BenchmarkResult::new( String::from("FOO=1 BAR=7 command | 2"), 11.0, - 12.0, + Some(12.0), 11.0, 13.0, 14.0, diff --git a/src/export/csv.rs b/src/export/csv.rs index a954a93..435b4d6 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -33,7 +33,13 @@ impl Exporter for CsvExporter { for res in results { let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; for f in &[ - res.mean, res.stddev, res.median, res.user, res.system, res.min, res.max, + res.mean, + res.stddev.unwrap_or(0.0), + res.median, + res.user, + res.system, + res.min, + res.max, ] { fields.push(Cow::Owned(f.to_string().into_bytes())) } @@ -59,7 +65,7 @@ fn test_csv() { BenchmarkResult::new( String::from("FOO=one BAR=two command | 1"), 1.0, - 2.0, + Some(2.0), 1.0, 3.0, 4.0, @@ -77,7 +83,7 @@ fn test_csv() { BenchmarkResult::new( String::from("FOO=one BAR=seven command | 2"), 11.0, - 12.0, + Some(12.0), 11.0, 13.0, 14.0, diff --git a/src/export/markdown.rs b/src/export/markdown.rs index 302915f..0999767 100644 --- a/src/export/markdown.rs +++ b/src/export/markdown.rs @@ -53,19 +53,27 @@ fn start_table(unit: Unit) -> Vec { fn add_table_row(dest: &mut Vec, entry: &BenchmarkResultWithRelativeSpeed, unit: Unit) { let result = &entry.result; let mean_str = format_duration_value(result.mean, Some(unit)).0; - let stddev_str = format_duration_value(result.stddev, Some(unit)).0; + let stddev_str = if let Some(stddev) = result.stddev { + format!(" ± {}", format_duration_value(stddev, Some(unit)).0) + } else { + "".into() + }; let min_str = format_duration_value(result.min, Some(unit)).0; let max_str = format_duration_value(result.max, Some(unit)).0; let rel_str = format!("{:.2}", entry.relative_speed); let rel_stddev_str = if entry.is_fastest { "".into() } else { - format!(" ± {:.2}", entry.relative_speed_stddev) + if let Some(stddev) = entry.relative_speed_stddev { + format!(" ± {:.2}", stddev) + } else { + "".into() + } }; dest.extend( format!( - "| `{command}` | {mean} ± {stddev} | {min} | {max} | {rel}{rel_stddev} |\n", + "| `{command}` | {mean}{stddev} | {min} | {max} | {rel}{rel_stddev} |\n", command = result.command.replace("|", "\\|"), mean = mean_str, stddev = stddev_str, @@ -93,7 +101,7 @@ fn test_markdown_format_ms() { BenchmarkResult::new( String::from("sleep 0.1"), 0.1057, // mean - 0.0016, // std dev + Some(0.0016), // std dev 0.1057, // median 0.0009, // user_mean 0.0011, // system_mean @@ -106,7 +114,7 @@ fn test_markdown_format_ms() { BenchmarkResult::new( String::from("sleep 2"), 2.0050, // mean - 0.0020, // std dev + Some(0.0020), // std dev 2.0050, // median 0.0009, // user_mean 0.0012, // system_mean @@ -142,7 +150,7 @@ fn test_markdown_format_s() { BenchmarkResult::new( String::from("sleep 2"), 2.0050, // mean - 0.0020, // std dev + Some(0.0020), // std dev 2.0050, // median 0.0009, // user_mean 0.0012, // system_mean @@ -155,7 +163,7 @@ fn test_markdown_format_s() { BenchmarkResult::new( String::from("sleep 0.1"), 0.1057, // mean - 0.0016, // std dev + Some(0.0016), // std dev 0.1057, // median 0.0009, // user_mean 0.0011, // system_mean @@ -190,7 +198,7 @@ fn test_markdown_format_time_unit_s() { BenchmarkResult::new( String::from("sleep 0.1"), 0.1057, // mean - 0.0016, // std dev + Some(0.0016), // std dev 0.1057, // median 0.0009, // user_mean 0.0011, // system_mean @@ -203,7 +211,7 @@ fn test_markdown_format_time_unit_s() { BenchmarkResult::new( String::from("sleep 2"), 2.0050, // mean - 0.0020, // std dev + Some(0.0020), // std dev 2.0050, // median 0.0009, // user_mean 0.0012, // system_mean @@ -244,7 +252,7 @@ fn test_markdown_format_time_unit_ms() { BenchmarkResult::new( String::from("sleep 2"), 2.0050, // mean - 0.0020, // std dev + Some(0.0020), // std dev 2.0050, // median 0.0009, // user_mean 0.0012, // system_mean @@ -257,7 +265,7 @@ fn test_markdown_format_time_unit_ms() { BenchmarkResult::new( String::from("sleep 0.1"), 0.1057, // mean - 0.0016, // std dev + Some(0.0016), // std dev 0.1057, // median 0.0009, // user_mean 0.0011, // system_mean diff --git a/src/main.rs b/src/main.rs index 63c87bf..6db752a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,9 +61,13 @@ pub fn write_benchmark_comparison(results: &[BenchmarkResult]) { for item in others { println!( - "{} ± {} times faster than '{}'", + "{}{} times faster than '{}'", format!("{:8.2}", item.relative_speed).bold().green(), - format!("{:.2}", item.relative_speed_stddev).green(), + if let Some(stddev) = item.relative_speed_stddev { + format!(" ± {}", format!("{:.2}", stddev).green()) + } else { + "".into() + }, &item.result.command.magenta() ); } diff --git a/src/relative_speed.rs b/src/relative_speed.rs index a87b8f5..093e3c5 100644 --- a/src/relative_speed.rs +++ b/src/relative_speed.rs @@ -6,7 +6,7 @@ use crate::{benchmark_result::BenchmarkResult, units::Scalar}; pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: Scalar, - pub relative_speed_stddev: Scalar, + pub relative_speed_stddev: Option, pub is_fastest: bool, } @@ -32,10 +32,15 @@ pub fn compute(results: &[BenchmarkResult]) -> Option Some( + ratio + * ((result_stddev / result.mean).powi(2) + + (fastest_stddev / fastest.mean).powi(2)) + .sqrt(), + ), + _ => None, + }; BenchmarkResultWithRelativeSpeed { result, @@ -55,7 +60,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { BenchmarkResult { command: name.into(), mean, - stddev: 1.0, + stddev: Some(1.0), median: mean, user: mean, system: 0.0,