vsndrg/hyperfine
1
0 Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-10-31 16:40:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make stddev fields optional, hide "± σ" for 1 run

Browse Source
This commit is contained in:
David Peter authored and David Peter committed 2021-11-14 10:28:24 +01:00
1 parent bffd476128
commit abd0d335ab
7 files changed
+106 -79
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/benchmark.rs
+36 -32
View File
@@ -391,10 +391,11 @@ pub fn run_benchmark(
// Compute statistical quantities
let t_num = times_real.len();
let t_mean = mean(&times_real);
let mut t_stddev = 0.0;
if times_real.len() != 1 {
t_stddev = standard_deviation(&times_real, Some(t_mean));
}
let t_stddev = if times_real.len() > 1 {
Some(standard_deviation(&times_real, Some(t_mean)))
} else {
None
};
let t_median = median(&times_real);
let t_min = min(&times_real);
let t_max = max(&times_real);
@@ -404,7 +405,6 @@ pub fn run_benchmark(
// Formatting and console output
let (mean_str, time_unit) = format_duration_unit(t_mean, options.time_unit);
let stddev_str = format_duration(t_stddev, Some(time_unit));
let min_str = format_duration(t_min, Some(time_unit));
let max_str = format_duration(t_max, Some(time_unit));
let num_str = format!("{} runs", t_num);
@@ -412,34 +412,38 @@ pub fn run_benchmark(
let user_str = format_duration(user_mean, Some(time_unit));
let system_str = format_duration(system_mean, Some(time_unit));
if options.output_style != OutputStyleOption::Disabled && times_real.len() == 1 {
println!(
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
"abs".green().bold(),
mean_str.green().bold(),
" ".to_string(), // alignment
user_str.blue(),
system_str.blue()
);
} else if options.output_style != OutputStyleOption::Disabled {
println!(
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
"mean".green().bold(),
"σ".green(),
mean_str.green().bold(),
stddev_str.green(),
user_str.blue(),
system_str.blue()
);
if options.output_style != OutputStyleOption::Disabled {
if times_real.len() == 1 {
println!(
" Time ({} ≡): {:>8} {:>8} [User: {}, System: {}]",
"abs".green().bold(),
mean_str.green().bold(),
" ".to_string(), // alignment
user_str.blue(),
system_str.blue()
);
} else {
let stddev_str = format_duration(t_stddev.unwrap(), Some(time_unit));
println!(
" Range ({} … {}): {:>8} … {:>8} {}",
"min".cyan(),
"max".purple(),
min_str.cyan(),
max_str.purple(),
num_str.dimmed()
);
println!(
" Time ({} ± {}): {:>8} ± {:>8} [User: {}, System: {}]",
"mean".green().bold(),
"σ".green(),
mean_str.green().bold(),
stddev_str.green(),
user_str.blue(),
system_str.blue()
);
println!(
" Range ({} … {}): {:>8} … {:>8} {}",
"min".cyan(),
"max".purple(),
min_str.cyan(),
max_str.purple(),
num_str.dimmed()
);
}
}
// Warnings
src/benchmark_result.rs
+3 -3
View File
@@ -15,8 +15,8 @@ pub struct BenchmarkResult {
/// The mean run time
pub mean: Second,
/// The standard deviation of all run times
pub stddev: Second,
/// The standard deviation of all run times. Not available if only one run has been performed
pub stddev: Option<Second>,
/// The median run time
pub median: Second,
@@ -51,7 +51,7 @@ impl BenchmarkResult {
pub fn new(
command: String,
mean: Second,
stddev: Second,
stddev: Option<Second>,
median: Second,
user: Second,
system: Second,
src/export/asciidoc.rs
+22 -22
View File
@@ -60,7 +60,7 @@ fn table_row(entry: &BenchmarkResult, unit: Unit) -> Vec<u8> {
| {}…{}\n",
entry.command.replace("|", "\\|"),
form(entry.mean).0,
form(entry.stddev).0,
entry.stddev.map(|s| form(s).0).unwrap_or("?".into()),
form(entry.min).0,
form(entry.max).0
)
@@ -84,14 +84,14 @@ fn test_asciidoc_header() {
fn test_asciidoc_table_row() {
use std::collections::BTreeMap;
let result = BenchmarkResult::new(
String::from("sleep 1"), // command
0.10491992406666667, // mean
0.00397851689425097, // stddev
0.10491992406666667, // median
0.005182013333333333, // user
0.0, // system
0.1003342584, // min
0.10745223440000001, // max
String::from("sleep 1"), // command
0.10491992406666667, // mean
Some(0.00397851689425097), // stddev
0.10491992406666667, // median
0.005182013333333333, // user
0.0, // system
0.1003342584, // min
0.10745223440000001, // max
vec![
// times
0.1003342584,
@@ -108,7 +108,7 @@ fn test_asciidoc_table_row() {
| {}…{}\n",
result.command,
Unit::MilliSecond.format(result.mean),
Unit::MilliSecond.format(result.stddev),
Unit::MilliSecond.format(result.stddev.unwrap()),
Unit::MilliSecond.format(result.min),
Unit::MilliSecond.format(result.max)
)
@@ -119,7 +119,7 @@ fn test_asciidoc_table_row() {
| {}…{}\n",
result.command,
Unit::Second.format(result.mean),
Unit::Second.format(result.stddev),
Unit::Second.format(result.stddev.unwrap()),
Unit::Second.format(result.min),
Unit::Second.format(result.max)
)
@@ -137,14 +137,14 @@ fn test_asciidoc_table_row() {
fn test_asciidoc_table_row_command_escape() {
use std::collections::BTreeMap;
let result = BenchmarkResult::new(
String::from("sleep 1|"), // command
0.10491992406666667, // mean
0.00397851689425097, // stddev
0.10491992406666667, // median
0.005182013333333333, // user
0.0, // system
0.1003342584, // min
0.10745223440000001, // max
String::from("sleep 1|"), // command
0.10491992406666667, // mean
Some(0.00397851689425097), // stddev
0.10491992406666667, // median
0.005182013333333333, // user
0.0, // system
0.1003342584, // min
0.10745223440000001, // max
vec![
// times
0.1003342584,
@@ -159,7 +159,7 @@ fn test_asciidoc_table_row_command_escape() {
| {} ± {}\n\
| {}…{}\n",
Unit::Second.format(result.mean),
Unit::Second.format(result.stddev),
Unit::Second.format(result.stddev.unwrap()),
Unit::Second.format(result.min),
Unit::Second.format(result.max)
)
@@ -179,7 +179,7 @@ fn test_asciidoc() {
BenchmarkResult::new(
String::from("FOO=1 BAR=2 command | 1"),
1.0,
2.0,
Some(2.0),
1.0,
3.0,
4.0,
@@ -197,7 +197,7 @@ fn test_asciidoc() {
BenchmarkResult::new(
String::from("FOO=1 BAR=7 command | 2"),
11.0,
12.0,
Some(12.0),
11.0,
13.0,
14.0,
src/export/csv.rs
+9 -3
View File
@@ -33,7 +33,13 @@ impl Exporter for CsvExporter {
for res in results {
let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
for f in &[
res.mean, res.stddev, res.median, res.user, res.system, res.min, res.max,
res.mean,
res.stddev.unwrap_or(0.0),
res.median,
res.user,
res.system,
res.min,
res.max,
] {
fields.push(Cow::Owned(f.to_string().into_bytes()))
}
@@ -59,7 +65,7 @@ fn test_csv() {
BenchmarkResult::new(
String::from("FOO=one BAR=two command | 1"),
1.0,
2.0,
Some(2.0),
1.0,
3.0,
4.0,
@@ -77,7 +83,7 @@ fn test_csv() {
BenchmarkResult::new(
String::from("FOO=one BAR=seven command | 2"),
11.0,
12.0,
Some(12.0),
11.0,
13.0,
14.0,
src/export/markdown.rs
+19 -11
View File
@@ -53,19 +53,27 @@ fn start_table(unit: Unit) -> Vec<u8> {
fn add_table_row(dest: &mut Vec<u8>, entry: &BenchmarkResultWithRelativeSpeed, unit: Unit) {
let result = &entry.result;
let mean_str = format_duration_value(result.mean, Some(unit)).0;
let stddev_str = format_duration_value(result.stddev, Some(unit)).0;
let stddev_str = if let Some(stddev) = result.stddev {
format!(" ± {}", format_duration_value(stddev, Some(unit)).0)
} else {
"".into()
};
let min_str = format_duration_value(result.min, Some(unit)).0;
let max_str = format_duration_value(result.max, Some(unit)).0;
let rel_str = format!("{:.2}", entry.relative_speed);
let rel_stddev_str = if entry.is_fastest {
"".into()
} else {
format!(" ± {:.2}", entry.relative_speed_stddev)
if let Some(stddev) = entry.relative_speed_stddev {
format!(" ± {:.2}", stddev)
} else {
"".into()
}
};
dest.extend(
format!(
"| `{command}` | {mean} ± {stddev} | {min} | {max} | {rel}{rel_stddev} |\n",
"| `{command}` | {mean}{stddev} | {min} | {max} | {rel}{rel_stddev} |\n",
command = result.command.replace("|", "\\|"),
mean = mean_str,
stddev = stddev_str,
@@ -93,7 +101,7 @@ fn test_markdown_format_ms() {
BenchmarkResult::new(
String::from("sleep 0.1"),
0.1057, // mean
0.0016, // std dev
Some(0.0016), // std dev
0.1057, // median
0.0009, // user_mean
0.0011, // system_mean
@@ -106,7 +114,7 @@ fn test_markdown_format_ms() {
BenchmarkResult::new(
String::from("sleep 2"),
2.0050, // mean
0.0020, // std dev
Some(0.0020), // std dev
2.0050, // median
0.0009, // user_mean
0.0012, // system_mean
@@ -142,7 +150,7 @@ fn test_markdown_format_s() {
BenchmarkResult::new(
String::from("sleep 2"),
2.0050, // mean
0.0020, // std dev
Some(0.0020), // std dev
2.0050, // median
0.0009, // user_mean
0.0012, // system_mean
@@ -155,7 +163,7 @@ fn test_markdown_format_s() {
BenchmarkResult::new(
String::from("sleep 0.1"),
0.1057, // mean
0.0016, // std dev
Some(0.0016), // std dev
0.1057, // median
0.0009, // user_mean
0.0011, // system_mean
@@ -190,7 +198,7 @@ fn test_markdown_format_time_unit_s() {
BenchmarkResult::new(
String::from("sleep 0.1"),
0.1057, // mean
0.0016, // std dev
Some(0.0016), // std dev
0.1057, // median
0.0009, // user_mean
0.0011, // system_mean
@@ -203,7 +211,7 @@ fn test_markdown_format_time_unit_s() {
BenchmarkResult::new(
String::from("sleep 2"),
2.0050, // mean
0.0020, // std dev
Some(0.0020), // std dev
2.0050, // median
0.0009, // user_mean
0.0012, // system_mean
@@ -244,7 +252,7 @@ fn test_markdown_format_time_unit_ms() {
BenchmarkResult::new(
String::from("sleep 2"),
2.0050, // mean
0.0020, // std dev
Some(0.0020), // std dev
2.0050, // median
0.0009, // user_mean
0.0012, // system_mean
@@ -257,7 +265,7 @@ fn test_markdown_format_time_unit_ms() {
BenchmarkResult::new(
String::from("sleep 0.1"),
0.1057, // mean
0.0016, // std dev
Some(0.0016), // std dev
0.1057, // median
0.0009, // user_mean
0.0011, // system_mean
src/main.rs
+6 -2
View File
@@ -61,9 +61,13 @@ pub fn write_benchmark_comparison(results: &[BenchmarkResult]) {
for item in others {
println!(
"{} ± {} times faster than '{}'",
"{}{} times faster than '{}'",
format!("{:8.2}", item.relative_speed).bold().green(),
format!("{:.2}", item.relative_speed_stddev).green(),
if let Some(stddev) = item.relative_speed_stddev {
format!(" ± {}", format!("{:.2}", stddev).green())
} else {
"".into()
},
&item.result.command.magenta()
);
}
src/relative_speed.rs
+11 -6
View File
@@ -6,7 +6,7 @@ use crate::{benchmark_result::BenchmarkResult, units::Scalar};
pub struct BenchmarkResultWithRelativeSpeed<'a> {
pub result: &'a BenchmarkResult,
pub relative_speed: Scalar,
pub relative_speed_stddev: Scalar,
pub relative_speed_stddev: Option<Scalar>,
pub is_fastest: bool,
}
@@ -32,10 +32,15 @@ pub fn compute(results: &[BenchmarkResult]) -> Option<Vec<BenchmarkResultWithRel
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance asssumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = ratio
* ((result.stddev / result.mean).powi(2)
+ (fastest.stddev / fastest.mean).powi(2))
.sqrt();
let ratio_stddev = match (result.stddev, fastest.stddev) {
(Some(result_stddev), Some(fastest_stddev)) => Some(
ratio
* ((result_stddev / result.mean).powi(2)
+ (fastest_stddev / fastest.mean).powi(2))
.sqrt(),
),
_ => None,
};
BenchmarkResultWithRelativeSpeed {
result,
@@ -55,7 +60,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult {
BenchmarkResult {
command: name.into(),
mean,
stddev: 1.0,
stddev: Some(1.0),
median: mean,
user: mean,
system: 0.0,