diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index d565dd3..0136ff8 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -116,13 +116,17 @@ not every time as would happen with the \fB\-\-prepare\fR option. .HP \fB\-\-reference\fR \fICMD\fP .IP -Benchmark \fICMD\fP as the reference for the relative comparison of results. -If this option is not specified, the fastest command is used as the reference. +Use \fICMD\fP as the reference for the relative comparison of results. Without +parameters, benchmark \fICMD\fP as a separate reference command. If this option +is not specified, the fastest command is used as the reference. +With \fB\-\-parameter\-scan\fR or \fB\-\-parameter\-list\fR, select an existing +benchmark by its exact printed name. The name must match exactly one benchmark. .HP \fB\-\-reference\-name\fR \fINAME\fP .IP Give a meaningful \fINAME\fP to the reference command. Requires \fB\-\-reference\fR. +Cannot be used with parameterized benchmarks; use \fB\-\-command\-name\fR instead. .HP \fB\-p\fR, \fB\-\-prepare\fR \fICMD...\fP .IP diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 711ca2f..e62a591 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -16,6 +16,20 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> { pub relative_ordering: Ordering, } +impl BenchmarkResultWithRelativeSpeed<'_> { + pub fn reference_label(&self) -> &'static str { + if self.is_reference { + " (reference)" + } else { + match self.relative_ordering { + Ordering::Less => " (faster)", + Ordering::Equal => " (same speed)", + Ordering::Greater => " (slower)", + } + } + } +} + pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean_wall_clock_time() .partial_cmp(&r.mean_wall_clock_time()) @@ -37,10 +51,13 @@ fn compute_relative_speeds<'a>( let mut results: Vec<_> = results .iter() .map(|result| { - let is_reference = result == reference; + // Separate benchmarks can have identical names and measurements. + let is_reference = std::ptr::eq(result, reference); let relative_ordering = compare_mean_time(result, reference); - if result.mean_wall_clock_time() == Time::zero() { + if result.mean_wall_clock_time() == Time::zero() + || reference.mean_wall_clock_time() == Time::zero() + { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, @@ -64,11 +81,11 @@ fn compute_relative_speeds<'a>( result.measurements.stddev(), reference.measurements.stddev(), ) { - (Some(result_stddev), Some(fastest_stddev)) => Some( + (Some(result_stddev), Some(reference_stddev)) => Some( ratio * ((result_stddev / result.mean_wall_clock_time()) .powi(uom::typenum::P2::new()) - + (fastest_stddev / reference.mean_wall_clock_time()) + + (reference_stddev / reference.mean_wall_clock_time()) .powi(uom::typenum::P2::new())) .sqrt(), ), @@ -122,14 +139,14 @@ pub fn compute_with_check( Some(compute_relative_speeds(results, fastest, sort_order)) } -/// Same as compute_with_check, potentially resulting in relative speeds of infinity -pub fn compute( - results: &[BenchmarkResult], +/// Compute relative speeds against the given reference, or the fastest result. +pub fn compute<'a>( + results: &'a [BenchmarkResult], sort_order: SortOrder, -) -> Vec> { - let fastest = fastest_of(results); - - compute_relative_speeds(results, fastest, sort_order) + reference: Option<&'a BenchmarkResult>, +) -> Vec> { + let reference = reference.unwrap_or_else(|| fastest_of(results)); + compute_relative_speeds(results, reference, sort_order) } #[cfg(test)] @@ -192,3 +209,28 @@ fn test_compute_relative_speed_for_zero_times() { assert!(annotated_results.is_none()); } + +#[test] +fn reference_identity_distinguishes_equal_results() { + let results = vec![create_result("same", 1.0), create_result("same", 1.0)]; + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + + assert!(entries[0].is_reference); + assert!(!entries[1].is_reference); +} + +#[test] +fn reference_ratios_handle_zero_times() { + let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; + for result in &mut results { + let measurement = result.measurements.measurements[0]; + result.measurements.measurements.push(measurement); + } + let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); + assert_eq!(entries[1].relative_speed, f64::INFINITY); + + let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); + assert_eq!(entries[0].relative_speed, f64::INFINITY); + assert_eq!(entries[0].relative_speed_stddev, None); + assert_eq!(entries[1].relative_speed, 1.0); +} diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index ce5fd81..db036d6 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -54,7 +54,8 @@ impl<'a> Scheduler<'a> { // We export results after each individual benchmark, because // we would risk losing them if a later benchmark fails. - self.export_manager.write_results(&self.results, true)?; + self.export_manager + .write_results(&self.results, true, self.options.reference_index)?; } Ok(()) @@ -71,9 +72,8 @@ impl<'a> Scheduler<'a> { let reference = self .options - .reference_command - .as_ref() - .map(|_| &self.results[0]) + .reference_index + .map(|index| &self.results[index]) .unwrap_or_else(|| relative_speed::fastest_of(&self.results)); if let Some(annotated_results) = relative_speed::compute_with_check_from_reference( @@ -130,9 +130,14 @@ impl<'a> Scheduler<'a> { console_writeln!(stdout, "{}", "Relative speed comparison".bold())?; for item in annotated_results { + let relationship = if self.options.reference_index.is_some() { + item.reference_label() + } else { + "" + }; console_writeln!( stdout, - " {}{} {}", + " {}{} {}{}", format!("{:10.2}", item.relative_speed).bold().green(), if item.is_reference { " ".into() @@ -142,6 +147,7 @@ impl<'a> Scheduler<'a> { " ".into() }, item.result.command_with_unused_parameters, + relationship, )?; } } @@ -163,7 +169,8 @@ impl<'a> Scheduler<'a> { } pub fn final_export(&self) -> Result<()> { - self.export_manager.write_results(&self.results, false) + self.export_manager + .write_results(&self.results, false, self.options.reference_index) } } diff --git a/src/cli.rs b/src/cli.rs index 39a2238..0a38675 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -105,8 +105,11 @@ fn build_command() -> Command { .action(ArgAction::Set) .value_name("CMD") .help( - "The reference command for the relative comparison of results. \ - If this is unset, results are compared with the fastest command as reference." + "The reference for the relative comparison of results. Without parameters, \ + CMD is run as a separate reference command. With --parameter-scan or \ + --parameter-list, CMD must exactly match one unique benchmark name as \ + shown in the output. If this is unset, results are compared with the \ + fastest command as reference." ) ) .arg( @@ -114,7 +117,8 @@ fn build_command() -> Command { .long("reference-name") .action(ArgAction::Set) .value_name("CMD") - .help("Give a meaningful name to the reference command.") + .help("Give a meaningful name to the reference command. This cannot be used \ + with parameterized benchmarks; use --command-name instead.") .requires("reference") ) .arg( diff --git a/src/export/csv.rs b/src/export/csv.rs index 3932f92..15cc3a5 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -18,6 +18,7 @@ impl Exporter for CsvExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { const CSV_UNIT: TimeUnit = TimeUnit::Second; const CSV_PRECISION: usize = 6; @@ -172,7 +173,7 @@ fn test_csv() { let actual = String::from_utf8( exporter - .serialize(&results, Some(TimeUnit::Second), SortOrder::Command) + .serialize(&results, Some(TimeUnit::Second), SortOrder::Command, None) .unwrap(), ) .unwrap(); diff --git a/src/export/json.rs b/src/export/json.rs index 6c9b21d..c5b6a84 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -22,6 +22,7 @@ impl Exporter for JsonExporter { results: &[BenchmarkResult], _unit: Option, _sort_order: SortOrder, + _reference_index: Option, ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); if let Ok(ref mut content) = output { diff --git a/src/export/markup.rs b/src/export/markup.rs index f3f8711..0eea533 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -1,10 +1,11 @@ use crate::benchmark::relative_speed::BenchmarkResultWithRelativeSpeed; use crate::benchmark::{benchmark_result::BenchmarkResult, relative_speed}; use crate::options::SortOrder; -use crate::quantity::{FormatQuantity, IsUnit, TimeUnit}; +use crate::quantity::{FormatQuantity, IsUnit, Time, TimeUnit, Zero}; use super::Exporter; use anyhow::Result; +use std::cmp::Ordering; pub enum Alignment { Left, @@ -16,6 +17,8 @@ pub trait MarkupExporter { &self, entries: &[BenchmarkResultWithRelativeSpeed], unit: TimeUnit, + reference_pending: bool, + explicit_reference: bool, ) -> String { // prepare table header strings let notation = format!("[{}]", unit.short_name()); @@ -58,13 +61,25 @@ pub trait MarkupExporter { }; let min_str = measurement.measurements.min().format_value(unit); let max_str = measurement.measurements.max().format_value(unit); - let rel_str = format!("{:.2}", entry.relative_speed); - let rel_stddev_str = if entry.is_reference { - "".into() - } else if let Some(stddev) = entry.relative_speed_stddev { - format!(" ± {stddev:.2}") + // The ratio of two zero times is undefined, even if they compare equal. + let relative_unavailable = reference_pending + || (explicit_reference + && !entry.is_reference + && entry.relative_ordering == Ordering::Equal + && measurement.mean_wall_clock_time() == Time::zero()); + let relative = if relative_unavailable { + "N/A".to_string() } else { - "".into() + let stddev = match (entry.is_reference, entry.relative_speed_stddev) { + (false, Some(stddev)) => format!(" ± {stddev:.2}"), + _ => String::new(), + }; + let label = if explicit_reference { + entry.reference_label() + } else { + "" + }; + format!("{:.2}{stddev}{label}", entry.relative_speed) }; // prepare table row entries @@ -73,7 +88,7 @@ pub trait MarkupExporter { &format!("{mean_str}{stddev_str}"), &min_str, &max_str, - &format!("{rel_str}{rel_stddev_str}"), + &relative, ])) } @@ -114,11 +129,20 @@ impl Exporter for T { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result> { let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); - let entries = relative_speed::compute(results, sort_order); + // Do not report ratios against another benchmark while the selected + // reference is still pending in an intermediate export. + let reference_pending = reference_index.is_some_and(|i| i >= results.len()); + let entries = relative_speed::compute( + results, + sort_order, + reference_index.and_then(|i| results.get(i)), + ); - let table = self.table_results(&entries, unit); + let table = + self.table_results(&entries, unit, reference_pending, reference_index.is_some()); Ok(table.as_bytes().to_vec()) } } diff --git a/src/export/mod.rs b/src/export/mod.rs index c8d51d7..c8dd9a0 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -51,6 +51,7 @@ trait Exporter { results: &[BenchmarkResult], unit: Option, sort_order: SortOrder, + reference_index: Option, ) -> Result>; } @@ -130,11 +131,18 @@ impl ExportManager { /// results are written to all file targets (to always have them up to date, even /// if a benchmark fails). In the latter case, we only print to stdout targets (in /// order not to clutter the output of hyperfine with intermediate results). - pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { + /// `reference_index` refers to the complete benchmark sequence, so it can + /// point past `results` while that benchmark is still pending. + pub fn write_results( + &self, + results: &[BenchmarkResult], + intermediate: bool, + reference_index: Option, + ) -> Result<()> { for e in &self.exporters { let content = || { e.exporter - .serialize(results, self.time_unit, self.sort_order) + .serialize(results, self.time_unit, self.sort_order, reference_index) }; match e.target { diff --git a/src/export/tests.rs b/src/export/tests.rs index 9566f9d..abe340a 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -14,7 +14,7 @@ fn get_output( sort_order: SortOrder, ) -> String { let exporter = E::default(); - String::from_utf8(exporter.serialize(results, unit, sort_order).unwrap()).unwrap() + String::from_utf8(exporter.serialize(results, unit, sort_order, None).unwrap()).unwrap() } /// Ensure the makrup output includes the table header and the multiple diff --git a/src/options.rs b/src/options.rs index 0b902d5..f5a3285 100644 --- a/src/options.rs +++ b/src/options.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; use std::{cmp, env, fmt, io}; -use anyhow::ensure; +use anyhow::{bail, ensure}; use clap::ArgMatches; use crate::command::Commands; @@ -208,12 +208,15 @@ pub struct Options { /// Whether or not to ignore non-zero exit codes pub command_failure_action: CmdFailureAction, - // Command to use as a reference for relative speed comparison + // Standalone reference command, cleared when an existing benchmark is selected. pub reference_command: Option, // Name of the reference command pub reference_name: Option, + // Index of the reference in the benchmark sequence, if any. + pub reference_index: Option, + /// Command(s) to run before each timing run pub preparation_command: Option>, @@ -257,6 +260,7 @@ impl Default for Options { command_failure_action: CmdFailureAction::RaiseError, reference_command: None, reference_name: None, + reference_index: None, preparation_command: None, conclusion_command: None, setup_command: None, @@ -464,6 +468,35 @@ impl Options { } pub fn validate_against_command_list(&mut self, commands: &Commands) -> Result<()> { + if let Some(reference) = &self.reference_command { + if commands + .iter() + .next() + .is_some_and(|command| !command.get_parameters().is_empty()) + { + ensure!( + self.reference_name.is_none(), + "--reference-name cannot be used with a parameterized reference; use --command-name to name the benchmark" + ); + let mut matches = commands + .iter() + .enumerate() + .filter(|(_, command)| command.get_name_with_unused_parameters() == *reference); + let Some((index, _)) = matches.next() else { + bail!( + "Reference '{reference}' does not match any parameterized benchmark. Use its full displayed benchmark name." + ); + }; + ensure!( + matches.next().is_none(), + "Reference '{reference}' matches multiple parameterized benchmarks. Use --command-name to give them unique names." + ); + self.reference_index = Some(index); + self.reference_command = None; + } else { + self.reference_index = Some(0); + } + } let has_reference_command = self.reference_command.is_some(); let num_commands = commands.num_commands(has_reference_command); diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5..e09eb63 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -383,26 +383,6 @@ fn reference_is_executed_first() { .run(); } -#[test] -fn reference_is_executed_first_parameter_value() { - ExecutionOrderTest::new() - .arg("--runs=2") - .reference("reference") - .arg("--parameter-list") - .arg("number") - .arg("1,2,3") - .command("command {number}") - .expect_output("reference") - .expect_output("reference") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") - .run(); -} - #[test] fn reference_is_executed_separately_from_commands() { ExecutionOrderTest::new() diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e6832b2..42c23ab 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -705,6 +705,68 @@ fn shows_benchmark_comparison_relative_to_reference() { "); } +#[test] +fn command_sorted_comparison_and_markup_identify_reference() { + let _settings = snapshot_settings().bind_to_scope(); + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.md"); + let output = hyperfine_debug() + .args([ + "--style=basic", + "--runs=1", + "--sort=command", + "--reference", + "sleep 2", + "--reference-name", + "baseline", + "--export-markdown", + ]) + .arg(&export_path) + .args(["sleep 1", "sleep 2", "sleep 3"]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + let comparison = stdout.split_once("Relative speed comparison\n").unwrap().1; + insta::assert_snapshot!(comparison, @r" + 1.00 baseline (reference) + 2.00 sleep 1 (faster) + 1.00 sleep 2 (same speed) + 1.50 sleep 3 (slower) + "); + let markdown = std::fs::read_to_string(export_path).unwrap(); + insta::assert_snapshot!(markdown, @r" + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `baseline` | 2.000 | 2.000 | 2.000 | 1.00 (reference) | + | `sleep 1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) | + | `sleep 2` | 2.000 | 2.000 | 2.000 | 1.00 (same speed) | + | `sleep 3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) | + "); + + // Equal zero times have no meaningful relative factor. + let output = hyperfine_debug() + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "--reference", + "sleep 0", + "--reference-name", + "baseline", + "sleep 0", + ]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let markdown = String::from_utf8(output.stdout).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); +} + #[test] fn shows_reference_name() { let _settings = snapshot_settings().bind_to_scope(); @@ -795,48 +857,180 @@ fn rejects_negative_parameter_steps() { } } +#[cfg(unix)] #[test] -fn performs_reference_and_all_benchmarks_in_parameter_scan() { +fn selects_reference_with_parameterized_prepare() { + let directory = tempfile::tempdir().unwrap(); + let csv_path = directory.path().join("results.csv"); + let json_path = directory.path().join("results.json"); + let output = hyperfine_raw_command() + .args([ + "--style=basic", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.2,0.4,0.6", + "--prepare", + "sleep {delay}", + "--reference", + "echo a (delay = 0.4)", + "--export-csv", + ]) + .arg(&csv_path) + .arg("--export-json") + .arg(&json_path) + .arg("echo a") + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Benchmark ").count(), 3); + for (number, delay) in ["0.2", "0.4", "0.6"].iter().enumerate() { + assert!( + stdout.contains(&format!( + "Benchmark {}: echo a (delay = {})", + number + 1, + delay + )), + "{}", + stdout + ); + } + assert!(stdout.contains("echo a (delay = 0.4) ran"), "{}", stdout); + + let mut csv = csv::Reader::from_path(csv_path).unwrap(); + assert!(csv + .headers() + .unwrap() + .iter() + .any(|header| header == "parameter_delay")); + assert_eq!( + csv.records().collect::, _>>().unwrap().len(), + 3 + ); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(json_path).unwrap()).unwrap(); + let results = json["results"].as_array().unwrap(); + assert_eq!(results.len(), 3); + assert_eq!( + results + .iter() + .map(|result| result["parameters"]["delay"].as_str().unwrap()) + .collect::>(), + ["0.2", "0.4", "0.6"] + ); +} + +#[test] +fn rejects_unmatched_reference_before_export_or_setup() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.csv"); + let marker_path = directory.path().join("setup-ran"); + std::fs::write(&export_path, "previous contents").unwrap(); + + hyperfine() + .args(["--reference", "sleep 1", "-P", "secs", "2", "3", "--setup"]) + .arg(format!("echo touched > \"{}\"", marker_path.display())) + .arg("--export-csv") + .arg(&export_path) + .arg("sleep {secs}") + .assert() + .failure() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains( + "does not match any parameterized benchmark", + )); + assert_eq!( + std::fs::read_to_string(export_path).unwrap(), + "previous contents" + ); + assert!(!marker_path.exists()); +} + +#[test] +fn rejects_ambiguous_parameterized_reference() { + hyperfine_debug() + .args([ + "-L", + "x", + "1,2", + "--command-name", + "case", + "--reference", + "case", + "sleep {x}", + ]) + .assert() + .failure() + .stderr(predicate::str::contains( + "matches multiple parameterized benchmarks", + )); +} + +#[cfg(unix)] +#[test] +fn intermediate_markdown_waits_for_selected_reference() { + let directory = tempfile::tempdir().unwrap(); + let export_path = directory.path().join("results.md"); + hyperfine() + .args([ + "--style=none", + "--shell=none", + "--runs=1", + "-L", + "delay", + "0.01,0.02,0.03", + "--reference", + "sleep 0.02", + "--prepare", + "true", + "--prepare", + "false", + "--prepare", + "true", + "--export-markdown", + ]) + .arg(&export_path) + .arg("sleep {delay}") + .assert() + .failure(); + let markdown = std::fs::read_to_string(export_path).unwrap(); + let row = markdown + .lines() + .find(|line| line.contains("sleep 0.01")) + .unwrap(); + assert!(row.ends_with("| N/A |"), "{}", row); +} + +#[test] +fn markdown_export_uses_selected_parameterized_reference() { let _settings = snapshot_settings().bind_to_scope(); assert_cmd_snapshot!(hyperfine_debug() - .arg("--style=basic") - .arg("--reference=sleep 25") - .arg("--parameter-scan") - .arg("time") - .arg("30") - .arg("45") - .arg("--parameter-step-size") - .arg("5") - .arg("sleep {time}"), @r" + .args([ + "--style=none", + "--runs=1", + "--export-markdown=-", + "-P", + "x", + "1", + "3", + "--command-name", + "case-{x}", + "--reference", + "case-2", + "sleep {x}", + ]), @r" success: true exit_code: 0 ----- stdout ----- - Benchmark 1: sleep 25 - Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 25.000 s … 25.000 s 10 runs - Benchmark 2: sleep 30 - Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 30.000 s … 30.000 s 10 runs + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `case-1` | 1.000 | 1.000 | 1.000 | 2.00 (faster) | + | `case-2` | 2.000 | 2.000 | 2.000 | 1.00 (reference) | + | `case-3` | 3.000 | 3.000 | 3.000 | 1.50 (slower) | - Benchmark 3: sleep 35 - Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 35.000 s … 35.000 s 10 runs - - Benchmark 4: sleep 40 - Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 40.000 s … 40.000 s 10 runs - - Benchmark 5: sleep 45 - Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] - Range (min … max): 45.000 s … 45.000 s 10 runs - - Summary - sleep 25 ran - 1.20 ± 0.00 times faster than sleep 30 - 1.40 ± 0.00 times faster than sleep 35 - 1.60 ± 0.00 times faster than sleep 40 - 1.80 ± 0.00 times faster than sleep 45 ----- stderr ----- ");