From 55dfe6722cb940c9a314e3f301e2d11b413c4622 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 6 Oct 2026 10:52:49 +0200 Subject: [PATCH] Better defaults --- CHANGELOG.md | 4 +- README.md | 19 +++-- doc/hyperfine.1 | 21 +++--- src/cli.rs | 5 +- src/export/tests.rs | 50 ++++++------- src/metric.rs | 2 +- src/options.rs | 5 ++ src/output/report.rs | 19 +++-- tests/integration_tests.rs | 149 ++++++++++++++++++------------------- 9 files changed, 144 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38912cd..375894f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Breaking changes -- Replace `--time-unit`/`-u` with `--metrics METRIC[:UNIT],…`. Select multiple metrics for terminal output; the first metric is primary for all non-JSON exports. Units are chosen automatically for terminal and markup output when omitted. Counters support decimal `k`, `M`, and `B` suffixes. +- Replace `--time-unit`/`-u` with `--metrics METRIC[:UNIT],…`. Select multiple metrics for terminal output; the first metric is primary for all non-JSON exports. Presets include `default`, `speed` (wall-clock, user, and system time), and `all` (all available metrics). Units are chosen automatically for terminal and markup output when omitted. Counters support decimal `k`, `M`, and `B` suffixes. - Remove `--sort`, `--reference`, and `--reference-name`. Commands and exports always remain in input order, and the first expanded benchmark is the reference for every metric. -- Replace time-only terminal output and the final speed ranking with a per-command metric table. Show wall-clock time and peak resident memory by default (wall-clock time only on Windows), with percentage changes from the first command for subsequent benchmarks. +- Replace time-only terminal output and the final speed ranking with a per-command metric table. Show wall-clock time and peak resident memory by default (wall-clock time only on Windows), with percentage changes from the first command for subsequent benchmarks. Color all metric rows and make the entire primary row bold. - Export only the primary metric to CSV, Markdown, AsciiDoc, and org-mode. CSV uses an explicitly selected unit or the metric's base unit when omitted. JSON continues to contain all collected metrics in base units and identifies the primary metric. - Benchmarked commands are now always executed directly, without an intermediate shell (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell `. - The JSON format for `--export-json` has changed (schema version 2). The new format now includes metadata, per-run measurements, and statistical summaries for all measured quantities, see #790 (@sharkdp). Code that reads those exported JSON files must be updated. For example, `results[i].mean` is now `results[i].summary.time_wall_clock.mean`, and `results[i].times` is replaced by `results[i].measurements[j].time_wall_clock.value`. The new structure looks like this: diff --git a/README.md b/README.md index 54422a5..734df42 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,11 @@ hyperfine \ './baseline' './candidate' ``` -Use `--metrics=default` for the platform defaults or `--metrics=all` for all available -metrics in the order listed below, with wall-clock time primary. Both presets choose display -units automatically and must be used alone, without other metrics or unit suffixes. +Use `--metrics=default` for the platform defaults, `--metrics=speed` for +`time_wall_clock,time_user,time_system`, or `--metrics=all` for all available metrics in the +order listed below. The `speed` preset works on all platforms, including with a shell. +All presets keep wall-clock time primary, choose display units automatically, and must be +used alone, without other metrics or unit suffixes. The `all` preset skips metrics unsupported by the platform or shell, or missing from any run of a benchmark. If a metric is unavailable for the reference, its change displays as `N/A`. @@ -69,12 +71,13 @@ standard deviation, minimum, and maximum. The first benchmark has no change colu For subsequent commands, a change of `+50.0%` means the mean is 50% greater than the reference; `-20.0%` means it is 20% smaller. These percentages describe the observed means, not statistical significance. Decreases are green and increases -are red when colors are enabled; signs remain visible without color. A percentage change from -a zero reference is undefined and displays as `N/A`. +are red when colors are enabled; signs remain visible without color. +A percentage change from a zero reference is undefined and displays as `N/A`. -The **first metric is primary**: it is emphasized in terminal output and is the only metric -in non-JSON exports. All comparisons use the first command as their reference, regardless -of which metric is primary. There is no sorting or separate reference selection. +The **first metric is primary**: its entire terminal row is bold, and it is the only metric +in non-JSON exports. All metric rows use colors when enabled. All comparisons use the first +command as their reference, regardless of which metric is primary. There is no sorting or +separate reference selection. | Metrics | Explicit units | |:---|:---| diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index e0fbc18..8f743fc 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -35,7 +35,7 @@ hyperfine \- command\-line benchmarking tool .RB [ \-\-style .IR TYPE ] .RB [ \-\-metrics -.IR METRIC[:UNIT],...|default|all ] +.IR METRIC[:UNIT],...|default|speed|all ] .RB [ \-\-export\-asciidoc .IR FILE ] .RB [ \-\-export\-csv @@ -238,20 +238,23 @@ interactive output without any colors. Set this to 'color' to keep the colors without any interactive output. Set this to 'none' to disable all the output of the tool. .HP -\fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|all\fP +\fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|speed|all\fP .IP Select the metrics to display, in order, with an optional unit for each. Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR on Windows, where peak resident memory is not yet supported. -The first metric is primary: it is emphasized in terminal output and is the only -metric in non\-JSON exports. +The first metric is primary: its entire terminal row is bold, and it is the +only metric in non\-JSON exports. All metric rows use colors when enabled. The first benchmark has no change column; subsequent benchmarks show percentage -changes from the first benchmark. +changes from the first benchmark. Decreases are green and increases are red +when colors are enabled. .IP -Use \fBdefault\fR for the platform defaults or \fBall\fR for all available -metrics in the order listed below, with wall\-clock time primary. -Both presets select display units automatically and must be used alone, -without other metrics or unit suffixes. +Use \fBdefault\fR for the platform defaults, \fBspeed\fR for +\fBtime_wall_clock,time_user,time_system\fR, or \fBall\fR for all available +metrics in the order listed below. The \fBspeed\fR preset works on all +platforms, including with a shell. +All presets keep wall\-clock time primary, select display units automatically, +and must be used alone, without other metrics or unit suffixes. The \fBall\fR preset skips metrics unsupported by the platform or shell, or missing from any run of a benchmark. If a metric is unavailable for the reference, its change displays as N/A. diff --git a/src/cli.rs b/src/cli.rs index edc33b8..cab618c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -57,8 +57,9 @@ fn build_command() -> Command { .default_value("default") .help("Metrics to display, in order. The first is used for non-JSON exports. \ Compare all commands with the first command. Presets: default (wall-clock \ - time and peak RSS; wall-clock time only on Windows), all (all available \ - metrics, skipping unavailable measurements). Both presets use automatic \ + time and peak RSS; wall-clock time only on Windows), speed (wall-clock, user, \ + and system time), all (all available metrics, skipping unavailable \ + measurements). All presets use automatic \ units with wall-clock time first; use a preset on its own. Metrics: time_wall_clock, \ time_user, time_system, memory_peak_resident, cpu_cycles, instructions, \ cache_references, cache_misses, branch_misses. Optional units: ns/us/ms/s/min/h \ diff --git a/src/export/tests.rs b/src/export/tests.rs index 2404453..8ecbe63 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -100,18 +100,18 @@ fn test_markup_export_auto_ms() { }, ]; - insta::assert_snapshot!(get_output::(&results, None), @r#" - | Command | Mean Time [ms] | Min [ms] | Max [ms] | Change | + insta::assert_snapshot!(get_output::(&results, None), @" + | Command | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change | |:---|---:|---:|---:|---:| | `sleep 0.1` | 110.0 ± 26.5 | 90.0 | 140.0 | reference | | `sleep 2` | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | +2627.3% | - "#); + "); insta::assert_snapshot!(get_output::(&results, None), @r#" [cols="<,>,>,>,>"] |=== | Command - | Mean Time [ms] + | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change @@ -130,12 +130,12 @@ fn test_markup_export_auto_ms() { |=== "#); - insta::assert_snapshot!(get_output::(&results, None), @r#" - | Command | Mean Time [ms] | Min [ms] | Max [ms] | Change | + insta::assert_snapshot!(get_output::(&results, None), @" + | Command | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change | |--+--+--+--+--| | =sleep 0.1= | 110.0 ± 26.5 | 90.0 | 140.0 | reference | | =sleep 2= | 3000.0 ± 1000.0 | 2000.0 | 4000.0 | +2627.3% | - "#); + "); } /// This (again) demonstrates that the first entry's units (s) are used to set @@ -209,18 +209,18 @@ fn test_markup_export_auto_s() { }, ]; - insta::assert_snapshot!(get_output::(&results, None), @r#" - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + insta::assert_snapshot!(get_output::(&results, None), @" + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `sleep 2` | 2.200 ± 0.100 | 2.100 | 2.300 | reference | | `sleep 0.1` | 0.200 ± 0.100 | 0.100 | 0.300 | -90.9% | - "#); + "); insta::assert_snapshot!(get_output::(&results, None), @r#" [cols="<,>,>,>,>"] |=== | Command - | Mean Time [s] + | Mean Wall Time [s] | Min [s] | Max [s] | Change @@ -239,12 +239,12 @@ fn test_markup_export_auto_s() { |=== "#); - insta::assert_snapshot!(get_output::(&results, None), @r#" - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + insta::assert_snapshot!(get_output::(&results, None), @" + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |--+--+--+--+--| | =sleep 2= | 2.200 ± 0.100 | 2.100 | 2.300 | reference | | =sleep 0.1= | 0.200 ± 0.100 | 0.100 | 0.300 | -90.9% | - "#); + "); } /// This (again) demonstrates that the given time unit (ms) is used to set @@ -318,18 +318,18 @@ fn test_markup_export_manual_ms() { }, ]; - insta::assert_snapshot!(get_output::(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @r#" - | Command | Mean Time [ms] | Min [ms] | Max [ms] | Change | + insta::assert_snapshot!(get_output::(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @" + | Command | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change | |:---|---:|---:|---:|---:| | `sleep 2` | 2200.0 ± 100.0 | 2100.0 | 2300.0 | reference | | `sleep 0.1` | 200.0 ± 100.0 | 100.0 | 300.0 | -90.9% | - "#); + "); insta::assert_snapshot!(get_output::(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @r#" [cols="<,>,>,>,>"] |=== | Command - | Mean Time [ms] + | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change @@ -348,12 +348,12 @@ fn test_markup_export_manual_ms() { |=== "#); - insta::assert_snapshot!(get_output::(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @r#" - | Command | Mean Time [ms] | Min [ms] | Max [ms] | Change | + insta::assert_snapshot!(get_output::(&timing_results, Some(Unit { symbol: "ms", scale: 0.001 })), @" + | Command | Mean Wall Time [ms] | Min [ms] | Max [ms] | Change | |--+--+--+--+--| | =sleep 2= | 2200.0 ± 100.0 | 2100.0 | 2300.0 | reference | | =sleep 0.1= | 200.0 ± 100.0 | 100.0 | 300.0 | -90.9% | - "#); + "); } /// The given time unit (s) is used to set the units for all entries. @@ -426,18 +426,18 @@ fn test_markup_export_manual_s() { }, ]; - insta::assert_snapshot!(get_output::(&results, Some(Unit { symbol: "s", scale: 1.0 })), @r#" - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + insta::assert_snapshot!(get_output::(&results, Some(Unit { symbol: "s", scale: 1.0 })), @" + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `sleep 2` | 2.020 ± 0.010 | 2.010 | 2.030 | reference | | `sleep 0.1` | 0.120 ± 0.010 | 0.110 | 0.130 | -94.1% | - "#); + "); insta::assert_snapshot!(get_output::(&results, Some(Unit { symbol: "s", scale: 1.0 })), @r#" [cols="<,>,>,>,>"] |=== | Command - | Mean Time [s] + | Mean Wall Time [s] | Min [s] | Max [s] | Change diff --git a/src/metric.rs b/src/metric.rs index 26dd8aa..da3547d 100644 --- a/src/metric.rs +++ b/src/metric.rs @@ -48,7 +48,7 @@ impl Metric { pub fn label(self) -> &'static str { match self { - Self::TimeWallClock => "Time", + Self::TimeWallClock => "Wall Time", Self::TimeUser => "User time", Self::TimeSystem => "System time", Self::MemoryPeakResident => "Memory", diff --git a/src/options.rs b/src/options.rs index 428d420..e24ed3b 100644 --- a/src/options.rs +++ b/src/options.rs @@ -397,6 +397,11 @@ impl Options { match matches.get_one::("metrics").unwrap().as_str() { "default" => {} + "speed" => { + options.metrics = [Metric::TimeWallClock, Metric::TimeUser, Metric::TimeSystem] + .map(|metric| MetricSelection { metric, unit: None }) + .to_vec(); + } "all" => { let shell = matches!( options.executor_kind, diff --git a/src/output/report.rs b/src/output/report.rs index 9788f1b..975cee5 100644 --- a/src/output/report.rs +++ b/src/output/report.rs @@ -157,13 +157,7 @@ pub fn print( } else { text.normal().to_string() } - } else if column == 1 { - if index == 1 { - text.green().bold().to_string() - } else { - text.green().to_string() - } - } else if column == 2 { + } else if column == 1 || column == 2 { text.green().to_string() } else if column == 3 { text.cyan().to_string() @@ -184,7 +178,16 @@ pub fn print( line.push_str(&" ".repeat(unit_padding)); } } - console_writeln!(stdout, " {}", line.trim_end())?; + let line = line.trim_end(); + console_writeln!( + stdout, + " {}", + if index == 1 { + line.bold() + } else { + line.normal() + } + )?; } Ok(()) } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c12e0e6..26cf9fc 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -45,34 +45,33 @@ fn runs_successfully() { .unwrap(); assert!(output.status.success(), "{:?}", output); let stdout = String::from_utf8(output.stdout).unwrap(); - assert!(stdout.contains("Time")); + assert!(stdout.contains("Wall Time")); + assert!(!stdout.contains("User time")); + assert!(!stdout.contains("System time")); assert_eq!(stdout.contains("Memory"), !cfg!(windows)); assert!(!stdout.contains("Change")); assert!(!stdout.contains("Outliers")); } #[test] -fn all_metrics_skip_unavailable_counters() { - let output = hyperfine_debug() - .args([ - "--metrics=all", - "--style=basic", - "--runs=2", - "sleep 1", - "sleep 2", - ]) - .output() - .unwrap(); - assert!(output.status.success(), "{:?}", output); - let stdout = String::from_utf8(output.stdout).unwrap(); - assert_eq!(stdout.matches("Time").count(), 2); - assert_eq!(stdout.matches("User time").count(), 2); - assert_eq!(stdout.matches("System time").count(), 2); - assert_eq!(stdout.contains("Memory"), !cfg!(windows)); - assert!(!stdout.contains("CPU cycles")); - assert!(!stdout.contains("Instructions")); - assert!(!stdout.contains("Change vs #1")); - assert!(stdout.contains("+100.0%")); +fn metric_presets_select_available_metrics() { + for preset in ["speed", "all"] { + let output = hyperfine_debug() + .arg(format!("--metrics={preset}")) + .args(["--style=basic", "--runs=2", "sleep 1", "sleep 2"]) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.matches("Wall Time").count(), 2); + assert_eq!(stdout.matches("User time").count(), 2); + assert_eq!(stdout.matches("System time").count(), 2); + assert_eq!(stdout.contains("Memory"), preset == "all" && !cfg!(windows)); + assert!(!stdout.contains("CPU cycles")); + assert!(!stdout.contains("Instructions")); + assert!(!stdout.contains("Change vs #1")); + assert!(stdout.contains("+100.0%")); + } // Explicit selections still require the metric, unlike the `all` preset. hyperfine_debug() @@ -549,27 +548,27 @@ fn returns_mean_time_in_correct_unit() { .arg("sleep 1.234") .assert() .success() - .stdout(predicate::str::is_match(r"Time\s+1\.234 s\s+±").unwrap()); + .stdout(predicate::str::is_match(r"Wall Time\s+1\.234 s\s+±").unwrap()); hyperfine_debug() .arg("sleep 0.123") .assert() .success() - .stdout(predicate::str::is_match(r"Time\s+123\.0 ms\s+±").unwrap()); + .stdout(predicate::str::is_match(r"Wall Time\s+123\.0 ms\s+±").unwrap()); hyperfine_debug() .arg("--metrics=time_wall_clock:ms") .arg("sleep 1.234") .assert() .success() - .stdout(predicate::str::is_match(r"Time\s+1234\.0 ms\s+±").unwrap()); + .stdout(predicate::str::is_match(r"Wall Time\s+1234\.0 ms\s+±").unwrap()); hyperfine_debug() .arg("--metrics=time_wall_clock:us") .arg("sleep 1.234") .assert() .success() - .stdout(predicate::str::is_match(r"Time\s+1234000\.0 µs\s+±").unwrap()); + .stdout(predicate::str::is_match(r"Wall Time\s+1234000\.0 µs\s+±").unwrap()); } #[test] @@ -676,19 +675,19 @@ fn shows_multiple_metrics_relative_to_first_command() { ----- stdout ----- Benchmark 1: sleep 0.0817 (36 runs) mean ± σ min … max - Time 81.7 ms ± 0.0 ms 81.7 ms … 81.7 ms + Wall Time 81.7 ms ± 0.0 ms 81.7 ms … 81.7 ms User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB Benchmark 2: sleep 0.6155 (10 runs) mean ± σ min … max - Time 615.5 ms ± 0.0 ms 615.5 ms … 615.5 ms +653.4% + Wall Time 615.5 ms ± 0.0 ms 615.5 ms … 615.5 ms +653.4% User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs N/A Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB N/A Benchmark 3: sleep 0.1707 (17 runs) mean ± σ min … max - Time 170.7 ms ± 0.0 ms 170.7 ms … 170.7 ms +108.9% + Wall Time 170.7 ms ± 0.0 ms 170.7 ms … 170.7 ms +108.9% User time 0.0 µs ± 0.0 µs 0.0 µs … 0.0 µs N/A Memory 0.0 MiB ± 0.0 MiB 0.0 MiB … 0.0 MiB N/A @@ -713,24 +712,24 @@ fn shows_benchmark_comparison_with_same_time() { exit_code: 0 ----- stdout ----- Benchmark 1: A (10 runs) - mean ± σ min … max - Time 1.000 s ± 0.000 s 1.000 s … 1.000 s - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B + mean ± σ min … max + Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B Benchmark 2: B (10 runs) - mean ± σ min … max - Time 1.000 s ± 0.000 s 1.000 s … 1.000 s 0.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s 0.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 3: sleep 2.0 (10 runs) - mean ± σ min … max - Time 2.000 s ± 0.000 s 2.000 s … 2.000 s +100.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 2.000 s ± 0.000 s 2.000 s … 2.000 s +100.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 4: sleep 1000.0 (10 runs) - mean ± σ min … max - Time 1000.000 s ± 0.000 s 1000.000 s … 1000.000 s +99900.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 1000.000 s ± 0.000 s 1000.000 s … 1000.000 s +99900.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A ----- stderr ----- @@ -750,19 +749,19 @@ fn shows_benchmark_comparison_relative_to_reference() { exit_code: 0 ----- stdout ----- Benchmark 1: sleep 2.0 (10 runs) - mean ± σ min … max - Time 2.000 s ± 0.000 s 2.000 s … 2.000 s - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B + mean ± σ min … max + Wall Time 2.000 s ± 0.000 s 2.000 s … 2.000 s + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B Benchmark 2: sleep 1.0 (10 runs) - mean ± σ min … max - Time 1.000 s ± 0.000 s 1.000 s … 1.000 s -50.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s -50.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 3: sleep 3.0 (10 runs) - mean ± σ min … max - Time 3.000 s ± 0.000 s 3.000 s … 3.000 s +50.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 3.000 s ± 0.000 s 3.000 s … 3.000 s +50.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A ----- stderr ----- @@ -797,7 +796,7 @@ fn comparison_and_markup_identify_first_command_as_reference() { assert!(stdout.contains("+50.0%"), "{}", stdout); let markdown = std::fs::read_to_string(export_path).unwrap(); insta::assert_snapshot!(markdown, @" - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `baseline` | 2.000 | 2.000 | 2.000 | reference | | `sleep 1` | 1.000 | 1.000 | 1.000 | -50.0% | @@ -840,19 +839,19 @@ fn shows_name_of_first_command() { exit_code: 0 ----- stdout ----- Benchmark 1: refabc123 (10 runs) - mean ± σ min … max - Time 2.000 s ± 0.000 s 2.000 s … 2.000 s - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B + mean ± σ min … max + Wall Time 2.000 s ± 0.000 s 2.000 s … 2.000 s + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B Benchmark 2: sleep 1.0 (10 runs) - mean ± σ min … max - Time 1.000 s ± 0.000 s 1.000 s … 1.000 s -50.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 1.000 s ± 0.000 s 1.000 s … 1.000 s -50.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 3: sleep 3.0 (10 runs) - mean ± σ min … max - Time 3.000 s ± 0.000 s 3.000 s … 3.000 s +50.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 3.000 s ± 0.000 s 3.000 s … 3.000 s +50.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A ----- stderr ----- @@ -876,24 +875,24 @@ fn performs_all_benchmarks_in_parameter_scan() { exit_code: 0 ----- stdout ----- Benchmark 1: sleep 30 (10 runs) - mean ± σ min … max - Time 30.000 s ± 0.000 s 30.000 s … 30.000 s - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B + mean ± σ min … max + Wall Time 30.000 s ± 0.000 s 30.000 s … 30.000 s + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B Benchmark 2: sleep 35 (10 runs) - mean ± σ min … max - Time 35.000 s ± 0.000 s 35.000 s … 35.000 s +16.7% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 35.000 s ± 0.000 s 35.000 s … 35.000 s +16.7% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 3: sleep 40 (10 runs) - mean ± σ min … max - Time 40.000 s ± 0.000 s 40.000 s … 40.000 s +33.3% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 40.000 s ± 0.000 s 40.000 s … 40.000 s +33.3% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A Benchmark 4: sleep 45 (10 runs) - mean ± σ min … max - Time 45.000 s ± 0.000 s 45.000 s … 45.000 s +50.0% - Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A + mean ± σ min … max + Wall Time 45.000 s ± 0.000 s 45.000 s … 45.000 s +50.0% + Memory 0.0 B ± 0.0 B 0.0 B … 0.0 B N/A ----- stderr ----- @@ -1074,7 +1073,7 @@ fn markdown_export_uses_first_parameterized_benchmark_as_reference() { exit_code: 0 ----- stdout ----- - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `case-1` | 1.000 | 1.000 | 1.000 | reference | | `case-2` | 2.000 | 2.000 | 2.000 | +100.0% | @@ -1098,7 +1097,7 @@ fn intermediate_results_are_not_exported_to_stdout() { exit_code: 0 ----- stdout ----- - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | reference | | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | +100.0% | @@ -1184,7 +1183,7 @@ fn markdown_export_preserves_backticks_and_pipes_in_command_names() { exit_code: 0 ----- stdout ----- - | Command | Mean Time [s] | Min [s] | Max [s] | Change | + | Command | Mean Wall Time [s] | Min [s] | Max [s] | Change | |:---|---:|---:|---:|---:| | `` echo `uname` \| cat `` | 1.000 ± 0.000 | 1.000 | 1.000 | reference |