From 1e2e0be25c341f0f56621bc0fd7fa26c7699c6a4 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 09:54:04 +0200 Subject: [PATCH] Use inline snapshots for execution logs and benchmark reports --- tests/execution_order_tests.rs | 585 +++++++++++++++++---------------- tests/integration_tests.rs | 275 ++++++++++++---- 2 files changed, 517 insertions(+), 343 deletions(-) diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5..66b2513 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -1,4 +1,4 @@ -use std::{fs::File, io::Read, path::PathBuf}; +use std::{fs, path::PathBuf}; use tempfile::{tempdir, TempDir}; @@ -7,7 +7,6 @@ use common::hyperfine; struct ExecutionOrderTest { cmd: assert_cmd::Command, - expected_content: String, logfile_path: PathBuf, #[allow(dead_code)] tempdir: TempDir, @@ -20,7 +19,6 @@ impl ExecutionOrderTest { ExecutionOrderTest { cmd: hyperfine(), - expected_content: String::new(), logfile_path, tempdir, } @@ -69,26 +67,14 @@ impl ExecutionOrderTest { self.command(output) } - fn expect_output(&mut self, output: &str) -> &mut Self { - self.expected_content.push_str(output); - - #[cfg(windows)] - { - self.expected_content.push_str(" \r"); - } - - self.expected_content.push('\n'); - self - } - - fn run(&mut self) { + fn run(&mut self) -> String { self.cmd.assert().success(); - let mut f = File::open(&self.logfile_path).unwrap(); - let mut content = String::new(); - f.read_to_string(&mut content).unwrap(); - - assert_eq!(content, self.expected_content); + let content = fs::read_to_string(&self.logfile_path).unwrap(); + // cmd.exe's echo includes the space before redirection and uses CRLF. + #[cfg(windows)] + let content = content.replace(" \r\n", "\n"); + content } } @@ -100,167 +86,186 @@ impl Default for ExecutionOrderTest { #[test] fn benchmarks_are_executed_sequentially_one() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 2 + "); } #[test] fn benchmarks_are_executed_sequentially() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + command 2 + "); } #[test] fn warmup_runs_are_executed_before_benchmarking_runs() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--warmup=3") .command("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 1 + command 1 + command 1 + "); } #[test] fn setup_commands_are_executed_before_each_series_of_timing_runs() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .setup("setup") .command("command 1") .command("command 2") - .expect_output("setup") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("setup") - .expect_output("command 2") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + setup + command 1 + command 1 + setup + command 2 + command 2 + "); } #[test] fn prepare_commands_are_executed_before_each_timing_run() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .prepare("prepare") .command("command 1") .command("command 2") - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 2") - .expect_output("prepare") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + prepare + command 1 + prepare + command 1 + prepare + command 2 + prepare + command 2 + "); } #[test] fn conclude_commands_are_executed_after_each_timing_run() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .conclude("conclude") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + conclude + command 1 + conclude + command 2 + conclude + command 2 + conclude + "); } #[test] fn prepare_commands_are_executed_before_each_warmup() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .prepare("prepare") .command("command 1") .command("command 2") - // warmup 1 - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 1") - // benchmark 1 - .expect_output("prepare") - .expect_output("command 1") - // warmup 2 - .expect_output("prepare") - .expect_output("command 2") - .expect_output("prepare") - .expect_output("command 2") - // benchmark 2 - .expect_output("prepare") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + prepare + command 1 + prepare + command 1 + prepare + command 1 + prepare + command 2 + prepare + command 2 + prepare + command 2 + "); } #[test] fn conclude_commands_are_executed_after_each_warmup() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .conclude("conclude") .command("command 1") .command("command 2") - // warmup 1 - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 1") - .expect_output("conclude") - // benchmark 1 - .expect_output("command 1") - .expect_output("conclude") - // warmup 2 - .expect_output("command 2") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") - // benchmark 2 - .expect_output("command 2") - .expect_output("conclude") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + conclude + command 1 + conclude + command 1 + conclude + command 2 + conclude + command 2 + conclude + command 2 + conclude + "); } #[test] fn cleanup_commands_are_executed_once_after_each_benchmark() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .cleanup("cleanup") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("cleanup") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + cleanup + command 2 + command 2 + cleanup + "); } #[test] fn setup_prepare_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -268,30 +273,31 @@ fn setup_prepare_cleanup_combined() { .command("command1") .command("command2") .cleanup("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("prepare") - .expect_output("command1") - .expect_output("prepare") - .expect_output("command1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("prepare") - .expect_output("command2") - .expect_output("prepare") - .expect_output("command2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + command1 + prepare + command1 + prepare + command1 + cleanup + setup + prepare + command2 + prepare + command2 + prepare + command2 + cleanup + "); } #[test] fn setup_prepare_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -300,53 +306,57 @@ fn setup_prepare_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + command1 + conclude + prepare + command1 + conclude + prepare + command1 + conclude + cleanup + setup + prepare + command2 + conclude + prepare + command2 + conclude + prepare + command2 + conclude + cleanup + "); } #[test] fn single_parameter_value() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + command 2 + command 3 + command 3 + "); } #[test] fn multiple_parameter_values() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") @@ -355,70 +365,82 @@ fn multiple_parameter_values() { .arg("letter") .arg("a,b") .command("command {number} {letter}") - .expect_output("command 1 a") - .expect_output("command 1 a") - .expect_output("command 2 a") - .expect_output("command 2 a") - .expect_output("command 3 a") - .expect_output("command 3 a") - .expect_output("command 1 b") - .expect_output("command 1 b") - .expect_output("command 2 b") - .expect_output("command 2 b") - .expect_output("command 3 b") - .expect_output("command 3 b") .run(); + + insta::assert_snapshot!(output, @r" + command 1 a + command 1 a + command 2 a + command 2 a + command 3 a + command 3 a + command 1 b + command 1 b + command 2 b + command 2 b + command 3 b + command 3 b + "); } #[test] fn reference_is_executed_first() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .reference("reference") .command("command 1") .command("command 2") - .expect_output("reference") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + reference + command 1 + command 2 + "); } #[test] fn reference_is_executed_first_parameter_value() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .reference("reference") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("reference") - .expect_output("reference") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") .run(); + + insta::assert_snapshot!(output, @r" + reference + reference + command 1 + command 1 + command 2 + command 2 + command 3 + command 3 + "); } #[test] fn reference_is_executed_separately_from_commands() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .reference("command 1") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + "); } #[test] fn setup_prepare_reference_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -428,48 +450,48 @@ fn setup_prepare_reference_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") - // reference - .expect_output("setup") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + reference + conclude + prepare + reference + conclude + prepare + reference + conclude + cleanup + setup + prepare + command1 + conclude + prepare + command1 + conclude + prepare + command1 + conclude + cleanup + setup + prepare + command2 + conclude + prepare + command2 + conclude + prepare + command2 + conclude + cleanup + "); } #[test] fn setup_separate_prepare_separate_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -480,36 +502,37 @@ fn setup_separate_prepare_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") - // 1 - .expect_output("setup") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + cleanup + setup + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + cleanup + "); } #[test] fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -523,41 +546,41 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") - // reference - .expect_output("setup") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepareref + reference + concluderef + prepareref + reference + concluderef + prepareref + reference + concluderef + cleanup + setup + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + cleanup + setup + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + cleanup + "); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1141827..504f7b9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -15,6 +15,16 @@ pub fn hyperfine_debug() -> assert_cmd::Command { cmd } +// Ignore trailing padding on report lines, including the space-only separators. +// Keep indentation and internal spacing so snapshots still check the layout. +fn stdout(output: &assert_cmd::assert::Assert) -> String { + String::from_utf8_lossy(&output.get_output().stdout) + .lines() + .map(str::trim_end) + .collect::>() + .join("\n") +} + #[test] fn runs_successfully() { hyperfine() @@ -594,21 +604,38 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb #[test] fn shows_benchmark_comparison_with_relative_times() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("sleep 1.0") .arg("sleep 2.0") .arg("sleep 3.0") .assert() - .success() - .stdout( - predicate::str::contains("2.00 ± 0.00 times faster") - .and(predicate::str::contains("3.00 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 2: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + sleep 1.0 ran + 2.00 ± 0.00 times faster than sleep 2.0 + 3.00 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn shows_benchmark_comparison_with_same_time() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--command-name=A") .arg("--command-name=B") .arg("sleep 1.0") @@ -616,43 +643,98 @@ fn shows_benchmark_comparison_with_same_time() { .arg("sleep 2.0") .arg("sleep 1000.0") .assert() - .success() - .stdout( - predicate::str::contains("As fast (1.00 ± 0.00) as") - .and(predicate::str::contains("2.00 ± 0.00 times faster")) - .and(predicate::str::contains("1000.00 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: A + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 2: B + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 4: sleep 1000.0 + Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1000.000 s … 1000.000 s 10 runs + + Summary + A ran + As fast (1.00 ± 0.00) as B + 2.00 ± 0.00 times faster than sleep 2.0 + 1000.00 ± 0.00 times faster than sleep 1000.0 + "); } #[test] fn shows_benchmark_comparison_relative_to_reference() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("sleep 1.0") .arg("sleep 3.0") .assert() - .success() - .stdout( - predicate::str::contains("2.00 ± 0.00 times slower") - .and(predicate::str::contains("1.50 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + sleep 2.0 ran + 2.00 ± 0.00 times slower than sleep 1.0 + 1.50 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn shows_reference_name() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("--reference-name=refabc123") .arg("sleep 1.0") .arg("sleep 3.0") .assert() - .success() - .stdout(predicate::str::contains("Benchmark 1: refabc123")); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: refabc123 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + refabc123 ran + 2.00 ± 0.00 times slower than sleep 1.0 + 1.50 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn performs_all_benchmarks_in_parameter_scan() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--parameter-scan") .arg("time") .arg("30") @@ -661,19 +743,37 @@ fn performs_all_benchmarks_in_parameter_scan() { .arg("5") .arg("sleep {time}") .assert() - .success() - .stdout( - predicate::str::contains("Benchmark 1: sleep 30") - .and(predicate::str::contains("Benchmark 2: sleep 35")) - .and(predicate::str::contains("Benchmark 3: sleep 40")) - .and(predicate::str::contains("Benchmark 4: sleep 45")) - .and(predicate::str::contains("Benchmark 5: sleep 50").not()), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 30 + Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 30.000 s … 30.000 s 10 runs + + Benchmark 2: sleep 35 + Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 35.000 s … 35.000 s 10 runs + + Benchmark 3: sleep 40 + Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 40.000 s … 40.000 s 10 runs + + Benchmark 4: sleep 45 + Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 45.000 s … 45.000 s 10 runs + + Summary + sleep 30 ran + 1.17 ± 0.00 times faster than sleep 35 + 1.33 ± 0.00 times faster than sleep 40 + 1.50 ± 0.00 times faster than sleep 45 + "); } #[test] fn performs_reference_and_all_benchmarks_in_parameter_scan() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 25") .arg("--parameter-scan") .arg("time") @@ -683,31 +783,56 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { .arg("5") .arg("sleep {time}") .assert() - .success() - .stdout( - predicate::str::contains("Benchmark 1: sleep 25") - .and(predicate::str::contains("Benchmark 2: sleep 30")) - .and(predicate::str::contains("Benchmark 3: sleep 35")) - .and(predicate::str::contains("Benchmark 4: sleep 40")) - .and(predicate::str::contains("Benchmark 5: sleep 45")) - .and(predicate::str::contains("Benchmark 6: sleep 50").not()), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 25 + Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 25.000 s … 25.000 s 10 runs + + Benchmark 2: sleep 30 + Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 30.000 s … 30.000 s 10 runs + + Benchmark 3: sleep 35 + Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 35.000 s … 35.000 s 10 runs + + Benchmark 4: sleep 40 + Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 40.000 s … 40.000 s 10 runs + + Benchmark 5: sleep 45 + Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 45.000 s … 45.000 s 10 runs + + Summary + sleep 25 ran + 1.20 ± 0.00 times faster than sleep 30 + 1.40 ± 0.00 times faster than sleep 35 + 1.60 ± 0.00 times faster than sleep 40 + 1.80 ± 0.00 times faster than sleep 45 + "); } #[test] fn intermediate_results_are_not_exported_to_stdout() { - hyperfine_debug() + let output = hyperfine_debug() .arg("--style=none") // To only see the Markdown export on stdout .arg("--export-markdown") .arg("-") .arg("sleep 1") .arg("sleep 2") .assert() - .success() - .stdout( - (predicate::str::contains("sleep 1").count(1)) - .and(predicate::str::contains("sleep 2").count(1)), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 | + | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 | + "); } #[test] @@ -749,27 +874,53 @@ fn unused_parameters_are_shown_in_benchmark_name() { #[test] fn speed_comparison_sort_order() { - for sort_order in ["auto", "mean-time"] { - hyperfine_debug() - .arg("sleep 2") - .arg("sleep 1") - .arg(format!("--sort={sort_order}")) - .assert() - .success() - .stdout(predicate::str::contains( - "sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2", - )); + insta::allow_duplicates! { + for sort_order in ["auto", "mean-time"] { + let output = hyperfine_debug() + .arg("--style=basic") + .arg("sleep 2") + .arg("sleep 1") + .arg(format!("--sort={sort_order}")) + .assert() + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Summary + sleep 1 ran + 2.00 ± 0.00 times faster than sleep 2 + "); + } } - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") .arg("--sort=command") .assert() - .success() - .stdout(predicate::str::contains( - "2.00 ± 0.00 sleep 2\n 1.00 sleep 1", - )); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Relative speed comparison + 2.00 ± 0.00 sleep 2 + 1.00 sleep 1 + "); } #[cfg(windows)]