From 1e2e0be25c341f0f56621bc0fd7fa26c7699c6a4 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 09:54:04 +0200 Subject: [PATCH 1/3] Use inline snapshots for execution logs and benchmark reports --- tests/execution_order_tests.rs | 585 +++++++++++++++++---------------- tests/integration_tests.rs | 275 ++++++++++++---- 2 files changed, 517 insertions(+), 343 deletions(-) diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 6ccc2e5..66b2513 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -1,4 +1,4 @@ -use std::{fs::File, io::Read, path::PathBuf}; +use std::{fs, path::PathBuf}; use tempfile::{tempdir, TempDir}; @@ -7,7 +7,6 @@ use common::hyperfine; struct ExecutionOrderTest { cmd: assert_cmd::Command, - expected_content: String, logfile_path: PathBuf, #[allow(dead_code)] tempdir: TempDir, @@ -20,7 +19,6 @@ impl ExecutionOrderTest { ExecutionOrderTest { cmd: hyperfine(), - expected_content: String::new(), logfile_path, tempdir, } @@ -69,26 +67,14 @@ impl ExecutionOrderTest { self.command(output) } - fn expect_output(&mut self, output: &str) -> &mut Self { - self.expected_content.push_str(output); - - #[cfg(windows)] - { - self.expected_content.push_str(" \r"); - } - - self.expected_content.push('\n'); - self - } - - fn run(&mut self) { + fn run(&mut self) -> String { self.cmd.assert().success(); - let mut f = File::open(&self.logfile_path).unwrap(); - let mut content = String::new(); - f.read_to_string(&mut content).unwrap(); - - assert_eq!(content, self.expected_content); + let content = fs::read_to_string(&self.logfile_path).unwrap(); + // cmd.exe's echo includes the space before redirection and uses CRLF. + #[cfg(windows)] + let content = content.replace(" \r\n", "\n"); + content } } @@ -100,167 +86,186 @@ impl Default for ExecutionOrderTest { #[test] fn benchmarks_are_executed_sequentially_one() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 2 + "); } #[test] fn benchmarks_are_executed_sequentially() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + command 2 + "); } #[test] fn warmup_runs_are_executed_before_benchmarking_runs() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--warmup=3") .command("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 1") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 1 + command 1 + command 1 + "); } #[test] fn setup_commands_are_executed_before_each_series_of_timing_runs() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .setup("setup") .command("command 1") .command("command 2") - .expect_output("setup") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("setup") - .expect_output("command 2") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + setup + command 1 + command 1 + setup + command 2 + command 2 + "); } #[test] fn prepare_commands_are_executed_before_each_timing_run() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .prepare("prepare") .command("command 1") .command("command 2") - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 2") - .expect_output("prepare") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + prepare + command 1 + prepare + command 1 + prepare + command 2 + prepare + command 2 + "); } #[test] fn conclude_commands_are_executed_after_each_timing_run() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .conclude("conclude") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + conclude + command 1 + conclude + command 2 + conclude + command 2 + conclude + "); } #[test] fn prepare_commands_are_executed_before_each_warmup() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .prepare("prepare") .command("command 1") .command("command 2") - // warmup 1 - .expect_output("prepare") - .expect_output("command 1") - .expect_output("prepare") - .expect_output("command 1") - // benchmark 1 - .expect_output("prepare") - .expect_output("command 1") - // warmup 2 - .expect_output("prepare") - .expect_output("command 2") - .expect_output("prepare") - .expect_output("command 2") - // benchmark 2 - .expect_output("prepare") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + prepare + command 1 + prepare + command 1 + prepare + command 1 + prepare + command 2 + prepare + command 2 + prepare + command 2 + "); } #[test] fn conclude_commands_are_executed_after_each_warmup() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .conclude("conclude") .command("command 1") .command("command 2") - // warmup 1 - .expect_output("command 1") - .expect_output("conclude") - .expect_output("command 1") - .expect_output("conclude") - // benchmark 1 - .expect_output("command 1") - .expect_output("conclude") - // warmup 2 - .expect_output("command 2") - .expect_output("conclude") - .expect_output("command 2") - .expect_output("conclude") - // benchmark 2 - .expect_output("command 2") - .expect_output("conclude") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + conclude + command 1 + conclude + command 1 + conclude + command 2 + conclude + command 2 + conclude + command 2 + conclude + "); } #[test] fn cleanup_commands_are_executed_once_after_each_benchmark() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .cleanup("cleanup") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("cleanup") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + cleanup + command 2 + command 2 + cleanup + "); } #[test] fn setup_prepare_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -268,30 +273,31 @@ fn setup_prepare_cleanup_combined() { .command("command1") .command("command2") .cleanup("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("prepare") - .expect_output("command1") - .expect_output("prepare") - .expect_output("command1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("prepare") - .expect_output("command2") - .expect_output("prepare") - .expect_output("command2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + command1 + prepare + command1 + prepare + command1 + cleanup + setup + prepare + command2 + prepare + command2 + prepare + command2 + cleanup + "); } #[test] fn setup_prepare_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -300,53 +306,57 @@ fn setup_prepare_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + command1 + conclude + prepare + command1 + conclude + prepare + command1 + conclude + cleanup + setup + prepare + command2 + conclude + prepare + command2 + conclude + prepare + command2 + conclude + cleanup + "); } #[test] fn single_parameter_value() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + command 2 + command 3 + command 3 + "); } #[test] fn multiple_parameter_values() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") @@ -355,70 +365,82 @@ fn multiple_parameter_values() { .arg("letter") .arg("a,b") .command("command {number} {letter}") - .expect_output("command 1 a") - .expect_output("command 1 a") - .expect_output("command 2 a") - .expect_output("command 2 a") - .expect_output("command 3 a") - .expect_output("command 3 a") - .expect_output("command 1 b") - .expect_output("command 1 b") - .expect_output("command 2 b") - .expect_output("command 2 b") - .expect_output("command 3 b") - .expect_output("command 3 b") .run(); + + insta::assert_snapshot!(output, @r" + command 1 a + command 1 a + command 2 a + command 2 a + command 3 a + command 3 a + command 1 b + command 1 b + command 2 b + command 2 b + command 3 b + command 3 b + "); } #[test] fn reference_is_executed_first() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .reference("reference") .command("command 1") .command("command 2") - .expect_output("reference") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + reference + command 1 + command 2 + "); } #[test] fn reference_is_executed_first_parameter_value() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=2") .reference("reference") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") - .expect_output("reference") - .expect_output("reference") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") - .expect_output("command 2") - .expect_output("command 3") - .expect_output("command 3") .run(); + + insta::assert_snapshot!(output, @r" + reference + reference + command 1 + command 1 + command 2 + command 2 + command 3 + command 3 + "); } #[test] fn reference_is_executed_separately_from_commands() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--runs=1") .reference("command 1") .command("command 1") .command("command 2") - .expect_output("command 1") - .expect_output("command 1") - .expect_output("command 2") .run(); + + insta::assert_snapshot!(output, @r" + command 1 + command 1 + command 2 + "); } #[test] fn setup_prepare_reference_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -428,48 +450,48 @@ fn setup_prepare_reference_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") - // reference - .expect_output("setup") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("reference") - .expect_output("conclude") - .expect_output("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command1") - .expect_output("conclude") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("prepare") - .expect_output("command2") - .expect_output("conclude") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare + reference + conclude + prepare + reference + conclude + prepare + reference + conclude + cleanup + setup + prepare + command1 + conclude + prepare + command1 + conclude + prepare + command1 + conclude + cleanup + setup + prepare + command2 + conclude + prepare + command2 + conclude + prepare + command2 + conclude + cleanup + "); } #[test] fn setup_separate_prepare_separate_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -480,36 +502,37 @@ fn setup_separate_prepare_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") - // 1 - .expect_output("setup") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + cleanup + setup + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + cleanup + "); } #[test] fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { - ExecutionOrderTest::new() + let output = ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -523,41 +546,41 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") - // reference - .expect_output("setup") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("prepareref") - .expect_output("reference") - .expect_output("concluderef") - .expect_output("cleanup") - // 1 - .expect_output("setup") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("prepare1") - .expect_output("command1") - .expect_output("conclude1") - .expect_output("cleanup") - // 2 - .expect_output("setup") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("prepare2") - .expect_output("command2") - .expect_output("conclude2") - .expect_output("cleanup") .run(); + + insta::assert_snapshot!(output, @r" + setup + prepareref + reference + concluderef + prepareref + reference + concluderef + prepareref + reference + concluderef + cleanup + setup + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + prepare1 + command1 + conclude1 + cleanup + setup + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + prepare2 + command2 + conclude2 + cleanup + "); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1141827..504f7b9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -15,6 +15,16 @@ pub fn hyperfine_debug() -> assert_cmd::Command { cmd } +// Ignore trailing padding on report lines, including the space-only separators. +// Keep indentation and internal spacing so snapshots still check the layout. +fn stdout(output: &assert_cmd::assert::Assert) -> String { + String::from_utf8_lossy(&output.get_output().stdout) + .lines() + .map(str::trim_end) + .collect::>() + .join("\n") +} + #[test] fn runs_successfully() { hyperfine() @@ -594,21 +604,38 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb #[test] fn shows_benchmark_comparison_with_relative_times() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("sleep 1.0") .arg("sleep 2.0") .arg("sleep 3.0") .assert() - .success() - .stdout( - predicate::str::contains("2.00 ± 0.00 times faster") - .and(predicate::str::contains("3.00 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 2: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + sleep 1.0 ran + 2.00 ± 0.00 times faster than sleep 2.0 + 3.00 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn shows_benchmark_comparison_with_same_time() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--command-name=A") .arg("--command-name=B") .arg("sleep 1.0") @@ -616,43 +643,98 @@ fn shows_benchmark_comparison_with_same_time() { .arg("sleep 2.0") .arg("sleep 1000.0") .assert() - .success() - .stdout( - predicate::str::contains("As fast (1.00 ± 0.00) as") - .and(predicate::str::contains("2.00 ± 0.00 times faster")) - .and(predicate::str::contains("1000.00 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: A + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 2: B + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 4: sleep 1000.0 + Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1000.000 s … 1000.000 s 10 runs + + Summary + A ran + As fast (1.00 ± 0.00) as B + 2.00 ± 0.00 times faster than sleep 2.0 + 1000.00 ± 0.00 times faster than sleep 1000.0 + "); } #[test] fn shows_benchmark_comparison_relative_to_reference() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("sleep 1.0") .arg("sleep 3.0") .assert() - .success() - .stdout( - predicate::str::contains("2.00 ± 0.00 times slower") - .and(predicate::str::contains("1.50 ± 0.00 times faster")), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2.0 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + sleep 2.0 ran + 2.00 ± 0.00 times slower than sleep 1.0 + 1.50 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn shows_reference_name() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("--reference-name=refabc123") .arg("sleep 1.0") .arg("sleep 3.0") .assert() - .success() - .stdout(predicate::str::contains("Benchmark 1: refabc123")); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: refabc123 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1.0 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Benchmark 3: sleep 3.0 + Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 3.000 s … 3.000 s 10 runs + + Summary + refabc123 ran + 2.00 ± 0.00 times slower than sleep 1.0 + 1.50 ± 0.00 times faster than sleep 3.0 + "); } #[test] fn performs_all_benchmarks_in_parameter_scan() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--parameter-scan") .arg("time") .arg("30") @@ -661,19 +743,37 @@ fn performs_all_benchmarks_in_parameter_scan() { .arg("5") .arg("sleep {time}") .assert() - .success() - .stdout( - predicate::str::contains("Benchmark 1: sleep 30") - .and(predicate::str::contains("Benchmark 2: sleep 35")) - .and(predicate::str::contains("Benchmark 3: sleep 40")) - .and(predicate::str::contains("Benchmark 4: sleep 45")) - .and(predicate::str::contains("Benchmark 5: sleep 50").not()), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 30 + Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 30.000 s … 30.000 s 10 runs + + Benchmark 2: sleep 35 + Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 35.000 s … 35.000 s 10 runs + + Benchmark 3: sleep 40 + Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 40.000 s … 40.000 s 10 runs + + Benchmark 4: sleep 45 + Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 45.000 s … 45.000 s 10 runs + + Summary + sleep 30 ran + 1.17 ± 0.00 times faster than sleep 35 + 1.33 ± 0.00 times faster than sleep 40 + 1.50 ± 0.00 times faster than sleep 45 + "); } #[test] fn performs_reference_and_all_benchmarks_in_parameter_scan() { - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("--reference=sleep 25") .arg("--parameter-scan") .arg("time") @@ -683,31 +783,56 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { .arg("5") .arg("sleep {time}") .assert() - .success() - .stdout( - predicate::str::contains("Benchmark 1: sleep 25") - .and(predicate::str::contains("Benchmark 2: sleep 30")) - .and(predicate::str::contains("Benchmark 3: sleep 35")) - .and(predicate::str::contains("Benchmark 4: sleep 40")) - .and(predicate::str::contains("Benchmark 5: sleep 45")) - .and(predicate::str::contains("Benchmark 6: sleep 50").not()), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 25 + Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 25.000 s … 25.000 s 10 runs + + Benchmark 2: sleep 30 + Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 30.000 s … 30.000 s 10 runs + + Benchmark 3: sleep 35 + Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 35.000 s … 35.000 s 10 runs + + Benchmark 4: sleep 40 + Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 40.000 s … 40.000 s 10 runs + + Benchmark 5: sleep 45 + Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 45.000 s … 45.000 s 10 runs + + Summary + sleep 25 ran + 1.20 ± 0.00 times faster than sleep 30 + 1.40 ± 0.00 times faster than sleep 35 + 1.60 ± 0.00 times faster than sleep 40 + 1.80 ± 0.00 times faster than sleep 45 + "); } #[test] fn intermediate_results_are_not_exported_to_stdout() { - hyperfine_debug() + let output = hyperfine_debug() .arg("--style=none") // To only see the Markdown export on stdout .arg("--export-markdown") .arg("-") .arg("sleep 1") .arg("sleep 2") .assert() - .success() - .stdout( - (predicate::str::contains("sleep 1").count(1)) - .and(predicate::str::contains("sleep 2").count(1)), - ); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + + | Command | Mean [s] | Min [s] | Max [s] | Relative | + |:---|---:|---:|---:|---:| + | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 | + | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 | + "); } #[test] @@ -749,27 +874,53 @@ fn unused_parameters_are_shown_in_benchmark_name() { #[test] fn speed_comparison_sort_order() { - for sort_order in ["auto", "mean-time"] { - hyperfine_debug() - .arg("sleep 2") - .arg("sleep 1") - .arg(format!("--sort={sort_order}")) - .assert() - .success() - .stdout(predicate::str::contains( - "sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2", - )); + insta::allow_duplicates! { + for sort_order in ["auto", "mean-time"] { + let output = hyperfine_debug() + .arg("--style=basic") + .arg("sleep 2") + .arg("sleep 1") + .arg(format!("--sort={sort_order}")) + .assert() + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Summary + sleep 1 ran + 2.00 ± 0.00 times faster than sleep 2 + "); + } } - hyperfine_debug() + let output = hyperfine_debug() + .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") .arg("--sort=command") .assert() - .success() - .stdout(predicate::str::contains( - "2.00 ± 0.00 sleep 2\n 1.00 sleep 1", - )); + .success(); + + insta::assert_snapshot!(stdout(&output), @r" + Benchmark 1: sleep 2 + Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 2.000 s … 2.000 s 10 runs + + Benchmark 2: sleep 1 + Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] + Range (min … max): 1.000 s … 1.000 s 10 runs + + Relative speed comparison + 2.00 ± 0.00 sleep 2 + 1.00 sleep 1 + "); } #[cfg(windows)] From c8f8c4ea5b642a2d7c2cc2c37acc9450f4b9f7c7 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 10:00:29 +0200 Subject: [PATCH 2/3] Keep snapshot conversions limited to integration tests --- tests/execution_order_tests.rs | 585 ++++++++++++++++----------------- 1 file changed, 281 insertions(+), 304 deletions(-) diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index 66b2513..6ccc2e5 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -1,4 +1,4 @@ -use std::{fs, path::PathBuf}; +use std::{fs::File, io::Read, path::PathBuf}; use tempfile::{tempdir, TempDir}; @@ -7,6 +7,7 @@ use common::hyperfine; struct ExecutionOrderTest { cmd: assert_cmd::Command, + expected_content: String, logfile_path: PathBuf, #[allow(dead_code)] tempdir: TempDir, @@ -19,6 +20,7 @@ impl ExecutionOrderTest { ExecutionOrderTest { cmd: hyperfine(), + expected_content: String::new(), logfile_path, tempdir, } @@ -67,14 +69,26 @@ impl ExecutionOrderTest { self.command(output) } - fn run(&mut self) -> String { + fn expect_output(&mut self, output: &str) -> &mut Self { + self.expected_content.push_str(output); + + #[cfg(windows)] + { + self.expected_content.push_str(" \r"); + } + + self.expected_content.push('\n'); + self + } + + fn run(&mut self) { self.cmd.assert().success(); - let content = fs::read_to_string(&self.logfile_path).unwrap(); - // cmd.exe's echo includes the space before redirection and uses CRLF. - #[cfg(windows)] - let content = content.replace(" \r\n", "\n"); - content + let mut f = File::open(&self.logfile_path).unwrap(); + let mut content = String::new(); + f.read_to_string(&mut content).unwrap(); + + assert_eq!(content, self.expected_content); } } @@ -86,186 +100,167 @@ impl Default for ExecutionOrderTest { #[test] fn benchmarks_are_executed_sequentially_one() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=1") .command("command 1") .command("command 2") + .expect_output("command 1") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 2 - "); } #[test] fn benchmarks_are_executed_sequentially() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .command("command 1") .command("command 2") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 1 - command 2 - command 2 - "); } #[test] fn warmup_runs_are_executed_before_benchmarking_runs() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .arg("--warmup=3") .command("command 1") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 1") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 1 - command 1 - command 1 - command 1 - "); } #[test] fn setup_commands_are_executed_before_each_series_of_timing_runs() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .setup("setup") .command("command 1") .command("command 2") + .expect_output("setup") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("setup") + .expect_output("command 2") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - setup - command 1 - command 1 - setup - command 2 - command 2 - "); } #[test] fn prepare_commands_are_executed_before_each_timing_run() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .prepare("prepare") .command("command 1") .command("command 2") + .expect_output("prepare") + .expect_output("command 1") + .expect_output("prepare") + .expect_output("command 1") + .expect_output("prepare") + .expect_output("command 2") + .expect_output("prepare") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - prepare - command 1 - prepare - command 1 - prepare - command 2 - prepare - command 2 - "); } #[test] fn conclude_commands_are_executed_after_each_timing_run() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .conclude("conclude") .command("command 1") .command("command 2") + .expect_output("command 1") + .expect_output("conclude") + .expect_output("command 1") + .expect_output("conclude") + .expect_output("command 2") + .expect_output("conclude") + .expect_output("command 2") + .expect_output("conclude") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - conclude - command 1 - conclude - command 2 - conclude - command 2 - conclude - "); } #[test] fn prepare_commands_are_executed_before_each_warmup() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .prepare("prepare") .command("command 1") .command("command 2") + // warmup 1 + .expect_output("prepare") + .expect_output("command 1") + .expect_output("prepare") + .expect_output("command 1") + // benchmark 1 + .expect_output("prepare") + .expect_output("command 1") + // warmup 2 + .expect_output("prepare") + .expect_output("command 2") + .expect_output("prepare") + .expect_output("command 2") + // benchmark 2 + .expect_output("prepare") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - prepare - command 1 - prepare - command 1 - prepare - command 1 - prepare - command 2 - prepare - command 2 - prepare - command 2 - "); } #[test] fn conclude_commands_are_executed_after_each_warmup() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=2") .arg("--runs=1") .conclude("conclude") .command("command 1") .command("command 2") + // warmup 1 + .expect_output("command 1") + .expect_output("conclude") + .expect_output("command 1") + .expect_output("conclude") + // benchmark 1 + .expect_output("command 1") + .expect_output("conclude") + // warmup 2 + .expect_output("command 2") + .expect_output("conclude") + .expect_output("command 2") + .expect_output("conclude") + // benchmark 2 + .expect_output("command 2") + .expect_output("conclude") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - conclude - command 1 - conclude - command 1 - conclude - command 2 - conclude - command 2 - conclude - command 2 - conclude - "); } #[test] fn cleanup_commands_are_executed_once_after_each_benchmark() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .cleanup("cleanup") .command("command 1") .command("command 2") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("cleanup") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 1 - cleanup - command 2 - command 2 - cleanup - "); } #[test] fn setup_prepare_cleanup_combined() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -273,31 +268,30 @@ fn setup_prepare_cleanup_combined() { .command("command1") .command("command2") .cleanup("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command1") + .expect_output("prepare") + .expect_output("command1") + .expect_output("prepare") + .expect_output("command1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command2") + .expect_output("prepare") + .expect_output("command2") + .expect_output("prepare") + .expect_output("command2") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - setup - prepare - command1 - prepare - command1 - prepare - command1 - cleanup - setup - prepare - command2 - prepare - command2 - prepare - command2 - cleanup - "); } #[test] fn setup_prepare_conclude_cleanup_combined() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -306,57 +300,53 @@ fn setup_prepare_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - setup - prepare - command1 - conclude - prepare - command1 - conclude - prepare - command1 - conclude - cleanup - setup - prepare - command2 - conclude - prepare - command2 - conclude - prepare - command2 - conclude - cleanup - "); } #[test] fn single_parameter_value() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("command 3") + .expect_output("command 3") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 1 - command 2 - command 2 - command 3 - command 3 - "); } #[test] fn multiple_parameter_values() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .arg("--parameter-list") .arg("number") @@ -365,82 +355,70 @@ fn multiple_parameter_values() { .arg("letter") .arg("a,b") .command("command {number} {letter}") + .expect_output("command 1 a") + .expect_output("command 1 a") + .expect_output("command 2 a") + .expect_output("command 2 a") + .expect_output("command 3 a") + .expect_output("command 3 a") + .expect_output("command 1 b") + .expect_output("command 1 b") + .expect_output("command 2 b") + .expect_output("command 2 b") + .expect_output("command 3 b") + .expect_output("command 3 b") .run(); - - insta::assert_snapshot!(output, @r" - command 1 a - command 1 a - command 2 a - command 2 a - command 3 a - command 3 a - command 1 b - command 1 b - command 2 b - command 2 b - command 3 b - command 3 b - "); } #[test] fn reference_is_executed_first() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=1") .reference("reference") .command("command 1") .command("command 2") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - reference - command 1 - command 2 - "); } #[test] fn reference_is_executed_first_parameter_value() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=2") .reference("reference") .arg("--parameter-list") .arg("number") .arg("1,2,3") .command("command {number}") + .expect_output("reference") + .expect_output("reference") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") + .expect_output("command 2") + .expect_output("command 3") + .expect_output("command 3") .run(); - - insta::assert_snapshot!(output, @r" - reference - reference - command 1 - command 1 - command 2 - command 2 - command 3 - command 3 - "); } #[test] fn reference_is_executed_separately_from_commands() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--runs=1") .reference("command 1") .command("command 1") .command("command 2") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("command 2") .run(); - - insta::assert_snapshot!(output, @r" - command 1 - command 1 - command 2 - "); } #[test] fn setup_prepare_reference_conclude_cleanup_combined() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -450,48 +428,48 @@ fn setup_prepare_reference_conclude_cleanup_combined() { .command("command2") .conclude("conclude") .cleanup("cleanup") + // reference + .expect_output("setup") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("reference") + .expect_output("conclude") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command1") + .expect_output("conclude") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("prepare") + .expect_output("command2") + .expect_output("conclude") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - setup - prepare - reference - conclude - prepare - reference - conclude - prepare - reference - conclude - cleanup - setup - prepare - command1 - conclude - prepare - command1 - conclude - prepare - command1 - conclude - cleanup - setup - prepare - command2 - conclude - prepare - command2 - conclude - prepare - command2 - conclude - cleanup - "); } #[test] fn setup_separate_prepare_separate_conclude_cleanup_combined() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -502,37 +480,36 @@ fn setup_separate_prepare_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - setup - prepare1 - command1 - conclude1 - prepare1 - command1 - conclude1 - prepare1 - command1 - conclude1 - cleanup - setup - prepare2 - command2 - conclude2 - prepare2 - command2 - conclude2 - prepare2 - command2 - conclude2 - cleanup - "); } #[test] fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { - let output = ExecutionOrderTest::new() + ExecutionOrderTest::new() .arg("--warmup=1") .arg("--runs=2") .setup("setup") @@ -546,41 +523,41 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() { .prepare("prepare2") .command("command2") .conclude("conclude2") + // reference + .expect_output("setup") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("prepareref") + .expect_output("reference") + .expect_output("concluderef") + .expect_output("cleanup") + // 1 + .expect_output("setup") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("prepare1") + .expect_output("command1") + .expect_output("conclude1") + .expect_output("cleanup") + // 2 + .expect_output("setup") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("prepare2") + .expect_output("command2") + .expect_output("conclude2") + .expect_output("cleanup") .run(); - - insta::assert_snapshot!(output, @r" - setup - prepareref - reference - concluderef - prepareref - reference - concluderef - prepareref - reference - concluderef - cleanup - setup - prepare1 - command1 - conclude1 - prepare1 - command1 - conclude1 - prepare1 - command1 - conclude1 - cleanup - setup - prepare2 - command2 - conclude2 - prepare2 - command2 - conclude2 - prepare2 - command2 - conclude2 - cleanup - "); } From a24f76c0cee2bc39cd1802f5d89e842bcd8fab93 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 10:11:53 +0200 Subject: [PATCH 3/3] Use insta-cmd for benchmark report snapshots --- Cargo.lock | 32 ++++++++ Cargo.toml | 3 +- tests/integration_tests.rs | 145 +++++++++++++++++++++---------------- 3 files changed, 115 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5cca2e..3b5c95e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -412,6 +412,7 @@ dependencies = [ "csv", "indicatif", "insta", + "insta-cmd", "libc", "nix", "once_cell", @@ -458,11 +459,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console 0.16.6", "once_cell", + "regex", "serde", "similar", + "strip-ansi-escapes", "tempfile", ] +[[package]] +name = "insta-cmd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403" +dependencies = [ + "insta", + "serde", + "serde_json", +] + [[package]] name = "instant" version = "0.1.13" @@ -1024,6 +1038,15 @@ dependencies = [ "rand 0.6.5", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1155,6 +1178,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index dec668a..0ba6a49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,8 @@ features = [ [dev-dependencies] approx = "0.5" assert_cmd = "2.2" -insta = { version = "1.48", features = ["yaml"] } +insta = { version = "1.48", features = ["yaml", "filters"] } +insta-cmd = "0.7" predicates = "3.1" tempfile = "3.27" diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 504f7b9..36295d9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,6 +1,8 @@ mod common; use common::{hyperfine, hyperfine_raw_command}; +use assert_cmd::assert::OutputAssertExt; +use insta_cmd::assert_cmd_snapshot; use predicates::prelude::*; /// Platform-specific I/O utility. @@ -9,20 +11,17 @@ use predicates::prelude::*; /// See: const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" }; -pub fn hyperfine_debug() -> assert_cmd::Command { - let mut cmd = hyperfine(); +pub fn hyperfine_debug() -> std::process::Command { + let mut cmd = hyperfine_raw_command(); cmd.arg("--debug-mode"); cmd } -// Ignore trailing padding on report lines, including the space-only separators. -// Keep indentation and internal spacing so snapshots still check the layout. -fn stdout(output: &assert_cmd::assert::Assert) -> String { - String::from_utf8_lossy(&output.get_output().stdout) - .lines() - .map(str::trim_end) - .collect::>() - .join("\n") +fn snapshot_settings() -> insta::Settings { + let mut settings = insta::Settings::clone_current(); + // Ignore trailing report padding while preserving indentation and internal spacing. + settings.add_filter(r"(?m)[ \t]+$", ""); + settings } #[test] @@ -604,15 +603,15 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb #[test] fn shows_benchmark_comparison_with_relative_times() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 1.0") .arg("sleep 2.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 1.0 Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 1.000 s … 1.000 s 10 runs @@ -629,23 +628,25 @@ fn shows_benchmark_comparison_with_relative_times() { sleep 1.0 ran 2.00 ± 0.00 times faster than sleep 2.0 3.00 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn shows_benchmark_comparison_with_same_time() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--command-name=A") .arg("--command-name=B") .arg("sleep 1.0") .arg("sleep 1.0") .arg("sleep 2.0") - .arg("sleep 1000.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 1000.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: A Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 1.000 s … 1.000 s 10 runs @@ -667,20 +668,22 @@ fn shows_benchmark_comparison_with_same_time() { As fast (1.00 ± 0.00) as B 2.00 ± 0.00 times faster than sleep 2.0 1000.00 ± 0.00 times faster than sleep 1000.0 + + ----- stderr ----- "); } #[test] fn shows_benchmark_comparison_relative_to_reference() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("sleep 1.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2.0 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -697,21 +700,23 @@ fn shows_benchmark_comparison_relative_to_reference() { sleep 2.0 ran 2.00 ± 0.00 times slower than sleep 1.0 1.50 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn shows_reference_name() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("--reference-name=refabc123") .arg("sleep 1.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: refabc123 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -728,12 +733,15 @@ fn shows_reference_name() { refabc123 ran 2.00 ± 0.00 times slower than sleep 1.0 1.50 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn performs_all_benchmarks_in_parameter_scan() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--parameter-scan") .arg("time") @@ -741,11 +749,10 @@ fn performs_all_benchmarks_in_parameter_scan() { .arg("45") .arg("--parameter-step-size") .arg("5") - .arg("sleep {time}") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep {time}"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 30 Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 30.000 s … 30.000 s 10 runs @@ -767,12 +774,15 @@ fn performs_all_benchmarks_in_parameter_scan() { 1.17 ± 0.00 times faster than sleep 35 1.33 ± 0.00 times faster than sleep 40 1.50 ± 0.00 times faster than sleep 45 + + ----- stderr ----- "); } #[test] fn performs_reference_and_all_benchmarks_in_parameter_scan() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 25") .arg("--parameter-scan") @@ -781,11 +791,10 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { .arg("45") .arg("--parameter-step-size") .arg("5") - .arg("sleep {time}") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep {time}"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 25 Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 25.000 s … 25.000 s 10 runs @@ -812,26 +821,31 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { 1.40 ± 0.00 times faster than sleep 35 1.60 ± 0.00 times faster than sleep 40 1.80 ± 0.00 times faster than sleep 45 + + ----- stderr ----- "); } #[test] fn intermediate_results_are_not_exported_to_stdout() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=none") // To only see the Markdown export on stdout .arg("--export-markdown") .arg("-") .arg("sleep 1") - .arg("sleep 2") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 2"), @r" + success: true + exit_code: 0 + ----- stdout ----- | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 | | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 | + + + ----- stderr ----- "); } @@ -874,17 +888,17 @@ fn unused_parameters_are_shown_in_benchmark_name() { #[test] fn speed_comparison_sort_order() { + let _settings = snapshot_settings().bind_to_scope(); insta::allow_duplicates! { for sort_order in ["auto", "mean-time"] { - let output = hyperfine_debug() + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") - .arg(format!("--sort={sort_order}")) - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg(format!("--sort={sort_order}")), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -896,19 +910,20 @@ fn speed_comparison_sort_order() { Summary sleep 1 ran 2.00 ± 0.00 times faster than sleep 2 + + ----- stderr ----- "); } } - let output = hyperfine_debug() + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") - .arg("--sort=command") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("--sort=command"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -920,6 +935,8 @@ fn speed_comparison_sort_order() { Relative speed comparison 2.00 ± 0.00 sleep 2 1.00 sleep 1 + + ----- stderr ----- "); }