From a24f76c0cee2bc39cd1802f5d89e842bcd8fab93 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 10:11:53 +0200 Subject: [PATCH] Use insta-cmd for benchmark report snapshots --- Cargo.lock | 32 ++++++++ Cargo.toml | 3 +- tests/integration_tests.rs | 145 +++++++++++++++++++++---------------- 3 files changed, 115 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5cca2e..3b5c95e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -412,6 +412,7 @@ dependencies = [ "csv", "indicatif", "insta", + "insta-cmd", "libc", "nix", "once_cell", @@ -458,11 +459,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console 0.16.6", "once_cell", + "regex", "serde", "similar", + "strip-ansi-escapes", "tempfile", ] +[[package]] +name = "insta-cmd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403" +dependencies = [ + "insta", + "serde", + "serde_json", +] + [[package]] name = "instant" version = "0.1.13" @@ -1024,6 +1038,15 @@ dependencies = [ "rand 0.6.5", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1155,6 +1178,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index dec668a..0ba6a49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,8 @@ features = [ [dev-dependencies] approx = "0.5" assert_cmd = "2.2" -insta = { version = "1.48", features = ["yaml"] } +insta = { version = "1.48", features = ["yaml", "filters"] } +insta-cmd = "0.7" predicates = "3.1" tempfile = "3.27" diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 504f7b9..36295d9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,6 +1,8 @@ mod common; use common::{hyperfine, hyperfine_raw_command}; +use assert_cmd::assert::OutputAssertExt; +use insta_cmd::assert_cmd_snapshot; use predicates::prelude::*; /// Platform-specific I/O utility. @@ -9,20 +11,17 @@ use predicates::prelude::*; /// See: const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" }; -pub fn hyperfine_debug() -> assert_cmd::Command { - let mut cmd = hyperfine(); +pub fn hyperfine_debug() -> std::process::Command { + let mut cmd = hyperfine_raw_command(); cmd.arg("--debug-mode"); cmd } -// Ignore trailing padding on report lines, including the space-only separators. -// Keep indentation and internal spacing so snapshots still check the layout. -fn stdout(output: &assert_cmd::assert::Assert) -> String { - String::from_utf8_lossy(&output.get_output().stdout) - .lines() - .map(str::trim_end) - .collect::>() - .join("\n") +fn snapshot_settings() -> insta::Settings { + let mut settings = insta::Settings::clone_current(); + // Ignore trailing report padding while preserving indentation and internal spacing. + settings.add_filter(r"(?m)[ \t]+$", ""); + settings } #[test] @@ -604,15 +603,15 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb #[test] fn shows_benchmark_comparison_with_relative_times() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 1.0") .arg("sleep 2.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 1.0 Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 1.000 s … 1.000 s 10 runs @@ -629,23 +628,25 @@ fn shows_benchmark_comparison_with_relative_times() { sleep 1.0 ran 2.00 ± 0.00 times faster than sleep 2.0 3.00 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn shows_benchmark_comparison_with_same_time() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--command-name=A") .arg("--command-name=B") .arg("sleep 1.0") .arg("sleep 1.0") .arg("sleep 2.0") - .arg("sleep 1000.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 1000.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: A Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 1.000 s … 1.000 s 10 runs @@ -667,20 +668,22 @@ fn shows_benchmark_comparison_with_same_time() { As fast (1.00 ± 0.00) as B 2.00 ± 0.00 times faster than sleep 2.0 1000.00 ± 0.00 times faster than sleep 1000.0 + + ----- stderr ----- "); } #[test] fn shows_benchmark_comparison_relative_to_reference() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("sleep 1.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2.0 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -697,21 +700,23 @@ fn shows_benchmark_comparison_relative_to_reference() { sleep 2.0 ran 2.00 ± 0.00 times slower than sleep 1.0 1.50 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn shows_reference_name() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 2.0") .arg("--reference-name=refabc123") .arg("sleep 1.0") - .arg("sleep 3.0") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 3.0"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: refabc123 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -728,12 +733,15 @@ fn shows_reference_name() { refabc123 ran 2.00 ± 0.00 times slower than sleep 1.0 1.50 ± 0.00 times faster than sleep 3.0 + + ----- stderr ----- "); } #[test] fn performs_all_benchmarks_in_parameter_scan() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--parameter-scan") .arg("time") @@ -741,11 +749,10 @@ fn performs_all_benchmarks_in_parameter_scan() { .arg("45") .arg("--parameter-step-size") .arg("5") - .arg("sleep {time}") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep {time}"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 30 Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 30.000 s … 30.000 s 10 runs @@ -767,12 +774,15 @@ fn performs_all_benchmarks_in_parameter_scan() { 1.17 ± 0.00 times faster than sleep 35 1.33 ± 0.00 times faster than sleep 40 1.50 ± 0.00 times faster than sleep 45 + + ----- stderr ----- "); } #[test] fn performs_reference_and_all_benchmarks_in_parameter_scan() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("--reference=sleep 25") .arg("--parameter-scan") @@ -781,11 +791,10 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { .arg("45") .arg("--parameter-step-size") .arg("5") - .arg("sleep {time}") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep {time}"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 25 Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 25.000 s … 25.000 s 10 runs @@ -812,26 +821,31 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() { 1.40 ± 0.00 times faster than sleep 35 1.60 ± 0.00 times faster than sleep 40 1.80 ± 0.00 times faster than sleep 45 + + ----- stderr ----- "); } #[test] fn intermediate_results_are_not_exported_to_stdout() { - let output = hyperfine_debug() + let _settings = snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=none") // To only see the Markdown export on stdout .arg("--export-markdown") .arg("-") .arg("sleep 1") - .arg("sleep 2") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("sleep 2"), @r" + success: true + exit_code: 0 + ----- stdout ----- | Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 | | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 | + + + ----- stderr ----- "); } @@ -874,17 +888,17 @@ fn unused_parameters_are_shown_in_benchmark_name() { #[test] fn speed_comparison_sort_order() { + let _settings = snapshot_settings().bind_to_scope(); insta::allow_duplicates! { for sort_order in ["auto", "mean-time"] { - let output = hyperfine_debug() + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") - .arg(format!("--sort={sort_order}")) - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg(format!("--sort={sort_order}")), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -896,19 +910,20 @@ fn speed_comparison_sort_order() { Summary sleep 1 ran 2.00 ± 0.00 times faster than sleep 2 + + ----- stderr ----- "); } } - let output = hyperfine_debug() + assert_cmd_snapshot!(hyperfine_debug() .arg("--style=basic") .arg("sleep 2") .arg("sleep 1") - .arg("--sort=command") - .assert() - .success(); - - insta::assert_snapshot!(stdout(&output), @r" + .arg("--sort=command"), @r" + success: true + exit_code: 0 + ----- stdout ----- Benchmark 1: sleep 2 Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s] Range (min … max): 2.000 s … 2.000 s 10 runs @@ -920,6 +935,8 @@ fn speed_comparison_sort_order() { Relative speed comparison 2.00 ± 0.00 sleep 2 1.00 sleep 1 + + ----- stderr ----- "); }