Use insta-cmd for benchmark report snapshots

Cargo.lock

@@ -412,6 +412,7 @@ dependencies = [
  "csv",
  "indicatif",
  "insta",
+ "insta-cmd",
  "libc",
  "nix",
  "once_cell",
@@ -458,11 +459,24 @@ checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82"
 dependencies = [
  "console 0.16.6",
  "once_cell",
+ "regex",
  "serde",
  "similar",
+ "strip-ansi-escapes",
  "tempfile",
 ]
 
+[[package]]
+name = "insta-cmd"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bffdf4af1db390cf0401535d7c1303cd079a074d28d8473b026fdb6559c41403"
+dependencies = [
+ "insta",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.13"
@@ -1024,6 +1038,15 @@ dependencies = [
  "rand 0.6.5",
 ]
 
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
+dependencies = [
+ "vte",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -1155,6 +1178,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "vte"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wait-timeout"
 version = "0.2.1"

Cargo.toml

@@ -64,7 +64,8 @@ features = [
 [dev-dependencies]
 approx = "0.5"
 assert_cmd = "2.2"
-insta = { version = "1.48", features = ["yaml"] }
+insta = { version = "1.48", features = ["yaml", "filters"] }
+insta-cmd = "0.7"
 predicates = "3.1"
 tempfile = "3.27"
 

tests/integration_tests.rs

@@ -1,6 +1,8 @@
 mod common;
 use common::{hyperfine, hyperfine_raw_command};
 
+use assert_cmd::assert::OutputAssertExt;
+use insta_cmd::assert_cmd_snapshot;
 use predicates::prelude::*;
 
 /// Platform-specific I/O utility.
@@ -9,20 +11,17 @@ use predicates::prelude::*;
 ///   See: <https://superuser.com/questions/853580/real-windows-equivalent-to-cat-stdin>
 const STDIN_READ_COMMAND: &str = if cfg!(windows) { "findstr x*" } else { "cat" };
 
-pub fn hyperfine_debug() -> assert_cmd::Command {
-    let mut cmd = hyperfine();
+pub fn hyperfine_debug() -> std::process::Command {
+    let mut cmd = hyperfine_raw_command();
     cmd.arg("--debug-mode");
     cmd
 }
 
-// Ignore trailing padding on report lines, including the space-only separators.
-// Keep indentation and internal spacing so snapshots still check the layout.
-fn stdout(output: &assert_cmd::assert::Assert) -> String {
-    String::from_utf8_lossy(&output.get_output().stdout)
-        .lines()
-        .map(str::trim_end)
-        .collect::<Vec<_>>()
-        .join("\n")
+fn snapshot_settings() -> insta::Settings {
+    let mut settings = insta::Settings::clone_current();
+    // Ignore trailing report padding while preserving indentation and internal spacing.
+    settings.add_filter(r"(?m)[ \t]+$", "");
+    settings
 }
 
 #[test]
@@ -604,15 +603,15 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
 
 #[test]
 fn shows_benchmark_comparison_with_relative_times() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("sleep 1.0")
         .arg("sleep 2.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: sleep 1.0
       Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):    1.000 s …  1.000 s    10 runs
@@ -629,23 +628,25 @@ fn shows_benchmark_comparison_with_relative_times() {
       sleep 1.0 ran
         2.00 ± 0.00 times faster than sleep 2.0
         3.00 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn shows_benchmark_comparison_with_same_time() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("--command-name=A")
         .arg("--command-name=B")
         .arg("sleep 1.0")
         .arg("sleep 1.0")
         .arg("sleep 2.0")
-        .arg("sleep 1000.0")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep 1000.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: A
       Time (mean ± σ):      1.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):    1.000 s …  1.000 s    10 runs
@@ -667,20 +668,22 @@ fn shows_benchmark_comparison_with_same_time() {
         As fast (1.00 ± 0.00) as B
         2.00 ± 0.00 times faster than sleep 2.0
      1000.00 ± 0.00 times faster than sleep 1000.0
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn shows_benchmark_comparison_relative_to_reference() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("--reference=sleep 2.0")
         .arg("sleep 1.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: sleep 2.0
       Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):    2.000 s …  2.000 s    10 runs
@@ -697,21 +700,23 @@ fn shows_benchmark_comparison_relative_to_reference() {
       sleep 2.0 ran
         2.00 ± 0.00 times slower than sleep 1.0
         1.50 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn shows_reference_name() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("--reference=sleep 2.0")
         .arg("--reference-name=refabc123")
         .arg("sleep 1.0")
-        .arg("sleep 3.0")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep 3.0"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: refabc123
       Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):    2.000 s …  2.000 s    10 runs
@@ -728,12 +733,15 @@ fn shows_reference_name() {
       refabc123 ran
         2.00 ± 0.00 times slower than sleep 1.0
         1.50 ± 0.00 times faster than sleep 3.0
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn performs_all_benchmarks_in_parameter_scan() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("--parameter-scan")
         .arg("time")
@@ -741,11 +749,10 @@ fn performs_all_benchmarks_in_parameter_scan() {
         .arg("45")
         .arg("--parameter-step-size")
         .arg("5")
-        .arg("sleep {time}")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep {time}"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: sleep 30
       Time (mean ± σ):     30.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):   30.000 s … 30.000 s    10 runs
@@ -767,12 +774,15 @@ fn performs_all_benchmarks_in_parameter_scan() {
         1.17 ± 0.00 times faster than sleep 35
         1.33 ± 0.00 times faster than sleep 40
         1.50 ± 0.00 times faster than sleep 45
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn performs_reference_and_all_benchmarks_in_parameter_scan() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("--reference=sleep 25")
         .arg("--parameter-scan")
@@ -781,11 +791,10 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
         .arg("45")
         .arg("--parameter-step-size")
         .arg("5")
-        .arg("sleep {time}")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep {time}"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: sleep 25
       Time (mean ± σ):     25.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):   25.000 s … 25.000 s    10 runs
@@ -812,26 +821,31 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
         1.40 ± 0.00 times faster than sleep 35
         1.60 ± 0.00 times faster than sleep 40
         1.80 ± 0.00 times faster than sleep 45
+
+    ----- stderr -----
     ");
 }
 
 #[test]
 fn intermediate_results_are_not_exported_to_stdout() {
-    let output = hyperfine_debug()
+    let _settings = snapshot_settings().bind_to_scope();
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=none") // To only see the Markdown export on stdout
         .arg("--export-markdown")
         .arg("-")
         .arg("sleep 1")
-        .arg("sleep 2")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("sleep 2"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
 
     | Command | Mean [s] | Min [s] | Max [s] | Relative |
     |:---|---:|---:|---:|---:|
     | `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
     | `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
+
+
+    ----- stderr -----
     ");
 }
 
@@ -874,17 +888,17 @@ fn unused_parameters_are_shown_in_benchmark_name() {
 
 #[test]
 fn speed_comparison_sort_order() {
+    let _settings = snapshot_settings().bind_to_scope();
     insta::allow_duplicates! {
         for sort_order in ["auto", "mean-time"] {
-            let output = hyperfine_debug()
+            assert_cmd_snapshot!(hyperfine_debug()
                 .arg("--style=basic")
                 .arg("sleep 2")
                 .arg("sleep 1")
-                .arg(format!("--sort={sort_order}"))
-                .assert()
-                .success();
-
-            insta::assert_snapshot!(stdout(&output), @r"
+                .arg(format!("--sort={sort_order}")), @r"
+            success: true
+            exit_code: 0
+            ----- stdout -----
             Benchmark 1: sleep 2
               Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
               Range (min … max):    2.000 s …  2.000 s    10 runs
@@ -896,19 +910,20 @@ fn speed_comparison_sort_order() {
             Summary
               sleep 1 ran
                 2.00 ± 0.00 times faster than sleep 2
+
+            ----- stderr -----
             ");
         }
     }
 
-    let output = hyperfine_debug()
+    assert_cmd_snapshot!(hyperfine_debug()
         .arg("--style=basic")
         .arg("sleep 2")
         .arg("sleep 1")
-        .arg("--sort=command")
-        .assert()
-        .success();
-
-    insta::assert_snapshot!(stdout(&output), @r"
+        .arg("--sort=command"), @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
     Benchmark 1: sleep 2
       Time (mean ± σ):      2.000 s ±  0.000 s    [User: 0.000 s, System: 0.000 s]
       Range (min … max):    2.000 s …  2.000 s    10 runs
@@ -920,6 +935,8 @@ fn speed_comparison_sort_order() {
     Relative speed comparison
             2.00 ±  0.00  sleep 2
             1.00          sleep 1
+
+    ----- stderr -----
     ");
 }