mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-01 00:40:19 +00:00
Use inline snapshots for execution logs and benchmark reports
This commit is contained in:
David Peter committed
1 parent
258af076cb
commit
1e2e0be25c
2 files changed
+517
-343
No files matched your search
+304
-281
@@ -1,4 +1,4 @@
|
||||
use std::{fs::File, io::Read, path::PathBuf};
|
||||
use std::{fs, path::PathBuf};
|
||||
|
||||
use tempfile::{tempdir, TempDir};
|
||||
|
||||
@@ -7,7 +7,6 @@ use common::hyperfine;
|
||||
|
||||
struct ExecutionOrderTest {
|
||||
cmd: assert_cmd::Command,
|
||||
expected_content: String,
|
||||
logfile_path: PathBuf,
|
||||
#[allow(dead_code)]
|
||||
tempdir: TempDir,
|
||||
@@ -20,7 +19,6 @@ impl ExecutionOrderTest {
|
||||
|
||||
ExecutionOrderTest {
|
||||
cmd: hyperfine(),
|
||||
expected_content: String::new(),
|
||||
logfile_path,
|
||||
tempdir,
|
||||
}
|
||||
@@ -69,26 +67,14 @@ impl ExecutionOrderTest {
|
||||
self.command(output)
|
||||
}
|
||||
|
||||
fn expect_output(&mut self, output: &str) -> &mut Self {
|
||||
self.expected_content.push_str(output);
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
self.expected_content.push_str(" \r");
|
||||
}
|
||||
|
||||
self.expected_content.push('\n');
|
||||
self
|
||||
}
|
||||
|
||||
fn run(&mut self) {
|
||||
fn run(&mut self) -> String {
|
||||
self.cmd.assert().success();
|
||||
|
||||
let mut f = File::open(&self.logfile_path).unwrap();
|
||||
let mut content = String::new();
|
||||
f.read_to_string(&mut content).unwrap();
|
||||
|
||||
assert_eq!(content, self.expected_content);
|
||||
let content = fs::read_to_string(&self.logfile_path).unwrap();
|
||||
// cmd.exe's echo includes the space before redirection and uses CRLF.
|
||||
#[cfg(windows)]
|
||||
let content = content.replace(" \r\n", "\n");
|
||||
content
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,167 +86,186 @@ impl Default for ExecutionOrderTest {
|
||||
|
||||
#[test]
|
||||
fn benchmarks_are_executed_sequentially_one() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=1")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benchmarks_are_executed_sequentially() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 1
|
||||
command 2
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_runs_are_executed_before_benchmarking_runs() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.arg("--warmup=3")
|
||||
.command("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 1
|
||||
command 1
|
||||
command 1
|
||||
command 1
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_commands_are_executed_before_each_series_of_timing_runs() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("setup")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("setup")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
command 1
|
||||
command 1
|
||||
setup
|
||||
command 2
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_commands_are_executed_before_each_timing_run() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.prepare("prepare")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 1")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 1")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 2")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
prepare
|
||||
command 1
|
||||
prepare
|
||||
command 1
|
||||
prepare
|
||||
command 2
|
||||
prepare
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conclude_commands_are_executed_after_each_timing_run() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.conclude("conclude")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("command 1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("command 1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("command 2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("command 2")
|
||||
.expect_output("conclude")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
conclude
|
||||
command 1
|
||||
conclude
|
||||
command 2
|
||||
conclude
|
||||
command 2
|
||||
conclude
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_commands_are_executed_before_each_warmup() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=2")
|
||||
.arg("--runs=1")
|
||||
.prepare("prepare")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
// warmup 1
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 1")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 1")
|
||||
// benchmark 1
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 1")
|
||||
// warmup 2
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 2")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 2")
|
||||
// benchmark 2
|
||||
.expect_output("prepare")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
prepare
|
||||
command 1
|
||||
prepare
|
||||
command 1
|
||||
prepare
|
||||
command 1
|
||||
prepare
|
||||
command 2
|
||||
prepare
|
||||
command 2
|
||||
prepare
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conclude_commands_are_executed_after_each_warmup() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=2")
|
||||
.arg("--runs=1")
|
||||
.conclude("conclude")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
// warmup 1
|
||||
.expect_output("command 1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("command 1")
|
||||
.expect_output("conclude")
|
||||
// benchmark 1
|
||||
.expect_output("command 1")
|
||||
.expect_output("conclude")
|
||||
// warmup 2
|
||||
.expect_output("command 2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("command 2")
|
||||
.expect_output("conclude")
|
||||
// benchmark 2
|
||||
.expect_output("command 2")
|
||||
.expect_output("conclude")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
conclude
|
||||
command 1
|
||||
conclude
|
||||
command 1
|
||||
conclude
|
||||
command 2
|
||||
conclude
|
||||
command 2
|
||||
conclude
|
||||
command 2
|
||||
conclude
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_commands_are_executed_once_after_each_benchmark() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.cleanup("cleanup")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("cleanup")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 2")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 1
|
||||
cleanup
|
||||
command 2
|
||||
command 2
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_prepare_cleanup_combined() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=1")
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
@@ -268,30 +273,31 @@ fn setup_prepare_cleanup_combined() {
|
||||
.command("command1")
|
||||
.command("command2")
|
||||
.cleanup("cleanup")
|
||||
// 1
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("cleanup")
|
||||
// 2
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
prepare
|
||||
command1
|
||||
prepare
|
||||
command1
|
||||
prepare
|
||||
command1
|
||||
cleanup
|
||||
setup
|
||||
prepare
|
||||
command2
|
||||
prepare
|
||||
command2
|
||||
prepare
|
||||
command2
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_prepare_conclude_cleanup_combined() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=1")
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
@@ -300,53 +306,57 @@ fn setup_prepare_conclude_cleanup_combined() {
|
||||
.command("command2")
|
||||
.conclude("conclude")
|
||||
.cleanup("cleanup")
|
||||
// 1
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("cleanup")
|
||||
// 2
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
cleanup
|
||||
setup
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_parameter_value() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.arg("--parameter-list")
|
||||
.arg("number")
|
||||
.arg("1,2,3")
|
||||
.command("command {number}")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 3")
|
||||
.expect_output("command 3")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 1
|
||||
command 2
|
||||
command 2
|
||||
command 3
|
||||
command 3
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_parameter_values() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.arg("--parameter-list")
|
||||
.arg("number")
|
||||
@@ -355,70 +365,82 @@ fn multiple_parameter_values() {
|
||||
.arg("letter")
|
||||
.arg("a,b")
|
||||
.command("command {number} {letter}")
|
||||
.expect_output("command 1 a")
|
||||
.expect_output("command 1 a")
|
||||
.expect_output("command 2 a")
|
||||
.expect_output("command 2 a")
|
||||
.expect_output("command 3 a")
|
||||
.expect_output("command 3 a")
|
||||
.expect_output("command 1 b")
|
||||
.expect_output("command 1 b")
|
||||
.expect_output("command 2 b")
|
||||
.expect_output("command 2 b")
|
||||
.expect_output("command 3 b")
|
||||
.expect_output("command 3 b")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1 a
|
||||
command 1 a
|
||||
command 2 a
|
||||
command 2 a
|
||||
command 3 a
|
||||
command 3 a
|
||||
command 1 b
|
||||
command 1 b
|
||||
command 2 b
|
||||
command 2 b
|
||||
command 3 b
|
||||
command 3 b
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_is_executed_first() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=1")
|
||||
.reference("reference")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("reference")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
reference
|
||||
command 1
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_is_executed_first_parameter_value() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=2")
|
||||
.reference("reference")
|
||||
.arg("--parameter-list")
|
||||
.arg("number")
|
||||
.arg("1,2,3")
|
||||
.command("command {number}")
|
||||
.expect_output("reference")
|
||||
.expect_output("reference")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 2")
|
||||
.expect_output("command 3")
|
||||
.expect_output("command 3")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
reference
|
||||
reference
|
||||
command 1
|
||||
command 1
|
||||
command 2
|
||||
command 2
|
||||
command 3
|
||||
command 3
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reference_is_executed_separately_from_commands() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--runs=1")
|
||||
.reference("command 1")
|
||||
.command("command 1")
|
||||
.command("command 2")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 1")
|
||||
.expect_output("command 2")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
command 1
|
||||
command 1
|
||||
command 2
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_prepare_reference_conclude_cleanup_combined() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=1")
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
@@ -428,48 +450,48 @@ fn setup_prepare_reference_conclude_cleanup_combined() {
|
||||
.command("command2")
|
||||
.conclude("conclude")
|
||||
.cleanup("cleanup")
|
||||
// reference
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("reference")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("reference")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("reference")
|
||||
.expect_output("conclude")
|
||||
.expect_output("cleanup")
|
||||
// 1
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude")
|
||||
.expect_output("cleanup")
|
||||
// 2
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("prepare")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
prepare
|
||||
reference
|
||||
conclude
|
||||
prepare
|
||||
reference
|
||||
conclude
|
||||
prepare
|
||||
reference
|
||||
conclude
|
||||
cleanup
|
||||
setup
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
prepare
|
||||
command1
|
||||
conclude
|
||||
cleanup
|
||||
setup
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
prepare
|
||||
command2
|
||||
conclude
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_separate_prepare_separate_conclude_cleanup_combined() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=1")
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
@@ -480,36 +502,37 @@ fn setup_separate_prepare_separate_conclude_cleanup_combined() {
|
||||
.prepare("prepare2")
|
||||
.command("command2")
|
||||
.conclude("conclude2")
|
||||
// 1
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("cleanup")
|
||||
// 2
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
cleanup
|
||||
setup
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
|
||||
ExecutionOrderTest::new()
|
||||
let output = ExecutionOrderTest::new()
|
||||
.arg("--warmup=1")
|
||||
.arg("--runs=2")
|
||||
.setup("setup")
|
||||
@@ -523,41 +546,41 @@ fn setup_separate_prepare_reference_separate_conclude_cleanup_combined() {
|
||||
.prepare("prepare2")
|
||||
.command("command2")
|
||||
.conclude("conclude2")
|
||||
// reference
|
||||
.expect_output("setup")
|
||||
.expect_output("prepareref")
|
||||
.expect_output("reference")
|
||||
.expect_output("concluderef")
|
||||
.expect_output("prepareref")
|
||||
.expect_output("reference")
|
||||
.expect_output("concluderef")
|
||||
.expect_output("prepareref")
|
||||
.expect_output("reference")
|
||||
.expect_output("concluderef")
|
||||
.expect_output("cleanup")
|
||||
// 1
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("prepare1")
|
||||
.expect_output("command1")
|
||||
.expect_output("conclude1")
|
||||
.expect_output("cleanup")
|
||||
// 2
|
||||
.expect_output("setup")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("prepare2")
|
||||
.expect_output("command2")
|
||||
.expect_output("conclude2")
|
||||
.expect_output("cleanup")
|
||||
.run();
|
||||
|
||||
insta::assert_snapshot!(output, @r"
|
||||
setup
|
||||
prepareref
|
||||
reference
|
||||
concluderef
|
||||
prepareref
|
||||
reference
|
||||
concluderef
|
||||
prepareref
|
||||
reference
|
||||
concluderef
|
||||
cleanup
|
||||
setup
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
prepare1
|
||||
command1
|
||||
conclude1
|
||||
cleanup
|
||||
setup
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
prepare2
|
||||
command2
|
||||
conclude2
|
||||
cleanup
|
||||
");
|
||||
}
|
||||
+213
-62
@@ -15,6 +15,16 @@ pub fn hyperfine_debug() -> assert_cmd::Command {
|
||||
cmd
|
||||
}
|
||||
|
||||
// Ignore trailing padding on report lines, including the space-only separators.
|
||||
// Keep indentation and internal spacing so snapshots still check the layout.
|
||||
fn stdout(output: &assert_cmd::assert::Assert) -> String {
|
||||
String::from_utf8_lossy(&output.get_output().stdout)
|
||||
.lines()
|
||||
.map(str::trim_end)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runs_successfully() {
|
||||
hyperfine()
|
||||
@@ -594,21 +604,38 @@ fn takes_both_preparation_and_conclusion_command_into_account_for_computing_numb
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_relative_times() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times faster")
|
||||
.and(predicate::str::contains("3.00 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1.0 ran
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
3.00 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_with_same_time() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--command-name=A")
|
||||
.arg("--command-name=B")
|
||||
.arg("sleep 1.0")
|
||||
@@ -616,43 +643,98 @@ fn shows_benchmark_comparison_with_same_time() {
|
||||
.arg("sleep 2.0")
|
||||
.arg("sleep 1000.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("As fast (1.00 ± 0.00) as")
|
||||
.and(predicate::str::contains("2.00 ± 0.00 times faster"))
|
||||
.and(predicate::str::contains("1000.00 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: A
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 2: B
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 1000.0
|
||||
Time (mean ± σ): 1000.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1000.000 s … 1000.000 s 10 runs
|
||||
|
||||
Summary
|
||||
A ran
|
||||
As fast (1.00 ± 0.00) as B
|
||||
2.00 ± 0.00 times faster than sleep 2.0
|
||||
1000.00 ± 0.00 times faster than sleep 1000.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_benchmark_comparison_relative_to_reference() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("2.00 ± 0.00 times slower")
|
||||
.and(predicate::str::contains("1.50 ± 0.00 times faster")),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2.0
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 2.0 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shows_reference_name() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 2.0")
|
||||
.arg("--reference-name=refabc123")
|
||||
.arg("sleep 1.0")
|
||||
.arg("sleep 3.0")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains("Benchmark 1: refabc123"));
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: refabc123
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1.0
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 3.0
|
||||
Time (mean ± σ): 3.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 3.000 s … 3.000 s 10 runs
|
||||
|
||||
Summary
|
||||
refabc123 ran
|
||||
2.00 ± 0.00 times slower than sleep 1.0
|
||||
1.50 ± 0.00 times faster than sleep 3.0
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
.arg("30")
|
||||
@@ -661,19 +743,37 @@ fn performs_all_benchmarks_in_parameter_scan() {
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 30")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 50").not()),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 30 ran
|
||||
1.17 ± 0.00 times faster than sleep 35
|
||||
1.33 ± 0.00 times faster than sleep 40
|
||||
1.50 ± 0.00 times faster than sleep 45
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("--reference=sleep 25")
|
||||
.arg("--parameter-scan")
|
||||
.arg("time")
|
||||
@@ -683,31 +783,56 @@ fn performs_reference_and_all_benchmarks_in_parameter_scan() {
|
||||
.arg("5")
|
||||
.arg("sleep {time}")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Benchmark 1: sleep 25")
|
||||
.and(predicate::str::contains("Benchmark 2: sleep 30"))
|
||||
.and(predicate::str::contains("Benchmark 3: sleep 35"))
|
||||
.and(predicate::str::contains("Benchmark 4: sleep 40"))
|
||||
.and(predicate::str::contains("Benchmark 5: sleep 45"))
|
||||
.and(predicate::str::contains("Benchmark 6: sleep 50").not()),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 25
|
||||
Time (mean ± σ): 25.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 25.000 s … 25.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 30
|
||||
Time (mean ± σ): 30.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 30.000 s … 30.000 s 10 runs
|
||||
|
||||
Benchmark 3: sleep 35
|
||||
Time (mean ± σ): 35.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 35.000 s … 35.000 s 10 runs
|
||||
|
||||
Benchmark 4: sleep 40
|
||||
Time (mean ± σ): 40.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 40.000 s … 40.000 s 10 runs
|
||||
|
||||
Benchmark 5: sleep 45
|
||||
Time (mean ± σ): 45.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 45.000 s … 45.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 25 ran
|
||||
1.20 ± 0.00 times faster than sleep 30
|
||||
1.40 ± 0.00 times faster than sleep 35
|
||||
1.60 ± 0.00 times faster than sleep 40
|
||||
1.80 ± 0.00 times faster than sleep 45
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_results_are_not_exported_to_stdout() {
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=none") // To only see the Markdown export on stdout
|
||||
.arg("--export-markdown")
|
||||
.arg("-")
|
||||
.arg("sleep 1")
|
||||
.arg("sleep 2")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
(predicate::str::contains("sleep 1").count(1))
|
||||
.and(predicate::str::contains("sleep 2").count(1)),
|
||||
);
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `sleep 1` | 1.000 ± 0.000 | 1.000 | 1.000 | 1.00 |
|
||||
| `sleep 2` | 2.000 ± 0.000 | 2.000 | 2.000 | 2.00 ± 0.00 |
|
||||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -749,27 +874,53 @@ fn unused_parameters_are_shown_in_benchmark_name() {
|
||||
|
||||
#[test]
|
||||
fn speed_comparison_sort_order() {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
hyperfine_debug()
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}"))
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"sleep 1 ran\n 2.00 ± 0.00 times faster than sleep 2",
|
||||
));
|
||||
insta::allow_duplicates! {
|
||||
for sort_order in ["auto", "mean-time"] {
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg(format!("--sort={sort_order}"))
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Summary
|
||||
sleep 1 ran
|
||||
2.00 ± 0.00 times faster than sleep 2
|
||||
");
|
||||
}
|
||||
}
|
||||
|
||||
hyperfine_debug()
|
||||
let output = hyperfine_debug()
|
||||
.arg("--style=basic")
|
||||
.arg("sleep 2")
|
||||
.arg("sleep 1")
|
||||
.arg("--sort=command")
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(predicate::str::contains(
|
||||
"2.00 ± 0.00 sleep 2\n 1.00 sleep 1",
|
||||
));
|
||||
.success();
|
||||
|
||||
insta::assert_snapshot!(stdout(&output), @r"
|
||||
Benchmark 1: sleep 2
|
||||
Time (mean ± σ): 2.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 2.000 s … 2.000 s 10 runs
|
||||
|
||||
Benchmark 2: sleep 1
|
||||
Time (mean ± σ): 1.000 s ± 0.000 s [User: 0.000 s, System: 0.000 s]
|
||||
Range (min … max): 1.000 s … 1.000 s 10 runs
|
||||
|
||||
Relative speed comparison
|
||||
2.00 ± 0.00 sleep 2
|
||||
1.00 sleep 1
|
||||
");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
|
||||
Reference in new issue
Block a user