Add --conclude option with tests

This implementation of --conclude is basically identical
to the existing implementation for --prepare. I have not
adapted any existing tests. All tests which test --conclude
functionality are new.

The --conclude commands will run in the same order in which
they are supplied on the cli in the same manner as --prepare
and there must either be one --conclude command, or else the
same number of --concludes as there are commands.
This commit is contained in:
Jack O'Connor authored and David Peter committed 2024-02-02 17:46:23 +01:00
1 parent 91388e7302
commit e13d0cb144
5 files changed
+239 -2

No files matched your search

+88
View File
@@ -54,6 +54,11 @@ impl ExecutionOrderTest {
self.command(output)
}
fn conclude(&mut self, output: &str) -> &mut Self {
self.arg("--conclude");
self.command(output)
}
fn cleanup(&mut self, output: &str) -> &mut Self {
self.arg("--cleanup");
self.command(output)
@@ -160,6 +165,24 @@ fn prepare_commands_are_executed_before_each_timing_run() {
.run();
}
#[test]
fn conclude_commands_are_executed_after_each_timing_run() {
ExecutionOrderTest::new()
.arg("--runs=2")
.conclude("conclude")
.command("command 1")
.command("command 2")
.expect_output("command 1")
.expect_output("conclude")
.expect_output("command 1")
.expect_output("conclude")
.expect_output("command 2")
.expect_output("conclude")
.expect_output("command 2")
.expect_output("conclude")
.run();
}
#[test]
fn prepare_commands_are_executed_before_each_warmup() {
ExecutionOrderTest::new()
@@ -187,6 +210,33 @@ fn prepare_commands_are_executed_before_each_warmup() {
.run();
}
#[test]
fn conclude_commands_are_executed_after_each_warmup() {
ExecutionOrderTest::new()
.arg("--warmup=2")
.arg("--runs=1")
.conclude("conclude")
.command("command 1")
.command("command 2")
// warmup 1
.expect_output("command 1")
.expect_output("conclude")
.expect_output("command 1")
.expect_output("conclude")
// benchmark 1
.expect_output("command 1")
.expect_output("conclude")
// warmup 2
.expect_output("command 2")
.expect_output("conclude")
.expect_output("command 2")
.expect_output("conclude")
// benchmark 2
.expect_output("command 2")
.expect_output("conclude")
.run();
}
#[test]
fn cleanup_commands_are_executed_once_after_each_benchmark() {
ExecutionOrderTest::new()
@@ -234,6 +284,44 @@ fn setup_prepare_cleanup_combined() {
.run();
}
#[test]
fn setup_prepare_conclude_cleanup_combined() {
ExecutionOrderTest::new()
.arg("--warmup=1")
.arg("--runs=2")
.setup("setup")
.prepare("prepare")
.command("command1")
.command("command2")
.conclude("conclude")
.cleanup("cleanup")
// 1
.expect_output("setup")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command1")
.expect_output("conclude")
.expect_output("cleanup")
// 2
.expect_output("setup")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("prepare")
.expect_output("command2")
.expect_output("conclude")
.expect_output("cleanup")
.run();
}
#[test]
fn single_parameter_value() {
ExecutionOrderTest::new()
+79
View File
@@ -75,6 +75,31 @@ fn fails_with_wrong_number_of_prepare_options() {
));
}
#[test]
fn fails_with_wrong_number_of_conclude_options() {
hyperfine()
.arg("--runs=1")
.arg("--conclude=echo a")
.arg("--conclude=echo b")
.arg("echo a")
.arg("echo b")
.assert()
.success();
hyperfine()
.arg("--runs=1")
.arg("--conclude=echo a")
.arg("--conclude=echo b")
.arg("echo a")
.arg("echo b")
.arg("echo c")
.assert()
.failure()
.stderr(predicate::str::contains(
"The '--conclude' option has to be provided",
));
}
#[test]
fn fails_with_duplicate_parameter_names() {
hyperfine()
@@ -167,6 +192,18 @@ fn fails_for_unknown_prepare_command() {
));
}
#[test]
fn fails_for_unknown_conclude_command() {
hyperfine()
.arg("--conclude=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577")
.arg("echo test")
.assert()
.failure()
.stderr(predicate::str::contains(
"The conclusion command terminated with a non-zero exit code.",
));
}
#[cfg(unix)]
#[test]
fn can_run_failing_commands_with_ignore_failure_option() {
@@ -320,6 +357,48 @@ fn takes_preparation_command_into_account_for_computing_number_of_runs() {
.stdout(predicate::str::contains("30 runs"));
}
#[test]
fn takes_conclusion_command_into_account_for_computing_number_of_runs() {
hyperfine_debug()
.arg("--conclude=sleep 0.02")
.arg("sleep 0.01")
.assert()
.success()
.stdout(predicate::str::contains("100 runs"));
// Shell overhead needs to be added to both the conclude command and the actual command,
// leading to a total benchmark time of (cmd + shell + conclude + shell = 0.1 s)
hyperfine_debug()
.arg("--shell=sleep 0.01")
.arg("--conclude=sleep 0.03")
.arg("sleep 0.05")
.assert()
.success()
.stdout(predicate::str::contains("30 runs"));
}
#[test]
fn takes_both_preparation_and_conclusion_command_into_account_for_computing_number_of_runs() {
hyperfine_debug()
.arg("--prepare=sleep 0.01")
.arg("--conclude=sleep 0.01")
.arg("sleep 0.01")
.assert()
.success()
.stdout(predicate::str::contains("100 runs"));
// Shell overhead needs to be added to both the prepare, conclude and the actual command,
// leading to a total benchmark time of (prepare + shell + cmd + shell + conclude + shell = 0.1 s)
hyperfine_debug()
.arg("--shell=sleep 0.01")
.arg("--prepare=sleep 0.01")
.arg("--conclude=sleep 0.01")
.arg("sleep 0.05")
.assert()
.success()
.stdout(predicate::str::contains("30 runs"));
}
#[test]
fn shows_benchmark_comparison_with_relative_times() {
hyperfine_debug()