Fix prepare and conclude behaviour and add tests.

src/benchmark/relative_speed.rs

@@ -155,6 +155,20 @@ fn test_compute_relative_speed() {
     assert_relative_eq!(2.5, annotated_results[2].relative_speed);
 }
 
+#[test]
+fn test_compute_relative_speed_with_reference() {
+    use approx::assert_relative_eq;
+
+    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
+    let reference = create_result("cmd2", 4.0);
+
+    let annotated_results =
+        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();
+
+    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
+    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
+}
+
 #[test]
 fn test_compute_relative_speed_for_zero_times() {
     let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];

src/options.rs

@@ -437,21 +437,25 @@ impl Options {
     }
 
     pub fn validate_against_command_list(&self, commands: &Commands) -> Result<()> {
+        let num_commands = commands.num_commands()
+            + if self.reference_command.is_some() {
+                1
+            } else {
+                0
+            };
         if let Some(preparation_command) = &self.preparation_command {
             ensure!(
-                preparation_command.len() <= 1
+                preparation_command.len() <= 1 || num_commands == preparation_command.len(),
-                    || commands.num_commands() == preparation_command.len(),
                 "The '--prepare' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 
         if let Some(conclusion_command) = &self.conclusion_command {
             ensure!(
-                conclusion_command.len() <= 1
+                conclusion_command.len() <= 1 || num_commands == conclusion_command.len(),
-                    || commands.num_commands() == conclusion_command.len(),
                 "The '--conclude' option has to be provided just once or N times, where N is the \
-             number of benchmark commands."
+             number of benchmark commands including a potential reference."
             );
         }
 

tests/execution_order_tests.rs

@@ -54,6 +54,11 @@ impl ExecutionOrderTest {
         self.command(output)
     }
 
+    fn reference(&mut self, output: &str) -> &mut Self {
+        self.arg("--reference");
+        self.command(output)
+    }
+
     fn conclude(&mut self, output: &str) -> &mut Self {
         self.arg("--conclude");
         self.command(output)
@@ -364,3 +369,195 @@ fn multiple_parameter_values() {
         .expect_output("command 3 b")
         .run();
 }
+
+#[test]
+fn reference_is_executed_first() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("reference")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_first_parameter_value() {
+    ExecutionOrderTest::new()
+        .arg("--runs=2")
+        .reference("reference")
+        .arg("--parameter-list")
+        .arg("number")
+        .arg("1,2,3")
+        .command("command {number}")
+        .expect_output("reference")
+        .expect_output("reference")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .expect_output("command 2")
+        .expect_output("command 3")
+        .expect_output("command 3")
+        .run();
+}
+
+#[test]
+fn reference_is_executed_separately_from_commands() {
+    ExecutionOrderTest::new()
+        .arg("--runs=1")
+        .reference("command 1")
+        .command("command 1")
+        .command("command 2")
+        .expect_output("command 1")
+        .expect_output("command 1")
+        .expect_output("command 2")
+        .run();
+}
+
+#[test]
+fn setup_prepare_reference_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .prepare("prepare")
+        .reference("reference")
+        .command("command1")
+        .command("command2")
+        .conclude("conclude")
+        .cleanup("cleanup")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("reference")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command1")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("prepare")
+        .expect_output("command2")
+        .expect_output("conclude")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separete_prepare_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}
+
+#[test]
+fn setup_separete_prepare_reference_separate_conclude_cleanup_combined() {
+    ExecutionOrderTest::new()
+        .arg("--warmup=1")
+        .arg("--runs=2")
+        .setup("setup")
+        .cleanup("cleanup")
+        .prepare("prepareref")
+        .reference("reference")
+        .conclude("concluderef")
+        .prepare("prepare1")
+        .command("command1")
+        .conclude("conclude1")
+        .prepare("prepare2")
+        .command("command2")
+        .conclude("conclude2")
+        // reference
+        .expect_output("setup")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("prepareref")
+        .expect_output("reference")
+        .expect_output("concluderef")
+        .expect_output("cleanup")
+        // 1
+        .expect_output("setup")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("prepare1")
+        .expect_output("command1")
+        .expect_output("conclude1")
+        .expect_output("cleanup")
+        // 2
+        .expect_output("setup")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("prepare2")
+        .expect_output("command2")
+        .expect_output("conclude2")
+        .expect_output("cleanup")
+        .run();
+}

tests/integration_tests.rs

@@ -61,6 +61,17 @@ fn fails_with_wrong_number_of_prepare_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo ref")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--prepare=echo a")
@@ -73,6 +84,19 @@ fn fails_with_wrong_number_of_prepare_options() {
         .stderr(predicate::str::contains(
             "The '--prepare' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--prepare=echo a")
+        .arg("--prepare=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--prepare' option has to be provided",
+        ));
 }
 
 #[test]
@@ -86,6 +110,17 @@ fn fails_with_wrong_number_of_conclude_options() {
         .assert()
         .success();
 
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo ref")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .success();
+
     hyperfine()
         .arg("--runs=1")
         .arg("--conclude=echo a")
@@ -98,6 +133,19 @@ fn fails_with_wrong_number_of_conclude_options() {
         .stderr(predicate::str::contains(
             "The '--conclude' option has to be provided",
         ));
+
+    hyperfine()
+        .arg("--runs=1")
+        .arg("--conclude=echo a")
+        .arg("--conclude=echo b")
+        .arg("--reference=echo ref")
+        .arg("echo a")
+        .arg("echo b")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "The '--conclude' option has to be provided",
+        ));
 }
 
 #[test]
@@ -413,6 +461,20 @@ fn shows_benchmark_comparison_with_relative_times() {
         );
 }
 
+#[test]
+fn shows_benchmark_comparison_relative_to_reference() {
+    hyperfine_debug()
+        .arg("--reference=sleep 2.0")
+        .arg("sleep 1.0")
+        .arg("sleep 3.0")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("2.00 ± 0.00 times slower")
+                .and(predicate::str::contains("1.50 ± 0.00 times faster")),
+        );
+}
+
 #[test]
 fn performs_all_benchmarks_in_parameter_scan() {
     hyperfine_debug()
@@ -434,6 +496,29 @@ fn performs_all_benchmarks_in_parameter_scan() {
         );
 }
 
+#[test]
+fn performs_reference_and_all_benchmarks_in_parameter_scan() {
+    hyperfine_debug()
+        .arg("--reference=sleep 25")
+        .arg("--parameter-scan")
+        .arg("time")
+        .arg("30")
+        .arg("45")
+        .arg("--parameter-step-size")
+        .arg("5")
+        .arg("sleep {time}")
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Benchmark 1: sleep 25")
+                .and(predicate::str::contains("Benchmark 2: sleep 30"))
+                .and(predicate::str::contains("Benchmark 3: sleep 35"))
+                .and(predicate::str::contains("Benchmark 4: sleep 40"))
+                .and(predicate::str::contains("Benchmark 5: sleep 45"))
+                .and(predicate::str::contains("Benchmark 6: sleep 50").not()),
+        );
+}
+
 #[test]
 fn intermediate_results_are_not_exported_to_stdout() {
     hyperfine_debug()