diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a7e3dd..a2add03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,14 @@ ## Features +- Added `--setup` (`s`) option that can be used to run `make all` or + similar. It runs once per set of tests, like `--cleanup` (`c`). + ## Changes +- The `-s` short option for `--style` has been usurped by the new + `--setup` option. + ## Bugfixes ## Other diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index e4a51ad..30e2bfd 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -62,6 +62,12 @@ Perform at most \fImaxruns\fP (number) runs for each command. Default: no limit. Perform exactly \fIruns\fP (number) runs for each command. If this option is not specified, \fBhyperfine\fR automatically determines the number of runs. .HP +\fB\-s\fR, \fB\-\-setup\fR \fIcmd\fP +.IP +Execute \fIcmd\fP once before each set of timing runs. This is useful +for compiling software or doing other one-off setup. +The \fB\-\-setup\fR option can only be specified once. +.HP \fB\-p\fR, \fB\-\-prepare\fR \fIcmd...\fP .IP Execute \fIcmd\fP before each timing run. This is useful for clearing disk caches, @@ -119,14 +125,15 @@ Example: .IP This performs benchmarks for 'gcc \-O2 main.cpp' and 'clang \-O2 main.cpp'. .HP -\fB\-s\fR, \fB\-\-style\fR \fItype\fP +\fB\-\-style\fR \fItype\fP .IP Set output style \fItype\fP (default: auto). Set this to 'basic' to disable output coloring and interactive elements. Set it to 'full' to enable all effects even if no interactive terminal was detected. Set this to 'nocolor' to keep the interactive output without any colors. Set this to 'color' to keep the colors without any interactive output. Set this to 'none' to disable all the output -of the tool. +of the tool. In hyperfine versions v0.4.0..v1.12.0 this option took the \fB\-s\fR, +short option, which is now used by \fB\--setup\fR. .HP \fB\-S\fR, \fB\-\-shell\fR \fIshell\fP .IP @@ -207,6 +214,18 @@ Export the results of a parameter scan benchmark to a markdown table: \fBhyperfine\fR \fB\-\-export\-markdown\fR output.md \fB\-\-parameter-scan\fR time 1 5 'sleep {time}' .fi .RE +.LP +Demonstrate when each of \fB\-\-setup\fR, \fB\-\-prepare\fR, \fIcmd\fP and \fB\-\-cleanup\fR will run: +.RS +.nf +\fBhyperfine\fR \fB\-L\fR n 1,2 \fB\-r\fR 2 \fB\-\-show-output\fR \\ + \fB\-\-setup\fR 'echo setup n={n}' \\ + \fB\-\-prepare\fR 'echo prepare={n}' \\ + \fB\-\-cleanup\fR 'echo cleanup n={n}' \\ + 'echo command n={n}' +.fi +.RE +.RE .SH AUTHOR .LP David Peter (sharkdp) diff --git a/src/app.rs b/src/app.rs index f1cf577..b720ee1 100644 --- a/src/app.rs +++ b/src/app.rs @@ -73,6 +73,20 @@ fn build_app() -> App<'static, 'static> { .help("Perform exactly NUM runs for each command. If this option is not specified, \ hyperfine automatically determines the number of runs."), ) + .arg( + Arg::with_name("setup") + .long("setup") + .short("s") + .takes_value(true) + .number_of_values(1) + .value_name("CMD") + .help( + "Execute CMD before each set of timing runs. This is useful for \ + compiling your software with the provided parameters, or to do any \ + other work that should happen once before a series of benchmark runs,\ + not every time as would happen with the --prepare option." + ), + ) .arg( Arg::with_name("prepare") .long("prepare") @@ -155,7 +169,6 @@ fn build_app() -> App<'static, 'static> { .arg( Arg::with_name("style") .long("style") - .short("s") .takes_value(true) .value_name("TYPE") .possible_values(&["auto", "basic", "full", "nocolor", "color", "none"]) diff --git a/src/benchmark.rs b/src/benchmark.rs index 301443c..8a8856f 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -185,6 +185,18 @@ fn run_intermediate_command( }) } +/// Run the command specified by `--setup`. +fn run_setup_command( + shell: &Shell, + command: &Option>, + show_output: bool, +) -> io::Result { + let error_output = "The setup command terminated with a non-zero exit code. \ + Append ' || true' to the command if you are sure that this can be ignored."; + + run_intermediate_command(shell, command, show_output, error_output) +} + /// Run the command specified by `--prepare`. fn run_preparation_command( shell: &Shell, @@ -263,6 +275,12 @@ pub fn run_benchmark( Command::new_parametrized(None, preparation_command, cmd.get_parameters().clone()) }); + // Run setup command + let setup_cmd = options.setup_command.as_ref().map(|setup_command| { + Command::new_parametrized(None, setup_command, cmd.get_parameters().clone()) + }); + run_setup_command(&options.shell, &setup_cmd, options.show_output)?; + // Warmup phase if options.warmup_count > 0 { let progress_bar = if options.output_style != OutputStyleOption::Disabled { diff --git a/src/main.rs b/src/main.rs index 6db752a..4a21018 100644 --- a/src/main.rs +++ b/src/main.rs @@ -193,6 +193,8 @@ fn build_hyperfine_options<'a>( (None, None) => {} }; + options.setup_command = matches.value_of("setup").map(String::from); + options.preparation_command = matches .values_of("prepare") .map(|values| values.map(String::from).collect::>()); diff --git a/src/options.rs b/src/options.rs index 20706b7..3f933dd 100644 --- a/src/options.rs +++ b/src/options.rs @@ -114,6 +114,9 @@ pub struct HyperfineOptions { /// Whether or not to ignore non-zero exit codes pub failure_action: CmdFailureAction, + /// Command to run before each batch of timing runs + pub setup_command: Option, + /// Command to run before each timing run pub preparation_command: Option>, @@ -146,6 +149,7 @@ impl Default for HyperfineOptions { runs: Runs::default(), min_time_sec: 3.0, failure_action: CmdFailureAction::RaiseError, + setup_command: None, preparation_command: None, cleanup_command: None, output_style: OutputStyleOption::Full, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ae502fc..9c8229c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -70,6 +70,11 @@ impl ExecutionOrderTest { self } + fn setup(&mut self, output: &str) -> &mut Self { + self.arg("--setup"); + self.command(output) + } + fn prepare(&mut self, output: &str) -> &mut Self { self.arg("--prepare"); self.command(output) @@ -147,6 +152,22 @@ fn warmup_runs_are_executed_before_benchmarking_runs() { .run(); } +#[test] +fn setup_commands_are_executed_before_each_series_of_timing_runs() { + ExecutionOrderTest::new() + .arg("--runs=2") + .setup("setup") + .command("command 1") + .command("command 2") + .expect_output("setup") + .expect_output("command 1") + .expect_output("command 1") + .expect_output("setup") + .expect_output("command 2") + .expect_output("command 2") + .run(); +} + #[test] fn prepare_commands_are_executed_before_each_timing_run() { ExecutionOrderTest::new() @@ -181,6 +202,23 @@ fn cleanup_commands_are_executed_once_after_each_benchmark() { .run(); } +#[test] +fn setup_prepare_cleanup_combined() { + ExecutionOrderTest::new() + .arg("--runs=2") + .setup("make") + .prepare("make testclean") + .command("make test") + .cleanup("make clean") + .expect_output("make") + .expect_output("make testclean") + .expect_output("make test") + .expect_output("make testclean") + .expect_output("make test") + .expect_output("make clean") + .run(); +} + #[test] fn single_parameter_value() { ExecutionOrderTest::new()