diff --git a/README.md b/README.md index a4d4a29..8e24014 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log' ### Intermediate shell -By default, commands are executed directly, without an intermediate shell (`--shell=none`). +By default, benchmarked commands, including warmup runs, are executed directly, without an intermediate shell (`--shell=none`). Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported. Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted. This avoids shell startup overhead and the noise from correcting for it, especially for fast commands @@ -125,10 +125,11 @@ You can also select a specific shell with `--shell `: hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' ``` -The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`. -If any of these commands need shell syntax, enable a shell explicitly. +The `--setup`, `--prepare`, `--conclude`, and `--cleanup` commands always run through a shell. +They use the shell selected by `--shell`, or the platform shell (`sh` on Unix, `cmd.exe` on Windows) +when `--shell` is omitted or set to `none`. -When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an +When benchmarked commands use a shell, hyperfine *corrects for the shell spawning time*. It runs the shell with an empty command multiple times to measure its startup time, then subtracts this time from each time measurement. diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index e2b8308..f91b656 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -194,14 +194,16 @@ possible parameter combinations. .HP \fB\-\-shell\fR \fISHELL\fP .IP -Set the shell to use for executing commands (default: none). This can be -the name or the path to the shell executable, or a full command line +Set the shell to use for executing benchmarked commands, including warmup runs +(default: none). This can be the name or the path to the shell executable, +or a full command line like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be -set to "none" to disable the shell. In this case, commands will be +set to "none" to disable the shell. In this case, benchmarked commands will be executed directly. They can still have arguments, but more complex things like "sleep 0.1; sleep 0.2" are not possible without a shell. -This option also applies to setup, prepare, conclude, and cleanup commands. +Setup, prepare, conclude, and cleanup commands always use a shell: the selected +shell, or the platform shell when this option is omitted or set to "none". .HP \fB\-S\fR .IP diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index a0e584c..cc24d88 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -5,9 +5,7 @@ use std::process::ExitStatus; use crate::benchmark::measurement::Measurement; use crate::benchmark::measurement::Measurements; use crate::command::Command; -use crate::options::{ - CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, -}; +use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell}; use crate::output::progress_bar::get_progress_bar; use crate::quantity::{second, Information, Time, Zero}; use crate::timer::execute_and_measure; @@ -42,6 +40,21 @@ pub trait Executor { output_policy: &CommandOutputPolicy, ) -> Result; + /// Run a lifecycle command through a shell, always treating failures as errors. + fn run_intermediate_command_and_measure( + &self, + command: &Command<'_>, + iteration: BenchmarkIteration, + output_policy: &CommandOutputPolicy, + ) -> Result { + self.run_command_and_measure( + command, + iteration, + Some(CmdFailureAction::RaiseError), + output_policy, + ) + } + /// Perform a calibration of this executor. For example, /// when running commands through a shell, we need to /// measure the shell spawning time separately in order @@ -58,13 +71,13 @@ pub trait Executor { fn run_command_and_measure_common( mut command: std::process::Command, iteration: BenchmarkIteration, - command_failure_action: CmdFailureAction, - command_input_policy: &CommandInputPolicy, + command_failure_action: Option, + options: &Options, command_output_policy: &CommandOutputPolicy, command_name: &str, collect_hardware_counters: bool, ) -> Result { - let stdin = command_input_policy.get_stdin()?; + let stdin = options.command_input_policy.get_stdin()?; let (stdout, stderr) = command_output_policy.get_stdout_stderr()?; command.stdin(stdin).stdout(stdout).stderr(stderr); @@ -83,10 +96,14 @@ fn run_command_and_measure_common( if !measurement.exit_status.success() { use crate::util::exit_code::extract_exit_code; - let should_fail = match command_failure_action { + let show_ignore_failure_hint = command_failure_action.is_none(); + let should_fail = match command_failure_action + .as_ref() + .unwrap_or(&options.command_failure_action) + { CmdFailureAction::RaiseError => true, CmdFailureAction::IgnoreAllFailures => false, - CmdFailureAction::IgnoreSpecificFailures(ref codes) => { + CmdFailureAction::IgnoreSpecificFailures(codes) => { // Only fail if the exit code is not in the list of codes to ignore if let Some(exit_code) = extract_exit_code(measurement.exit_status) { !codes.contains(&exit_code) @@ -105,15 +122,17 @@ fn run_command_and_measure_common( BenchmarkIteration::Benchmark(0) => "the first benchmark run".to_string(), BenchmarkIteration::Benchmark(i) => format!("benchmark iteration {i}"), }; - bail!( - "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ - Alternatively, use the '--show-output' option to debug what went wrong.", - cause=measurement.exit_status.code().map_or( - "The process has been terminated by a signal".into(), - |c| format!("Command terminated with non-zero exit code {c}") - - ), + let cause = measurement.exit_status.code().map_or_else( + || "was terminated by a signal".to_owned(), + |code| format!("terminated with non-zero exit code {code}"), ); + let hint = if show_ignore_failure_hint { + " Use the '-i'/'--ignore-failure' option if you want to ignore this. \ + Alternatively, use the '--show-output' option to debug what went wrong." + } else { + " Use the '--show-output' option to debug what went wrong." + }; + bail!("Command '{command_name}' {cause} in {when}.{hint}"); } } @@ -141,14 +160,26 @@ impl Executor for RawExecutor<'_> { run_command_and_measure_common( command.get_command()?, iteration, - command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), - &self.options.command_input_policy, + command_failure_action, + self.options, output_policy, &command.get_command_line(), matches!(iteration, BenchmarkIteration::Benchmark(_)), ) } + fn run_intermediate_command_and_measure( + &self, + command: &Command<'_>, + iteration: BenchmarkIteration, + output_policy: &CommandOutputPolicy, + ) -> Result { + // Keep shell startup in the hook's duration: raw benchmarks have no + // calibrated shell overhead to add back when estimating the run count. + ShellExecutor::new(&Shell::platform_default(), self.options) + .run_intermediate_command_and_measure(command, iteration, output_policy) + } + fn calibrate(&mut self) -> Result<()> { Ok(()) } @@ -197,8 +228,8 @@ impl Executor for ShellExecutor<'_> { let mut measurement = run_command_and_measure_common( command_builder, iteration, - command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), - &self.options.command_input_policy, + command_failure_action, + self.options, output_policy, &command.get_command_line(), false, diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index d0dce9d..251b1f1 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -11,9 +11,7 @@ use crate::benchmark::benchmark_result::Parameter; use crate::benchmark::executor::BenchmarkIteration; use crate::benchmark::measurement::{Measurement, Measurements}; use crate::command::Command; -use crate::options::{ - CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, -}; +use crate::options::{CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption}; use crate::outlier_detection::OUTLIER_THRESHOLD; use crate::output::console_writeln; use crate::output::progress_bar::{ @@ -55,22 +53,24 @@ impl<'a> Benchmark<'a> { } } - /// Run setup, cleanup, or preparation commands + /// Run setup, cleanup, preparation, or conclusion commands fn run_intermediate_command( &self, command: &Command<'_>, - error_output: &'static str, + kind: &'static str, output_policy: &CommandOutputPolicy, iteration: executor::BenchmarkIteration, ) -> Result { self.executor - .run_command_and_measure( - command, - iteration, - Some(CmdFailureAction::RaiseError), - output_policy, - ) - .map_err(|_| anyhow!(error_output)) + .run_intermediate_command_and_measure(command, iteration, output_policy) + .map_err(|error| { + let hint = if cfg!(windows) { + "Append ' || exit /b 0' to the command if this failure can be ignored." + } else { + "Append ' || true' to the command if this failure can be ignored." + }; + anyhow!("The {kind} command failed: {error:#} {hint}") + }) } /// Run the command specified by `--setup`. @@ -85,14 +85,11 @@ impl<'a> Benchmark<'a> { .as_ref() .map(|setup_command| Command::new_parametrized(None, setup_command, parameters)); - let error_output = "The setup command terminated with a non-zero exit code. \ - Append ' || true' to the command if you are sure that this can be ignored."; - Ok(command .map(|cmd| { self.run_intermediate_command( &cmd, - error_output, + "setup", output_policy, BenchmarkIteration::NonBenchmarkRun, ) @@ -113,14 +110,11 @@ impl<'a> Benchmark<'a> { .as_ref() .map(|cleanup_command| Command::new_parametrized(None, cleanup_command, parameters)); - let error_output = "The cleanup command terminated with a non-zero exit code. \ - Append ' || true' to the command if you are sure that this can be ignored."; - Ok(command .map(|cmd| { self.run_intermediate_command( &cmd, - error_output, + "cleanup", output_policy, BenchmarkIteration::NonBenchmarkRun, ) @@ -136,10 +130,7 @@ impl<'a> Benchmark<'a> { output_policy: &CommandOutputPolicy, iteration: executor::BenchmarkIteration, ) -> Result { - let error_output = "The preparation command terminated with a non-zero exit code. \ - Append ' || true' to the command if you are sure that this can be ignored."; - - self.run_intermediate_command(command, error_output, output_policy, iteration) + self.run_intermediate_command(command, "preparation", output_policy, iteration) } /// Run the command specified by `--conclude`. @@ -149,10 +140,7 @@ impl<'a> Benchmark<'a> { output_policy: &CommandOutputPolicy, iteration: executor::BenchmarkIteration, ) -> Result { - let error_output = "The conclusion command terminated with a non-zero exit code. \ - Append ' || true' to the command if you are sure that this can be ignored."; - - self.run_intermediate_command(command, error_output, output_policy, iteration) + self.run_intermediate_command(command, "conclusion", output_policy, iteration) } fn validate_measurement(&self, measurement: &Measurement) -> Result<()> { diff --git a/src/cli.rs b/src/cli.rs index 712aa0b..e611e14 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -236,14 +236,15 @@ fn build_command() -> Command { .value_name("SHELL") .overrides_with("shell") .value_hint(ValueHint::CommandString) - .help("Set the shell to use for executing commands (default: none). This can be the \ - name or the path to the shell executable, or a full command line \ + .help("Set the shell to use for executing benchmarked commands, including warmup runs \ + (default: none). This can be the name or the path to the shell executable, or a full command line \ like \"bash --norc\". It can also be set to \"default\" to explicitly select \ the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \ - \"none\" to disable the shell. In this case, commands will be executed \ + \"none\" to disable the shell. In this case, benchmarked commands will be executed \ directly. They can still have arguments, but more complex things like \ - \"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \ - applies to setup, prepare, conclude, and cleanup commands.") + \"sleep 0.1; sleep 0.2\" are not possible without a shell. Setup, prepare, conclude, \ + and cleanup commands always use a shell: the selected shell, or the platform \ + shell when this option is omitted or set to \"none\".") ) .arg( Arg::new("default-shell") diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2950d50..529f84b 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -275,7 +275,7 @@ fn fails_for_unknown_command() { .assert() .failure() .stderr(predicate::str::contains( - "Command terminated with non-zero exit code", + "Command 'some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577' terminated with non-zero exit code", )); } @@ -298,12 +298,15 @@ fn fails_for_failing_command_without_shell() { hyperfine() .arg("--shell=none") .arg("--runs=1") - .arg("false") + .arg("--command-name=named benchmark") + .args(["--parameter-list", "status", "0,1"]) + .arg("test {status} -eq 0") .assert() .failure() .stderr(predicate::str::contains( - "Command terminated with non-zero exit code", - )); + "Command 'test 1 -eq 0' terminated with non-zero exit code 1 in the first benchmark run.", + )) + .stderr(predicate::str::contains("--ignore-failure")); } #[test] @@ -315,9 +318,7 @@ fn fails_for_unknown_setup_command() { .arg("echo test") .assert() .failure() - .stderr(predicate::str::contains( - "The setup command terminated with a non-zero exit code.", - )); + .stderr(predicate::str::contains("The setup command failed:")); } #[test] @@ -329,9 +330,7 @@ fn fails_for_unknown_cleanup_command() { .arg("echo test") .assert() .failure() - .stderr(predicate::str::contains( - "The cleanup command terminated with a non-zero exit code.", - )); + .stderr(predicate::str::contains("The cleanup command failed:")); } #[test] @@ -342,9 +341,7 @@ fn fails_for_unknown_prepare_command() { .arg("echo test") .assert() .failure() - .stderr(predicate::str::contains( - "The preparation command terminated with a non-zero exit code.", - )); + .stderr(predicate::str::contains("The preparation command failed:")); } #[test] @@ -355,9 +352,7 @@ fn fails_for_unknown_conclude_command() { .arg("echo test") .assert() .failure() - .stderr(predicate::str::contains( - "The conclusion command terminated with a non-zero exit code.", - )); + .stderr(predicate::str::contains("The conclusion command failed:")); } #[cfg(unix)] @@ -368,7 +363,7 @@ fn can_run_failing_commands_with_ignore_failure_option() { .assert() .failure() .stderr(predicate::str::contains( - "Command terminated with non-zero exit code", + "Command 'false' terminated with non-zero exit code", )); hyperfine() @@ -400,7 +395,7 @@ fn can_ignore_specific_exit_codes() { .assert() .failure() .stderr(predicate::str::contains( - "Command terminated with non-zero exit code 2", + "Command 'exit 2' terminated with non-zero exit code 2", )); } @@ -441,7 +436,7 @@ fn can_ignore_multiple_exit_codes() { .assert() .failure() .stderr(predicate::str::contains( - "Command terminated with non-zero exit code 4", + "Command 'exit 4' terminated with non-zero exit code 4", )); } @@ -477,9 +472,14 @@ fn runs_commands_using_user_defined_shell() { .arg("--show-output") .arg("--shell") .arg("echo 'custom_shell' '--shell-arg'") + .arg("--setup=echo setup") + .arg("--prepare=echo prepare") + .arg("--conclude=echo conclude") + .arg("--cleanup=echo cleanup") .arg("echo benchmark") .assert() .success() + .stdout(predicate::str::contains("custom_shell --shell-arg").count(5)) .stdout( predicate::str::contains("custom_shell --shell-arg -c echo benchmark").or( predicate::str::contains("custom_shell --shell-arg /C echo benchmark"),