From ab86cb99cc595f3827c74bb7ae790dfced323540 Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 6 Oct 2026 08:41:59 +0200 Subject: [PATCH] Make --shell=none the default (#982) --- CHANGELOG.md | 7 ++++++ README.md | 43 ++++++++++++++++++++-------------- doc/hyperfine.1 | 20 ++++++++++------ src/cli.rs | 23 +++++++++++------- src/options.rs | 32 ++++++++++++------------- tests/execution_order_tests.rs | 4 +++- tests/integration_tests.rs | 15 ++++++++++++ 7 files changed, 93 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adec262..8088732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Unreleased + +## Breaking changes + +- Execute commands directly by default (`--shell=none`). Use `-S` (an alias for `--shell=default`) to restore the previous behavior (`sh` on Unix, `cmd.exe` on Windows), or select a shell with `--shell `. +- Repurpose `-S` as an alias for `--shell=default`. It no longer accepts a shell argument; replace `-S ` with `--shell `. + # v1.21.0 ## Features diff --git a/README.md b/README.md index b05482f..fa26b64 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,9 @@ A command-line benchmarking tool. ### Basic benchmarks -To run a benchmark, you can simply call `hyperfine ...`. The argument(s) can be any -shell command. For example: +To run a benchmark, you can simply call `hyperfine ...`. Each argument is an executable +and its arguments. Commands run directly by default; use `-S` for shell syntax such as +pipes, redirections, and wildcards. For example: ```sh hyperfine 'sleep 0.3' ``` @@ -53,7 +54,7 @@ by disk caches and whether they are cold or warm. If you want to run the benchmark on a warm cache, you can use the `-w`/`--warmup` option to perform a certain number of program executions before the actual benchmark: ```sh -hyperfine --warmup 3 'grep -R TODO *' +hyperfine -S --warmup 3 'grep -R TODO *' ``` Conversely, if you want to run the benchmark for a cold cache, you can use the `-p`/`--prepare` @@ -65,7 +66,7 @@ sync; echo 3 | sudo tee /proc/sys/vm/drop_caches To use this specific command with hyperfine, call `sudo -v` to temporarily gain sudo permissions and then call: ```sh -hyperfine --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *' +hyperfine -S --prepare 'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' 'grep -R TODO *' ``` ### Parameterized benchmarks @@ -101,28 +102,34 @@ hyperfine \ If you need a unique value for each individual run of a benchmark command, hyperfine also exposes the zero-based `$HYPERFINE_ITERATION` environment variable inside the benchmarked command itself: ```sh -hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log' +hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log' ``` ### Intermediate shell -By default, commands are executed using `sh` on Unix (resolved through `PATH`) or `cmd.exe` on Windows. -If you want to use a different shell, you can use the `-S, --shell ` option: +By default, commands are executed directly, without an intermediate shell (`--shell=none`). +Arguments are split using shell-like quoting, so quoted arguments containing spaces are supported. +Shell syntax such as pipes, redirections, environment-variable expansion, `*`, and `~` is not interpreted. +This avoids shell startup overhead and the noise from correcting for it, especially for fast commands +(< 5 ms). + +To enable shell syntax, use `-S` (an alias for `--shell=default`). This selects `sh` +on Unix (resolved through `PATH`) or `cmd.exe` on Windows: +```sh +hyperfine -S 'sleep 0.1 && echo done' +``` + +You can also select a specific shell with `--shell `: ```sh hyperfine --shell zsh 'for i in {1..10000}; do echo test; done' ``` -Note that hyperfine always *corrects for the shell spawning time*. To do this, it performs a calibration -procedure where it runs the shell with an empty command (multiple times), to measure the startup time -of the shell. It will then subtract this time from the total to show the actual time used by the command -in question. +The shell setting applies to all commands, including `--setup`, `--prepare`, `--conclude`, and `--cleanup`. +If any of these commands need shell syntax, enable a shell explicitly. -If you want to run a benchmark *without an intermediate shell*, you can use the `-N` or `--shell=none` -option. This is helpful for very fast commands (< 5 ms) where the shell startup overhead correction would -produce a significant amount of noise. Note that you cannot use shell syntax like `*` or `~` in this case. -``` -hyperfine -N 'grep -R TODO /home/user' -``` +When a shell is enabled, hyperfine *corrects for the shell spawning time*. It runs the shell with an +empty command multiple times to measure its startup time, then subtracts this time from each +measurement. ### Shell functions @@ -138,7 +145,7 @@ hyperfine --shell=bash my_function Otherwise, inline the function into the benchmarked command: ```sh -hyperfine 'my_function() { sleep 1; }; my_function' +hyperfine -S 'my_function() { sleep 1; }; my_function' ``` ### Exporting results diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index 0136ff8..e986238 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -3,7 +3,7 @@ hyperfine \- command\-line benchmarking tool .SH SYNOPSIS .B hyperfine -.RB [ \-hVN ] +.RB [ \-hVNS ] .RB [ \-\-ignore\-failure[=MODE] ] .RB [ \-\-warmup .IR NUM ] @@ -172,7 +172,7 @@ arithmetic. .RS Example: .RS -\fBhyperfine\fR \fB\-P\fR size 0 3 'sleep $((2**{size}))' +\fBhyperfine\fR \fB\-\-shell=bash\fR \fB\-P\fR size 0 3 'sleep $((2**{size}))' .RE .RE .IP @@ -212,15 +212,20 @@ This performs benchmarks for 'g++ \-O2 main.cpp' and 'clang++ \-O2 main.cpp'. The option can be specified multiple times to run benchmarks for all possible parameter combinations. .HP -\fB\-S\fR, \fB\-\-shell\fR \fISHELL\fP +\fB\-\-shell\fR \fISHELL\fP .IP -Set the shell to use for executing benchmarked commands. This can be +Set the shell to use for executing commands (default: none). This can be the name or the path to the shell executable, or a full command line like "bash \fB\-\-norc\fR". It can also be set to "default" to explicitly -select the default shell on this platform. Finally, this can also be +select the platform shell (sh on Unix, cmd.exe on Windows). Finally, this can also be set to "none" to disable the shell. In this case, commands will be executed directly. They can still have arguments, but more complex things like "sleep 0.1; sleep 0.2" are not possible without a shell. +This option also applies to setup, prepare, conclude, and cleanup commands. +.HP +\fB\-S\fR +.IP +An alias for '\-\-shell=default' (sh on Unix, cmd.exe on Windows). .HP \fB\-N\fR .IP @@ -331,7 +336,8 @@ Write the output to the given file. This option can be specified once for all commands or multiple times, once for each command. Note: If you want to log the output of each and every iteration, you can use a shell redirection and the $HYPERFINE_ITERATION -environment variable: 'my-command > output-${HYPERFINE_ITERATION}.log' +environment variable (requires \fB\-S\fR / \fB\-\-shell=default\fR): +'my-command > output-${HYPERFINE_ITERATION}.log' .HP \fB\-\-input\fR \fIWHERE\fP .IP @@ -377,7 +383,7 @@ Perform a benchmark of 'grep' with a warm disk cache by executing 3 runs up fron of the measurement: .RS .nf -\fBhyperfine\fR \fB\-\-warmup\fR 3 'grep -R TODO *' +\fBhyperfine\fR \fB\-S\fR \fB\-\-warmup\fR 3 'grep -R TODO *' .fi .RE .LP diff --git a/src/cli.rs b/src/cli.rs index 0a38675..245d860 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -36,8 +36,8 @@ fn build_command() -> Command { Arg::new("command") .help("The command to benchmark. This can be the name of an executable, a command \ line like \"grep -i todo\" or a shell command like \"sleep 0.5 && echo test\". \ - The latter is only available if the shell is not explicitly disabled via \ - '--shell=none'. If multiple commands are given, hyperfine will show a \ + The latter requires a shell, which can be enabled via '-S' or '--shell=...'. \ + If multiple commands are given, hyperfine will show a \ comparison of the respective runtimes.") .required(true) .action(ArgAction::Append) @@ -181,7 +181,7 @@ fn build_command() -> Command { Example: hyperfine --prepare 'make clean' -P threads 1 8 'make -j {threads}'\n\n\ This performs benchmarks for 'make -j 1', 'make -j 2', …, 'make -j 8'.\n\n\ To have the value increase following different patterns, use shell arithmetic.\n\n \ - Example: hyperfine -P size 0 3 'sleep $((2**{size}))'\n\n\ + Example: hyperfine --shell=bash -P size 0 3 'sleep $((2**{size}))'\n\n\ This performs benchmarks with power of 2 increases: 'sleep 1', 'sleep 2', 'sleep 4', …\n\ The exact syntax may vary depending on your shell and OS." ), @@ -220,18 +220,25 @@ fn build_command() -> Command { .arg( Arg::new("shell") .long("shell") - .short('S') .action(ArgAction::Set) .value_name("SHELL") .overrides_with("shell") .value_hint(ValueHint::CommandString) - .help("Set the shell to use for executing benchmarked commands. This can be the \ + .help("Set the shell to use for executing commands (default: none). This can be the \ name or the path to the shell executable, or a full command line \ like \"bash --norc\". It can also be set to \"default\" to explicitly select \ - the default shell on this platform. Finally, this can also be set to \ + the platform shell (sh on Unix, cmd.exe on Windows). It can also be set to \ \"none\" to disable the shell. In this case, commands will be executed \ directly. They can still have arguments, but more complex things like \ - \"sleep 0.1; sleep 0.2\" are not possible without a shell.") + \"sleep 0.1; sleep 0.2\" are not possible without a shell. This option also \ + applies to setup, prepare, conclude, and cleanup commands.") + ) + .arg( + Arg::new("default-shell") + .short('S') + .action(ArgAction::SetTrue) + .conflicts_with_all(["shell", "no-shell", "debug-mode"]) + .help("An alias for '--shell=default' (sh on Unix, cmd.exe on Windows).") ) .arg( Arg::new("no-shell") @@ -383,7 +390,7 @@ fn build_command() -> Command { This option can be specified once for all commands or multiple times, once for \ each command. Note: If you want to log the output of each and every iteration, \ you can use a shell redirection and the '$HYPERFINE_ITERATION' environment variable:\n \ - hyperfine 'my-command > output-${HYPERFINE_ITERATION}.log'\n\n", + hyperfine -S 'my-command > output-${HYPERFINE_ITERATION}.log'\n\n", ), ) .arg( diff --git a/src/options.rs b/src/options.rs index f5a3285..781926d 100644 --- a/src/options.rs +++ b/src/options.rs @@ -29,12 +29,6 @@ pub enum Shell { Custom(Vec), } -impl Default for Shell { - fn default() -> Self { - Shell::Default(DEFAULT_SHELL) - } -} - impl fmt::Display for Shell { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -45,6 +39,11 @@ impl fmt::Display for Shell { } impl Shell { + /// Select the platform shell (sh on Unix, cmd.exe on Windows). + pub fn platform_default() -> Self { + Shell::Default(DEFAULT_SHELL) + } + /// Parse given string as shell command line pub fn parse_from_str<'a>(s: &str) -> Result> { let v = shell_words::split(s).map_err(OptionsError::ShellParseError)?; @@ -181,19 +180,14 @@ impl CommandOutputPolicy { } } -#[derive(Debug, PartialEq)] +#[derive(Debug, Default, PartialEq)] pub enum ExecutorKind { + #[default] Raw, Shell(Shell), Mock(Option), } -impl Default for ExecutorKind { - fn default() -> Self { - ExecutorKind::Shell(Shell::default()) - } -} - /// The main settings for a hyperfine benchmark session pub struct Options { /// Upper and lower bound for the number of benchmark runs @@ -401,15 +395,19 @@ impl Options { options.executor_kind = if matches.get_flag("no-shell") { ExecutorKind::Raw + } else if matches.get_flag("default-shell") { + ExecutorKind::Shell(Shell::platform_default()) } else { match ( matches.get_flag("debug-mode"), matches.get_one::("shell"), ) { - (false, Some(shell)) if shell == "default" => ExecutorKind::Shell(Shell::default()), + (false, Some(shell)) if shell == "default" => { + ExecutorKind::Shell(Shell::platform_default()) + } (false, Some(shell)) if shell == "none" => ExecutorKind::Raw, (false, Some(shell)) => ExecutorKind::Shell(Shell::parse_from_str(shell)?), - (false, None) => ExecutorKind::Shell(Shell::default()), + (false, None) => ExecutorKind::default(), (true, Some(shell)) => ExecutorKind::Mock(Some(shell.into())), (true, None) => ExecutorKind::Mock(None), } @@ -532,8 +530,8 @@ impl Options { } #[test] -fn test_default_shell() { - let shell = Shell::default(); +fn test_platform_default_shell() { + let shell = Shell::platform_default(); let s = format!("{shell}"); assert_eq!(&s, DEFAULT_SHELL); diff --git a/tests/execution_order_tests.rs b/tests/execution_order_tests.rs index e09eb63..2acc1b3 100644 --- a/tests/execution_order_tests.rs +++ b/tests/execution_order_tests.rs @@ -17,9 +17,11 @@ impl ExecutionOrderTest { fn new() -> Self { let tempdir = tempdir().unwrap(); let logfile_path = tempdir.path().join("output.log"); + let mut cmd = hyperfine(); + cmd.arg("-S"); ExecutionOrderTest { - cmd: hyperfine(), + cmd, expected_content: String::new(), logfile_path, tempdir, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4b0245a..1daf7bd 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -259,6 +259,7 @@ fn fails_with_duplicate_parameter_names() { #[test] fn fails_for_unknown_command() { hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577") .assert() @@ -298,6 +299,7 @@ fn fails_for_failing_command_without_shell() { #[test] fn fails_for_unknown_setup_command() { hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--setup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577") .arg("echo test") @@ -311,6 +313,7 @@ fn fails_for_unknown_setup_command() { #[test] fn fails_for_unknown_cleanup_command() { hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--cleanup=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577") .arg("echo test") @@ -324,6 +327,7 @@ fn fails_for_unknown_cleanup_command() { #[test] fn fails_for_unknown_prepare_command() { hyperfine() + .arg("--shell=default") .arg("--prepare=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577") .arg("echo test") .assert() @@ -336,6 +340,7 @@ fn fails_for_unknown_prepare_command() { #[test] fn fails_for_unknown_conclude_command() { hyperfine() + .arg("--shell=default") .arg("--conclude=some-nonexisting-program-b5d9574198b7e4b12a71fa4747c0a577") .arg("echo test") .assert() @@ -369,6 +374,7 @@ fn can_run_failing_commands_with_ignore_failure_option() { fn can_ignore_specific_exit_codes() { // Test that specifying exit code 1 ignores it hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1") .arg("exit 1") @@ -377,6 +383,7 @@ fn can_ignore_specific_exit_codes() { // Test that other exit codes still fail hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1") .arg("exit 2") @@ -392,6 +399,7 @@ fn can_ignore_specific_exit_codes() { fn can_ignore_multiple_exit_codes() { // Test that all specified exit codes are ignored hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1,2,3") .arg("exit 1") @@ -399,6 +407,7 @@ fn can_ignore_multiple_exit_codes() { .success(); hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1,2,3") .arg("exit 2") @@ -406,6 +415,7 @@ fn can_ignore_multiple_exit_codes() { .success(); hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1,2,3") .arg("exit 3") @@ -414,6 +424,7 @@ fn can_ignore_multiple_exit_codes() { // Test that other exit codes still fail hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=1,2,3") .arg("exit 4") @@ -429,6 +440,7 @@ fn can_ignore_multiple_exit_codes() { fn ignore_failure_with_all_non_zero() { // Test explicit "all-non-zero" mode hyperfine() + .arg("--shell=default") .arg("--runs=1") .arg("--ignore-failure=all-non-zero") .arg("exit 5") @@ -1229,6 +1241,7 @@ fn speed_comparison_sort_order() { #[test] fn windows_quote_args() { hyperfine() + .arg("--shell=default") .arg("more \"example_input_file.txt\"") .assert() .success(); @@ -1238,6 +1251,7 @@ fn windows_quote_args() { #[test] fn windows_quote_before_quote_args() { hyperfine() + .arg("--shell=default") .arg("dir \"..\\src\\\" \"..\\tests\\\"") .assert() .success(); @@ -1255,6 +1269,7 @@ fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() { let command = |phase| format!(r#"echo {phase}:{iteration} >> "iteration output.txt""#); hyperfine() + .arg("--shell=default") .current_dir(tempdir.path()) .arg("--runs=2") .arg("--warmup=1")