Implement '--parameter-range var min max', see #19

src/hyperfine/benchmark.rs

@@ -11,7 +11,7 @@ use hyperfine::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
 use hyperfine::timer::{TimerStart, TimerStop};
 use hyperfine::timer::wallclocktimer::WallClockTimer;
 use hyperfine::shell::execute_and_time;
-use hyperfine::types::BenchmarkResult;
+use hyperfine::types::{BenchmarkResult, Command};
 
 /// Results from timing a single shell command
 #[derive(Debug, Default, Copy, Clone)]
@@ -37,13 +37,13 @@ fn subtract_shell_spawning_time(time: Second, shell_spawning_time: Second) -> Se
 
 /// Run the given shell command and measure the execution time
 pub fn time_shell_command(
-    shell_cmd: &str,
+    command: &Command,
     failure_action: CmdFailureAction,
     shell_spawning_time: Option<TimingResult>,
 ) -> io::Result<(TimingResult, bool)> {
     let wallclock_timer = WallClockTimer::start();
 
-    let result = execute_and_time(shell_cmd)?;
+    let result = execute_and_time(&command.get_shell_command())?;
 
     let mut time_user = result.user_time;
     let mut time_system = result.system_time;
@@ -86,7 +86,7 @@ pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result<TimingR
 
     for _ in 0..COUNT {
         // Just run the shell without any command
-        let res = time_shell_command("", CmdFailureAction::RaiseError, None);
+        let res = time_shell_command(&Command::new(""), CmdFailureAction::RaiseError, None);
 
         match res {
             Err(_) => {
@@ -116,7 +116,11 @@ pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result<TimingR
 /// Run the command specified by `--prepare`.
 fn run_preparation_command(command: &Option<String>) -> io::Result<TimingResult> {
     if let &Some(ref preparation_command) = command {
-        let res = time_shell_command(preparation_command, CmdFailureAction::RaiseError, None);
+        let res = time_shell_command(
+            &Command::new(preparation_command),
+            CmdFailureAction::RaiseError,
+            None,
+        );
         if res.is_err() {
             return Err(io::Error::new(
                 io::ErrorKind::Other,
@@ -134,7 +138,7 @@ fn run_preparation_command(command: &Option<String>) -> io::Result<TimingResult>
 /// Run the benchmark for a single shell command
 pub fn run_benchmark(
     num: usize,
-    cmd: &str,
+    cmd: &Command,
     shell_spawning_time: TimingResult,
     options: &HyperfineOptions,
 ) -> io::Result<BenchmarkResult> {
@@ -295,7 +299,7 @@ pub fn run_benchmark(
     println!(" ");
 
     Ok(BenchmarkResult::new(
-        cmd.to_string(),
+        cmd.get_shell_command(),
         t_mean,
         t_stddev,
         user_mean,

src/hyperfine/error.rs

@@ -0,0 +1,38 @@
+use std::num;
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug)]
+pub enum ParameterRangeError {
+    ParseIntError(num::ParseIntError),
+    EmptyRange,
+    TooLarge,
+}
+
+impl ParameterRangeError {
+    fn __description(&self) -> &str {
+        match *self {
+            ParameterRangeError::ParseIntError(ref e) => e.description(),
+            ParameterRangeError::EmptyRange => "Empty parameter range",
+            ParameterRangeError::TooLarge => "Parameter range is too large",
+        }
+    }
+}
+
+impl From<num::ParseIntError> for ParameterRangeError {
+    fn from(e: num::ParseIntError) -> ParameterRangeError {
+        ParameterRangeError::ParseIntError(e)
+    }
+}
+
+impl fmt::Display for ParameterRangeError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.__description())
+    }
+}
+
+impl Error for ParameterRangeError {
+    fn description(&self) -> &str {
+        self.__description()
+    }
+}

src/hyperfine/mod.rs

@@ -7,3 +7,4 @@ pub mod shell;
 pub mod timer;
 pub mod types;
 pub mod warnings;
+pub mod error;

src/hyperfine/types.rs

@@ -1,9 +1,52 @@
-/// The types module contains common internal types for the application.
-///
+/// This module contains common internal types.
+
+use std::fmt;
 
 /// Type alias for unit of time
 pub type Second = f64;
 
+/// A command that should be benchmarked.
+#[derive(Debug, Clone)]
+pub struct Command<'a> {
+    /// The command that should be executed (without parameter substitution)
+    expression: &'a str,
+
+    /// A possible parameter value.
+    parameter: Option<(&'a str, i32)>,
+}
+
+impl<'a> Command<'a> {
+    pub fn new(expression: &'a str) -> Command<'a> {
+        Command {
+            expression: expression,
+            parameter: None,
+        }
+    }
+
+    pub fn new_parametrized(expression: &'a str, parameter: &'a str, value: i32) -> Command<'a> {
+        Command {
+            expression: expression,
+            parameter: Some((parameter, value)),
+        }
+    }
+
+    pub fn get_shell_command(&self) -> String {
+        match self.parameter {
+            Some((param_name, param_value)) => self.expression.replace(
+                &format!("{{{param_name}}}", param_name = param_name),
+                &param_value.to_string(),
+            ),
+            None => self.expression.into(),
+        }
+    }
+}
+
+impl<'a> fmt::Display for Command<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.get_shell_command())
+    }
+}
+
 /// Action to take when an executed command fails.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum CmdFailureAction {

src/main.rs

@@ -30,6 +30,7 @@ extern crate approx;
 use std::cmp;
 use std::error::Error;
 use std::io;
+use std::ops::Range;
 
 use atty::Stream;
 use colored::*;
@@ -38,9 +39,11 @@ use clap::{App, AppSettings, Arg};
 mod hyperfine;
 
 use hyperfine::internal::write_benchmark_comparison;
-use hyperfine::types::{BenchmarkResult, CmdFailureAction, HyperfineOptions, OutputStyleOption};
+use hyperfine::types::{BenchmarkResult, CmdFailureAction, Command, HyperfineOptions,
+                       OutputStyleOption};
 use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark};
 use hyperfine::export::{ExportManager, ExportType};
+use hyperfine::error::ParameterRangeError;
 
 /// Print error message to stderr and terminate
 pub fn error(message: &str) -> ! {
@@ -49,7 +52,7 @@ pub fn error(message: &str) -> ! {
 }
 
 /// Runs the benchmark for the given commands
-fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result<Vec<BenchmarkResult>> {
+fn run(commands: &Vec<Command>, options: &HyperfineOptions) -> io::Result<Vec<BenchmarkResult>> {
     let shell_spawning_time = mean_shell_spawning_time(&options.output_style)?;
 
     let mut timing_results = vec![];
@@ -62,6 +65,26 @@ fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result<Vec<Bench
     Ok(timing_results)
 }
 
+/// A function to read the `--parameter-range` arguments
+fn parse_parameter_range_args<'a>(
+    mut vals: clap::Values<'a>,
+) -> Result<(&'a str, Range<i32>), ParameterRangeError> {
+    let param_name = vals.next().unwrap();
+    let param_min: i32 = vals.next().unwrap().parse()?;
+    let param_max: i32 = vals.next().unwrap().parse()?;
+
+    const MAX_PARAMETERS: i32 = 100000;
+    if param_max - param_min > MAX_PARAMETERS {
+        return Err(ParameterRangeError::TooLarge);
+    }
+
+    if param_max < param_min {
+        return Err(ParameterRangeError::EmptyRange);
+    }
+
+    return Ok((param_name, param_min..(param_max + 1)));
+}
+
 fn main() {
     // Process command line options
     let mut options = HyperfineOptions::default();
@@ -139,6 +162,17 @@ fn main() {
                 .short("i")
                 .help("Ignore non-zero exit codes."),
         )
+        .arg(
+            Arg::with_name("parameter-range")
+                .long("parameter-range")
+                .short("r")
+                .help(
+                    "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \
+                     string '{VAR}' in each command by the current parameter value.",
+                )
+                .takes_value(true)
+                .value_names(&["VAR", "MIN", "MAX"]),
+        )
         .arg(
             Arg::with_name("export-csv")
                 .long("export-csv")
@@ -213,7 +247,26 @@ fn main() {
         options.failure_action = CmdFailureAction::Ignore;
     }
 
-    let commands = matches.values_of("command").unwrap().collect();
+    let command_strings = matches.values_of("command").unwrap();
+
+    let commands = if let Some(args) = matches.values_of("parameter-range") {
+        match parse_parameter_range_args(args) {
+            Ok((param_name, param_range)) => {
+                let mut commands = vec![];
+                let command_strings = command_strings.collect::<Vec<&str>>();
+                for value in param_range.start..param_range.end {
+                    for ref cmd in &command_strings {
+                        commands.push(Command::new_parametrized(cmd, param_name, value));
+                    }
+                }
+                commands
+            }
+            Err(e) => error(e.description()),
+        }
+    } else {
+        command_strings.map(|c| Command::new(c)).collect()
+    };
+
     let res = run(&commands, &options);
 
     match res {