From e862cf8567ea1c6d9172d1d62a1db911c2affccf Mon Sep 17 00:00:00 2001 From: sharkdp Date: Sat, 24 Mar 2018 15:46:28 +0100 Subject: [PATCH] Implement '--parameter-range var min max', see #19 --- src/hyperfine/benchmark.rs | 18 +++++++----- src/hyperfine/error.rs | 38 ++++++++++++++++++++++++ src/hyperfine/mod.rs | 1 + src/hyperfine/types.rs | 47 ++++++++++++++++++++++++++++-- src/main.rs | 59 ++++++++++++++++++++++++++++++++++++-- 5 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 src/hyperfine/error.rs diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index 43a5802..b617d45 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -11,7 +11,7 @@ use hyperfine::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use hyperfine::timer::{TimerStart, TimerStop}; use hyperfine::timer::wallclocktimer::WallClockTimer; use hyperfine::shell::execute_and_time; -use hyperfine::types::BenchmarkResult; +use hyperfine::types::{BenchmarkResult, Command}; /// Results from timing a single shell command #[derive(Debug, Default, Copy, Clone)] @@ -37,13 +37,13 @@ fn subtract_shell_spawning_time(time: Second, shell_spawning_time: Second) -> Se /// Run the given shell command and measure the execution time pub fn time_shell_command( - shell_cmd: &str, + command: &Command, failure_action: CmdFailureAction, shell_spawning_time: Option, ) -> io::Result<(TimingResult, bool)> { let wallclock_timer = WallClockTimer::start(); - let result = execute_and_time(shell_cmd)?; + let result = execute_and_time(&command.get_shell_command())?; let mut time_user = result.user_time; let mut time_system = result.system_time; @@ -86,7 +86,7 @@ pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result { @@ -116,7 +116,11 @@ pub fn mean_shell_spawning_time(style: &OutputStyleOption) -> io::Result) -> io::Result { if let &Some(ref preparation_command) = command { - let res = time_shell_command(preparation_command, CmdFailureAction::RaiseError, None); + let res = time_shell_command( + &Command::new(preparation_command), + CmdFailureAction::RaiseError, + None, + ); if res.is_err() { return Err(io::Error::new( io::ErrorKind::Other, @@ -134,7 +138,7 @@ fn run_preparation_command(command: &Option) -> io::Result /// Run the benchmark for a single shell command pub fn run_benchmark( num: usize, - cmd: &str, + cmd: &Command, shell_spawning_time: TimingResult, options: &HyperfineOptions, ) -> io::Result { @@ -295,7 +299,7 @@ pub fn run_benchmark( println!(" "); Ok(BenchmarkResult::new( - cmd.to_string(), + cmd.get_shell_command(), t_mean, t_stddev, user_mean, diff --git a/src/hyperfine/error.rs b/src/hyperfine/error.rs new file mode 100644 index 0000000..66fca9b --- /dev/null +++ b/src/hyperfine/error.rs @@ -0,0 +1,38 @@ +use std::num; +use std::error::Error; +use std::fmt; + +#[derive(Debug)] +pub enum ParameterRangeError { + ParseIntError(num::ParseIntError), + EmptyRange, + TooLarge, +} + +impl ParameterRangeError { + fn __description(&self) -> &str { + match *self { + ParameterRangeError::ParseIntError(ref e) => e.description(), + ParameterRangeError::EmptyRange => "Empty parameter range", + ParameterRangeError::TooLarge => "Parameter range is too large", + } + } +} + +impl From for ParameterRangeError { + fn from(e: num::ParseIntError) -> ParameterRangeError { + ParameterRangeError::ParseIntError(e) + } +} + +impl fmt::Display for ParameterRangeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.__description()) + } +} + +impl Error for ParameterRangeError { + fn description(&self) -> &str { + self.__description() + } +} diff --git a/src/hyperfine/mod.rs b/src/hyperfine/mod.rs index 74e34a1..9b601c1 100644 --- a/src/hyperfine/mod.rs +++ b/src/hyperfine/mod.rs @@ -7,3 +7,4 @@ pub mod shell; pub mod timer; pub mod types; pub mod warnings; +pub mod error; diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs index 1ae672b..74126ff 100644 --- a/src/hyperfine/types.rs +++ b/src/hyperfine/types.rs @@ -1,9 +1,52 @@ -/// The types module contains common internal types for the application. -/// +/// This module contains common internal types. + +use std::fmt; /// Type alias for unit of time pub type Second = f64; +/// A command that should be benchmarked. +#[derive(Debug, Clone)] +pub struct Command<'a> { + /// The command that should be executed (without parameter substitution) + expression: &'a str, + + /// A possible parameter value. + parameter: Option<(&'a str, i32)>, +} + +impl<'a> Command<'a> { + pub fn new(expression: &'a str) -> Command<'a> { + Command { + expression: expression, + parameter: None, + } + } + + pub fn new_parametrized(expression: &'a str, parameter: &'a str, value: i32) -> Command<'a> { + Command { + expression: expression, + parameter: Some((parameter, value)), + } + } + + pub fn get_shell_command(&self) -> String { + match self.parameter { + Some((param_name, param_value)) => self.expression.replace( + &format!("{{{param_name}}}", param_name = param_name), + ¶m_value.to_string(), + ), + None => self.expression.into(), + } + } +} + +impl<'a> fmt::Display for Command<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.get_shell_command()) + } +} + /// Action to take when an executed command fails. #[derive(Debug, Clone, Copy, PartialEq)] pub enum CmdFailureAction { diff --git a/src/main.rs b/src/main.rs index 27dfa8d..37ab0cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ extern crate approx; use std::cmp; use std::error::Error; use std::io; +use std::ops::Range; use atty::Stream; use colored::*; @@ -38,9 +39,11 @@ use clap::{App, AppSettings, Arg}; mod hyperfine; use hyperfine::internal::write_benchmark_comparison; -use hyperfine::types::{BenchmarkResult, CmdFailureAction, HyperfineOptions, OutputStyleOption}; +use hyperfine::types::{BenchmarkResult, CmdFailureAction, Command, HyperfineOptions, + OutputStyleOption}; use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark}; use hyperfine::export::{ExportManager, ExportType}; +use hyperfine::error::ParameterRangeError; /// Print error message to stderr and terminate pub fn error(message: &str) -> ! { @@ -49,7 +52,7 @@ pub fn error(message: &str) -> ! { } /// Runs the benchmark for the given commands -fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result> { +fn run(commands: &Vec, options: &HyperfineOptions) -> io::Result> { let shell_spawning_time = mean_shell_spawning_time(&options.output_style)?; let mut timing_results = vec![]; @@ -62,6 +65,26 @@ fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result( + mut vals: clap::Values<'a>, +) -> Result<(&'a str, Range), ParameterRangeError> { + let param_name = vals.next().unwrap(); + let param_min: i32 = vals.next().unwrap().parse()?; + let param_max: i32 = vals.next().unwrap().parse()?; + + const MAX_PARAMETERS: i32 = 100000; + if param_max - param_min > MAX_PARAMETERS { + return Err(ParameterRangeError::TooLarge); + } + + if param_max < param_min { + return Err(ParameterRangeError::EmptyRange); + } + + return Ok((param_name, param_min..(param_max + 1))); +} + fn main() { // Process command line options let mut options = HyperfineOptions::default(); @@ -139,6 +162,17 @@ fn main() { .short("i") .help("Ignore non-zero exit codes."), ) + .arg( + Arg::with_name("parameter-range") + .long("parameter-range") + .short("r") + .help( + "Perform benchmark runs for each value in the range MIN..MAX. Replaces the \ + string '{VAR}' in each command by the current parameter value.", + ) + .takes_value(true) + .value_names(&["VAR", "MIN", "MAX"]), + ) .arg( Arg::with_name("export-csv") .long("export-csv") @@ -213,7 +247,26 @@ fn main() { options.failure_action = CmdFailureAction::Ignore; } - let commands = matches.values_of("command").unwrap().collect(); + let command_strings = matches.values_of("command").unwrap(); + + let commands = if let Some(args) = matches.values_of("parameter-range") { + match parse_parameter_range_args(args) { + Ok((param_name, param_range)) => { + let mut commands = vec![]; + let command_strings = command_strings.collect::>(); + for value in param_range.start..param_range.end { + for ref cmd in &command_strings { + commands.push(Command::new_parametrized(cmd, param_name, value)); + } + } + commands + } + Err(e) => error(e.description()), + } + } else { + command_strings.map(|c| Command::new(c)).collect() + }; + let res = run(&commands, &options); match res {