diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index 7113a32..43a5802 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -3,15 +3,15 @@ use std::io; use colored::*; use statistical::{mean, standard_deviation}; -use hyperfine::internal::{get_progress_bar, max, min, CmdFailureAction, HyperfineOptions, - OutputStyleOption, Second, MIN_EXECUTION_TIME}; +use hyperfine::internal::{get_progress_bar, max, min, MIN_EXECUTION_TIME}; +use hyperfine::types::{CmdFailureAction, HyperfineOptions, OutputStyleOption, Second}; use hyperfine::warnings::Warnings; use hyperfine::format::{format_duration, format_duration_unit}; use hyperfine::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD}; use hyperfine::timer::{TimerStart, TimerStop}; use hyperfine::timer::wallclocktimer::WallClockTimer; use hyperfine::shell::execute_and_time; -use hyperfine::export::ExportEntry; +use hyperfine::types::BenchmarkResult; /// Results from timing a single shell command #[derive(Debug, Default, Copy, Clone)] @@ -137,7 +137,7 @@ pub fn run_benchmark( cmd: &str, shell_spawning_time: TimingResult, options: &HyperfineOptions, -) -> io::Result { +) -> io::Result { println!( "{}{}: {}", "Benchmark #".bold(), @@ -294,7 +294,7 @@ pub fn run_benchmark( println!(" "); - Ok(ExportEntry::new( + Ok(BenchmarkResult::new( cmd.to_string(), t_mean, t_stddev, diff --git a/src/hyperfine/export/csv.rs b/src/hyperfine/export/csv.rs index acd6f19..e58b043 100644 --- a/src/hyperfine/export/csv.rs +++ b/src/hyperfine/export/csv.rs @@ -1,4 +1,6 @@ -use super::{ExportEntry, Exporter}; +use super::Exporter; + +use hyperfine::types::BenchmarkResult; use std::io::{Error, ErrorKind, Result}; @@ -8,7 +10,7 @@ use csv::WriterBuilder; pub struct CsvExporter {} impl Exporter for CsvExporter { - fn serialize(&self, results: &Vec) -> Result> { + fn serialize(&self, results: &Vec) -> Result> { let mut writer = WriterBuilder::new().from_writer(vec![]); for res in results { // The list of times can not be exported to the CSV file - remove it: diff --git a/src/hyperfine/export/json.rs b/src/hyperfine/export/json.rs index ec42a18..0a2cb66 100644 --- a/src/hyperfine/export/json.rs +++ b/src/hyperfine/export/json.rs @@ -1,4 +1,5 @@ -use super::{ExportEntry, Exporter}; +use super::Exporter; +use hyperfine::types::BenchmarkResult; use std::io::{Error, ErrorKind, Result}; @@ -6,14 +7,14 @@ use serde_json::to_vec_pretty; #[derive(Serialize, Debug)] struct HyperfineSummary<'a> { - results: &'a Vec, + results: &'a Vec, } #[derive(Default)] pub struct JsonExporter {} impl Exporter for JsonExporter { - fn serialize(&self, results: &Vec) -> Result> { + fn serialize(&self, results: &Vec) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { results }); for content in output.iter_mut() { content.push(b'\n'); diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index d691d83..b5fa2ec 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -1,4 +1,5 @@ -use super::{ExportEntry, Exporter}; +use super::Exporter; +use hyperfine::types::BenchmarkResult; use std::io::Result; @@ -8,7 +9,7 @@ const MULTIPLIER: f64 = 1e3; pub struct MarkdownExporter {} impl Exporter for MarkdownExporter { - fn serialize(&self, results: &Vec) -> Result> { + fn serialize(&self, results: &Vec) -> Result> { let mut destination = start_table(); for result in results { @@ -24,7 +25,7 @@ fn start_table() -> Vec { .bytes() .collect() } -fn add_table_row(dest: &mut Vec, entry: &ExportEntry) { +fn add_table_row(dest: &mut Vec, entry: &BenchmarkResult) { dest.extend( format!( "| `{}` | {:.1} ± {:.1} | {:.1} | {:.1} |\n", diff --git a/src/hyperfine/export/mod.rs b/src/hyperfine/export/mod.rs index 3ab275f..35ca48d 100644 --- a/src/hyperfine/export/mod.rs +++ b/src/hyperfine/export/mod.rs @@ -9,61 +9,7 @@ use self::markdown::MarkdownExporter; use std::io::{Result, Write}; use std::fs::File; -use hyperfine::internal::Second; - -/// Set of values that will be exported. -#[derive(Debug, Default, Clone, Serialize)] -pub struct ExportEntry { - /// The command that was run - command: String, - - /// The mean run time - mean: Second, - - /// The standard deviation of all run times - stddev: Second, - - /// Time spend in user space - user: Second, - - /// Time spent in system space - system: Second, - - /// Min time measured - min: Second, - - /// Max time measured - max: Second, - - /// All run time measurements - #[serde(skip_serializing_if = "Option::is_none")] - times: Option>, -} - -impl ExportEntry { - /// Create a new entry with the given values. - pub fn new( - command: String, - mean: Second, - stddev: Second, - user: Second, - system: Second, - min: Second, - max: Second, - times: Vec, - ) -> Self { - ExportEntry { - command, - mean, - stddev, - user, - system, - min, - max, - times: Some(times), - } - } -} +use hyperfine::types::BenchmarkResult; /// The desired form of exporter to use for a given file. #[derive(Clone)] @@ -81,7 +27,7 @@ pub enum ExportType { /// Interface for different exporters. trait Exporter { /// Export the given entries in the serialized form. - fn serialize(&self, results: &Vec) -> Result>; + fn serialize(&self, results: &Vec) -> Result>; } struct ExporterWithFilename { @@ -116,7 +62,7 @@ impl ExportManager { } /// Write the given results to all Exporters contained within this manager - pub fn write_results(&self, results: Vec) -> Result<()> { + pub fn write_results(&self, results: Vec) -> Result<()> { for e in &self.exporters { let file_content = e.exporter.serialize(&results)?; write_to_file(&e.filename, &file_content)?; diff --git a/src/hyperfine/format.rs b/src/hyperfine/format.rs index cba415c..311cf67 100644 --- a/src/hyperfine/format.rs +++ b/src/hyperfine/format.rs @@ -1,4 +1,4 @@ -use hyperfine::internal::Second; +use hyperfine::types::Second; #[derive(Debug, Clone, Copy, PartialEq)] pub enum Unit { diff --git a/src/hyperfine/internal.rs b/src/hyperfine/internal.rs index 4ef9b99..46a7cdd 100644 --- a/src/hyperfine/internal.rs +++ b/src/hyperfine/internal.rs @@ -1,68 +1,14 @@ +use colored::*; use indicatif::{ProgressBar, ProgressStyle}; -/// Type alias for unit of time -pub type Second = f64; +use hyperfine::types::{BenchmarkResult, OutputStyleOption, Second}; + +use std::cmp::Ordering; +use std::iter::Iterator; /// Threshold for warning about fast execution time pub const MIN_EXECUTION_TIME: Second = 5e-3; -/// Action to take when an executed command fails. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum CmdFailureAction { - /// Exit with an error message - RaiseError, - - /// Simply ignore the non-zero exit code - Ignore, -} - -/// Output style type option -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum OutputStyleOption { - /// Do not output with colors or any special formatting - Basic, - - /// Output with full color and formatting - Full, - - /// Keep elements such as progress bar, but use no coloring - NoColor, -} - -/// A set of options for hyperfine -pub struct HyperfineOptions { - /// Number of warmup runs - pub warmup_count: u64, - - /// Minimum number of benchmark runs - pub min_runs: u64, - - /// Minimum benchmarking time - pub min_time_sec: Second, - - /// Whether or not to ignore non-zero exit codes - pub failure_action: CmdFailureAction, - - /// Command to run before each timing run - pub preparation_command: Option, - - /// What color mode to use for output - pub output_style: OutputStyleOption, -} - -impl Default for HyperfineOptions { - fn default() -> HyperfineOptions { - HyperfineOptions { - warmup_count: 0, - min_runs: 10, - min_time_sec: 3.0, - failure_action: CmdFailureAction::RaiseError, - preparation_command: None, - output_style: OutputStyleOption::Full, - } - } -} - /// Return a pre-configured progress bar pub fn get_progress_bar(length: u64, msg: &str, option: &OutputStyleOption) -> ProgressBar { let progressbar_style = match *option { @@ -97,6 +43,39 @@ pub fn min(vals: &[f64]) -> f64 { .unwrap() } +pub fn write_benchmark_comparison(results: &Vec) { + if results.len() < 2 { + return; + } + + // Show which was faster, maybe expand to table later? + let mut fastest_item: &BenchmarkResult = &results[0]; + let mut longer_items: Vec<&BenchmarkResult> = Vec::new(); + + for run in &results[1..] { + if let Some(Ordering::Less) = fastest_item.mean.partial_cmp(&run.mean) { + longer_items.push(run); + } else { + longer_items.push(fastest_item); + fastest_item = run; + } + } + + println!("{}\n", "Result Comparison".bold()); + println!("'{}' ran", fastest_item.command.cyan()); + longer_items.sort_by(|l, r| l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)); + + for item in longer_items { + println!( + "{} faster than '{}'", + format!("{:8.2}x", item.mean / fastest_item.mean) + .bold() + .green(), + &item.command.magenta() + ); + } +} + #[test] fn test_max() { assert_eq!(1.0, max(&[1.0])); diff --git a/src/hyperfine/mod.rs b/src/hyperfine/mod.rs index 89b0500..74e34a1 100644 --- a/src/hyperfine/mod.rs +++ b/src/hyperfine/mod.rs @@ -1,8 +1,9 @@ pub mod benchmark; +pub mod export; pub mod format; pub mod internal; pub mod outlier_detection; -pub mod warnings; -pub mod timer; pub mod shell; -pub mod export; +pub mod timer; +pub mod types; +pub mod warnings; diff --git a/src/hyperfine/timer/internal.rs b/src/hyperfine/timer/internal.rs index 29bf254..04cc2a7 100644 --- a/src/hyperfine/timer/internal.rs +++ b/src/hyperfine/timer/internal.rs @@ -1,4 +1,4 @@ -use hyperfine::internal::Second; +use hyperfine::types::Second; #[derive(Debug, Copy, Clone)] pub struct CPUTimes { diff --git a/src/hyperfine/timer/unix_timer.rs b/src/hyperfine/timer/unix_timer.rs index 6868d52..33457c2 100644 --- a/src/hyperfine/timer/unix_timer.rs +++ b/src/hyperfine/timer/unix_timer.rs @@ -2,7 +2,7 @@ use super::internal::{CPUInterval, CPUTimes}; use hyperfine::timer::{TimerStart, TimerStop}; -use hyperfine::internal::Second; +use hyperfine::types::Second; use std::mem; use std::process::Child; diff --git a/src/hyperfine/timer/wallclocktimer.rs b/src/hyperfine/timer/wallclocktimer.rs index e68cda1..c39d2f0 100644 --- a/src/hyperfine/timer/wallclocktimer.rs +++ b/src/hyperfine/timer/wallclocktimer.rs @@ -1,7 +1,7 @@ use std::process::Child; use std::time::Instant; -use hyperfine::internal::Second; +use hyperfine::types::Second; use hyperfine::timer::{TimerStart, TimerStop}; pub struct WallClockTimer { diff --git a/src/hyperfine/timer/windows_timer.rs b/src/hyperfine/timer/windows_timer.rs index 6ee82e1..512600e 100644 --- a/src/hyperfine/timer/windows_timer.rs +++ b/src/hyperfine/timer/windows_timer.rs @@ -1,7 +1,7 @@ #![cfg(windows)] use super::internal::CPUTimes; -use hyperfine::internal::Second; +use hyperfine::types::Second; use hyperfine::timer::{TimerStart, TimerStop}; use winapi::um::processthreadsapi::GetProcessTimes; @@ -13,7 +13,7 @@ use std::process::Child; const HUNDRED_NS_PER_MS: i64 = 10; -pub fn get_cpu_timer(process: &Child) -> Box> { +pub fn get_cpu_timer(process: &Child) -> Box> { Box::new(CPUTimer::start_for_process(process)) } @@ -22,14 +22,15 @@ struct CPUTimer { } impl TimerStart for CPUTimer { - fn start() -> Self { panic!() } fn start_for_process(process: &Child) -> Self { - CPUTimer { handle: process.as_raw_handle() } - } + CPUTimer { + handle: process.as_raw_handle(), + } + } } impl TimerStop for CPUTimer { @@ -37,7 +38,10 @@ impl TimerStop for CPUTimer { fn stop(&self) -> Self::Result { let times = get_cpu_times(self.handle); - (times.user_usec as f64 * 1e-6, times.system_usec as f64 * 1e-6) + ( + times.user_usec as f64 * 1e-6, + times.system_usec as f64 * 1e-6, + ) } } @@ -59,9 +63,12 @@ fn get_cpu_times(handle: RawHandle) -> CPUTimes { // GetProcessTimes will exit with non-zero if success as per: https://msdn.microsoft.com/en-us/library/windows/desktop/ms683223(v=vs.85).aspx if res != 0 { // Extract times as laid out here: https://support.microsoft.com/en-us/help/188768/info-working-with-the-filetime-structure - // Both user_time and kernel_time are spans that the proces spent in either. - let user: i64 = (((user_time.dwHighDateTime as i64) << 32) + user_time.dwLowDateTime as i64) / HUNDRED_NS_PER_MS; - let kernel: i64 = (((kernel_time.dwHighDateTime as i64) << 32) + kernel_time.dwLowDateTime as i64) / HUNDRED_NS_PER_MS; + // Both user_time and kernel_time are spans that the proces spent in either. + let user: i64 = (((user_time.dwHighDateTime as i64) << 32) + + user_time.dwLowDateTime as i64) / HUNDRED_NS_PER_MS; + let kernel: i64 = (((kernel_time.dwHighDateTime as i64) << 32) + + kernel_time.dwLowDateTime as i64) + / HUNDRED_NS_PER_MS; (user, kernel) } else { (0, 0) @@ -69,6 +76,7 @@ fn get_cpu_times(handle: RawHandle) -> CPUTimes { }; CPUTimes { - user_usec, system_usec, + user_usec, + system_usec, } } diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs new file mode 100644 index 0000000..1ae672b --- /dev/null +++ b/src/hyperfine/types.rs @@ -0,0 +1,116 @@ +/// The types module contains common internal types for the application. +/// + +/// Type alias for unit of time +pub type Second = f64; + +/// Action to take when an executed command fails. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum CmdFailureAction { + /// Exit with an error message + RaiseError, + + /// Simply ignore the non-zero exit code + Ignore, +} + +/// Output style type option +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum OutputStyleOption { + /// Do not output with colors or any special formatting + Basic, + + /// Output with full color and formatting + Full, + + /// Keep elements such as progress bar, but use no coloring + NoColor, +} + +/// A set of options for hyperfine +pub struct HyperfineOptions { + /// Number of warmup runs + pub warmup_count: u64, + + /// Minimum number of benchmark runs + pub min_runs: u64, + + /// Minimum benchmarking time + pub min_time_sec: Second, + + /// Whether or not to ignore non-zero exit codes + pub failure_action: CmdFailureAction, + + /// Command to run before each timing run + pub preparation_command: Option, + + /// What color mode to use for output + pub output_style: OutputStyleOption, +} + +impl Default for HyperfineOptions { + fn default() -> HyperfineOptions { + HyperfineOptions { + warmup_count: 0, + min_runs: 10, + min_time_sec: 3.0, + failure_action: CmdFailureAction::RaiseError, + preparation_command: None, + output_style: OutputStyleOption::Full, + } + } +} + +/// Set of values that will be exported. +#[derive(Debug, Default, Clone, Serialize)] +pub struct BenchmarkResult { + /// The command that was run + pub command: String, + + /// The mean run time + pub mean: Second, + + /// The standard deviation of all run times + pub stddev: Second, + + /// Time spend in user space + pub user: Second, + + /// Time spent in system space + pub system: Second, + + /// Min time measured + pub min: Second, + + /// Max time measured + pub max: Second, + + /// All run time measurements + #[serde(skip_serializing_if = "Option::is_none")] + pub times: Option>, +} + +impl BenchmarkResult { + /// Create a new entry with the given values. + pub fn new( + command: String, + mean: Second, + stddev: Second, + user: Second, + system: Second, + min: Second, + max: Second, + times: Vec, + ) -> Self { + BenchmarkResult { + command, + mean, + stddev, + user, + system, + min, + max, + times: Some(times), + } + } +} diff --git a/src/hyperfine/warnings.rs b/src/hyperfine/warnings.rs index 4e21f26..d7520ad 100644 --- a/src/hyperfine/warnings.rs +++ b/src/hyperfine/warnings.rs @@ -1,6 +1,7 @@ use std::fmt; -use hyperfine::internal::{Second, MIN_EXECUTION_TIME}; +use hyperfine::internal::MIN_EXECUTION_TIME; +use hyperfine::types::Second; use hyperfine::format::format_duration; /// A list of all possible warnings diff --git a/src/main.rs b/src/main.rs index e965f85..27dfa8d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,9 +37,10 @@ use clap::{App, AppSettings, Arg}; mod hyperfine; -use hyperfine::internal::{CmdFailureAction, HyperfineOptions, OutputStyleOption}; +use hyperfine::internal::write_benchmark_comparison; +use hyperfine::types::{BenchmarkResult, CmdFailureAction, HyperfineOptions, OutputStyleOption}; use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark}; -use hyperfine::export::{ExportEntry, ExportManager, ExportType}; +use hyperfine::export::{ExportManager, ExportType}; /// Print error message to stderr and terminate pub fn error(message: &str) -> ! { @@ -48,7 +49,7 @@ pub fn error(message: &str) -> ! { } /// Runs the benchmark for the given commands -fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result> { +fn run(commands: &Vec<&str>, options: &HyperfineOptions) -> io::Result> { let shell_spawning_time = mean_shell_spawning_time(&options.output_style)?; let mut timing_results = vec![]; @@ -217,6 +218,7 @@ fn main() { match res { Ok(timing_results) => { + write_benchmark_comparison(&timing_results); let ans = export_manager.write_results(timing_results); if let Err(e) = ans { error(&format!(