diff --git a/Cargo.lock b/Cargo.lock index 0010ab9..e81d487 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,12 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + [[package]] name = "hyperfine" version = "1.21.0" @@ -380,12 +386,14 @@ dependencies = [ "clap_complete", "colored", "csv", + "humantime", "indicatif", "insta", "insta-cmd", "libc", "nix", "once_cell", + "platform-info", "predicates", "rand 0.10.3", "rust_decimal", @@ -531,6 +539,16 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "platform-info" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba7c3f5af00d3596a148cc80178ac690152b71c283d15a8bcea0cddfea759a36" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "portable-atomic" version = "1.15.0" diff --git a/Cargo.toml b/Cargo.toml index e0e33e0..8a6e7d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,8 @@ shell-words = "1.1" thiserror = "2.0" anyhow = "1.0" uom = { version = "0.36.0", default-features = false, features = ["std", "si", "f64"] } +humantime = "2" +platform-info = "2.2" [target.'cfg(not(windows))'.dependencies] libc = "0.2" diff --git a/README.md b/README.md index dc6d655..b05482f 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,28 @@ You can use the `--export-markdown ` option to create tables like the foll #### JSON +The JSON export includes the following metrics for each measured run (excluding warmup runs): + +- **`time_wall_clock`**: Time from start to finish, including time spent waiting, in seconds. + +- **`time_user`**: CPU time spent running the program's code, summed across threads, in seconds. + + - Linux/macOS: Includes child-process time when parents wait for their children to finish. + - Windows: Includes the command and its child processes. + +- **`time_system`**: CPU time spent running operating-system code for the program, for example + to read files, summed across threads, in seconds. + + - Linux/macOS: Includes child-process time when parents wait for their children to finish. + - Windows: Includes the command and its child processes. + +- **`memory_peak_resident`**: Peak memory held in physical RAM, in bytes. + + - Linux/macOS: Peak resident set size (RSS). This is the largest per-process peak among the + command and child processes (whose usage is collected when their parents wait for them), + *not the simultaneous total memory of the full process tree*. + - Windows: Currently not supported. + The JSON output is useful if you want to analyze the benchmark results in more detail. The [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot of helpful Python programs to further analyze benchmark results and create helpful diff --git a/scripts/advanced_statistics.py b/scripts/advanced_statistics.py index 5ad3579..99394eb 100755 --- a/scripts/advanced_statistics.py +++ b/scripts/advanced_statistics.py @@ -50,11 +50,9 @@ unit_str = str(unit) with open(args.file) as f: results = json.load(f)["results"] -commands = [b["command"] for b in results] -times = [b["times"] for b in results] - -for command, ts in zip(commands, times): - ts = [t * unit.factor() for t in ts] +for result in results: + summary = result["summary"]["time_wall_clock"] + ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]] p05 = np.percentile(ts, 5) p25 = np.percentile(ts, 25) @@ -63,13 +61,16 @@ for command, ts in zip(commands, times): iqr = p75 - p25 - print(f"Command '{command}'") - print(f" runs: {len(ts):8d}") - print(f" mean: {np.mean(ts):8.3f} {unit_str}") - print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}") - print(f" median: {np.median(ts):8.3f} {unit_str}") - print(f" min: {np.min(ts):8.3f} {unit_str}") - print(f" max: {np.max(ts):8.3f} {unit_str}") + print(f"Command '{result.get('name', result['command'])}'") + print(f" runs: {summary['count']:8d}") + print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}") + if summary["stddev"] is None: + print(f" stddev: {'N/A':>8}") + else: + print(f" stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}") + print(f" median: {summary['median'] * unit.factor():8.3f} {unit_str}") + print(f" min: {summary['min'] * unit.factor():8.3f} {unit_str}") + print(f" max: {summary['max'] * unit.factor():8.3f} {unit_str}") print() print(" percentiles:") print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}") diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index 48d49a8..754ee17 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -44,14 +44,14 @@ if args.benchmark_names: for i, filename in enumerate(args.files): with open(filename) as f: results = json.load(f)["results"] - benchmark_commands = [b["command"] for b in results] + benchmark_commands = [b.get("name", b["command"]) for b in results] if commands is None: commands = benchmark_commands else: assert ( commands == benchmark_commands ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" - data.append([round(b["mean"], 2) for b in results]) + data.append([b["summary"]["time_wall_clock"]["mean"] for b in results]) if args.benchmark_names: inputs.append(args.benchmark_names[i]) else: diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index cc11677..13d51c4 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -66,8 +66,10 @@ with open(args.file) as f: if args.labels: labels = args.labels.split(",") else: - labels = [b["command"] for b in results] -all_times = [b["times"] for b in results] + labels = [b.get("name", b["command"]) for b in results] +all_times = [ + [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results +] t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times))) t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times))) diff --git a/scripts/plot_parametrized.py b/scripts/plot_parametrized.py index 8ab4bc8..0db1cb6 100755 --- a/scripts/plot_parametrized.py +++ b/scripts/plot_parametrized.py @@ -71,7 +71,7 @@ def unique_parameter(benchmark): f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}" ) [(name, value)] = params_dict.items() - return (name, float(value)) + return (name, float(value["value"])) parameter_name = None @@ -87,8 +87,11 @@ for filename in args.file: ) parameter_name = this_parameter_name - times_mean = [b["mean"] for b in results] - times_stddev = [b["stddev"] for b in results] + summaries = [b["summary"]["time_wall_clock"] for b in results] + times_mean = [s["mean"] for s in summaries] + times_stddev = [ + s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries + ] plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2) diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index 61b88eb..9cd7808 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -52,8 +52,8 @@ with open(args.file) as f: results = json.load(f)["results"] for result in results: - label = result["command"] - times = result["times"] + label = result.get("name", result["command"]) + times = [m["time_wall_clock"]["value"] for m in result["measurements"]] num = len(times) nums = range(num) @@ -63,7 +63,9 @@ for result in results: if not args.no_moving_average: moving_average_width = ( - num // 5 if args.moving_average_width is None else args.moving_average_width + max(1, num // 5) + if args.moving_average_width is None + else args.moving_average_width ) average = moving_average(times, moving_average_width) @@ -74,7 +76,7 @@ if args.title: legend = [] for result in results: - legend.append(result["command"]) + legend.append(result.get("name", result["command"])) if not args.no_moving_average: legend.append("moving average") plt.legend(legend) diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index 5aeb74d..baaf175 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -37,11 +37,11 @@ with open(args.file, encoding="utf-8") as f: if args.labels: labels = args.labels.split(",") else: - labels = [b["command"] for b in results] -times = [b["times"] for b in results] + labels = [b.get("name", b["command"]) for b in results] +times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results] if args.sort_by == "median": - medians = [b["median"] for b in results] + medians = [b["summary"]["time_wall_clock"]["median"] for b in results] indices = sorted(range(len(labels)), key=lambda k: medians[k]) labels = [labels[i] for i in indices] times = [times[i] for i in indices] diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index 6183848..f6e48b3 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -26,8 +26,8 @@ if len(results) != 2: print("The input file has to contain exactly two benchmarks") sys.exit(1) -a, b = (x["command"] for x in results[:2]) -X, Y = (x["times"] for x in results[:2]) +a, b = (x.get("name", x["command"]) for x in results[:2]) +X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2]) print(f"Command 1: {a}") print(f"Command 2: {b}\n") diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 387884d..1d9f597 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -1,97 +1,55 @@ use std::collections::BTreeMap; -use serde::ser::SerializeStruct; -use serde::{Serialize, Serializer}; +use serde::Serialize; use crate::benchmark::measurement::Measurements; -use crate::quantity::{byte, second, Time}; -use crate::util::exit_code::extract_exit_code; +use crate::quantity::Time; -/// Parameter value and whether it was used in the command line template +/// Parameter value #[derive(Debug, Default, Clone, Serialize, PartialEq)] -#[serde(transparent)] pub struct Parameter { pub value: String, - #[serde(skip)] - pub is_unused: bool, } /// Meta data and performance metrics for a single benchmark -#[derive(Debug, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct BenchmarkResult { - /// The full command line of the program that is being benchmarked + /// The command line being benchmarked, after parameter substitution. + /// For example, `sleep {duration}` with `duration=1` becomes `sleep 1`, + /// regardless of any custom name. pub command: String, - /// The full command line, including parameters not used in the command template. - pub command_with_unused_parameters: String, + /// The custom name after parameter substitution, if it differs from `command`. + /// For example, `--command-name="wait {duration}s"` with `duration=1` + /// produces `Some("wait 1s")`. Without a custom name, or when the expanded + /// name equals `command`, this is `None`. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, - /// Performance measurements and exit statuses for each run + /// The custom name (or `command`), followed by parameters not used in the + /// command template, in command-line order. For example, `sleep {duration}` + /// with `duration=1`, `branch=main`, and the custom name `wait {duration}s` + /// produces `wait 1s (branch = main)`. Without the custom name, this is + /// `sleep 1 (branch = main)`. Used in summaries and table exports. + #[serde(skip_serializing)] + pub display_name: String, + + /// Performance metric measurements and exit codes for each run + #[serde(flatten)] pub measurements: Measurements, /// Parameter values for this benchmark + #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, } impl BenchmarkResult { + pub fn get_name(&self) -> &str { + self.name.as_deref().unwrap_or(&self.command) + } + /// The average wall clock time pub fn mean_wall_clock_time(&self) -> Time { self.measurements.time_wall_clock_mean() } } - -// Preserve the existing export format while storing typed measurements internally. -impl Serialize for BenchmarkResult { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut state = serializer.serialize_struct( - "BenchmarkResult", - if self.parameters.is_empty() { 11 } else { 12 }, - )?; - state.serialize_field("command", &self.command)?; - state.serialize_field("mean", &self.mean_wall_clock_time().get::())?; - state.serialize_field( - "stddev", - &self.measurements.stddev().map(|time| time.get::()), - )?; - state.serialize_field("median", &self.measurements.median().get::())?; - state.serialize_field("user", &self.measurements.time_user_mean().get::())?; - state.serialize_field( - "system", - &self.measurements.time_system_mean().get::(), - )?; - state.serialize_field("min", &self.measurements.min().get::())?; - state.serialize_field("max", &self.measurements.max().get::())?; - state.serialize_field( - "times", - &self - .measurements - .wall_clock_times() - .map(|time| time.get::()) - .collect::>(), - )?; - state.serialize_field( - "memory_usage_byte", - &self - .measurements - .measurements - .iter() - .map(|measurement| measurement.peak_memory_usage.get::() as u64) - .collect::>(), - )?; - state.serialize_field( - "exit_codes", - &self - .measurements - .measurements - .iter() - .map(|measurement| extract_exit_code(measurement.exit_status)) - .collect::>(), - )?; - if !self.parameters.is_empty() { - state.serialize_field("parameters", &self.parameters)?; - } - state.end() - } -} diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index e450d40..de077b7 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -2,6 +2,8 @@ use std::os::windows::process::CommandExt; use std::process::ExitStatus; +use crate::benchmark::measurement::Measurement; +use crate::benchmark::measurement::Measurements; use crate::command::Command; use crate::options::{ CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell, @@ -11,8 +13,6 @@ use crate::quantity::{second, Information, Time, Zero}; use crate::timer::execute_and_measure; use crate::util::randomized_environment_offset; -use super::measurement::{Measurement, Measurements}; - use anyhow::{bail, Context, Result}; #[derive(Clone, Copy)] @@ -76,10 +76,10 @@ fn run_command_and_measure_common( command.env("HYPERFINE_ITERATION", value); } - let result = execute_and_measure(command) + let measurement = execute_and_measure(command) .with_context(|| format!("Failed to run command '{command_name}'"))?; - if !result.exit_status.success() { + if !measurement.exit_status.success() { use crate::util::exit_code::extract_exit_code; let should_fail = match command_failure_action { @@ -87,7 +87,7 @@ fn run_command_and_measure_common( CmdFailureAction::IgnoreAllFailures => false, CmdFailureAction::IgnoreSpecificFailures(ref codes) => { // Only fail if the exit code is not in the list of codes to ignore - if let Some(exit_code) = extract_exit_code(result.exit_status) { + if let Some(exit_code) = extract_exit_code(measurement.exit_status) { !codes.contains(&exit_code) } else { // If we can't extract an exit code, treat it as a failure @@ -107,7 +107,7 @@ fn run_command_and_measure_common( bail!( "{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \ Alternatively, use the '--show-output' option to debug what went wrong.", - cause=result.exit_status.code().map_or( + cause=measurement.exit_status.code().map_or( "The process has been terminated by a signal".into(), |c| format!("Command terminated with non-zero exit code {c}") @@ -116,7 +116,7 @@ fn run_command_and_measure_common( } } - Ok(result) + Ok(measurement) } pub struct RawExecutor<'a> { @@ -192,7 +192,7 @@ impl Executor for ShellExecutor<'_> { command_builder.arg(command.get_command_line()); } - let mut result = run_command_and_measure_common( + let mut measurement = run_command_and_measure_common( command_builder, iteration, command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()), @@ -202,14 +202,24 @@ impl Executor for ShellExecutor<'_> { )?; // Subtract shell spawning time - if let Some(spawning_time) = self.shell_spawning_time { - result.time_wall_clock = - (result.time_wall_clock - spawning_time.time_wall_clock).max(Time::zero()); - result.time_user = (result.time_user - spawning_time.time_user).max(Time::zero()); - result.time_system = (result.time_system - spawning_time.time_system).max(Time::zero()); + fn ensure_non_negative(time: Time) -> Time { + if time < Time::zero() { + Time::zero() + } else { + time + } } - Ok(result) + if let Some(ref spawning_time) = self.shell_spawning_time { + measurement.time_wall_clock = + ensure_non_negative(measurement.time_wall_clock - spawning_time.time_wall_clock); + measurement.time_user = + ensure_non_negative(measurement.time_user - spawning_time.time_user); + measurement.time_system = + ensure_non_negative(measurement.time_system - spawning_time.time_system); + } + + Ok(measurement) } /// Measure the average shell spawning time @@ -229,14 +239,14 @@ impl Executor for ShellExecutor<'_> { for _ in 0..COUNT { // Just run the shell without any command - let res = self.run_command_and_measure( + let measurement = self.run_command_and_measure( &Command::new(None, ""), BenchmarkIteration::NonBenchmarkRun, None, &CommandOutputPolicy::Null, ); - match res { + match measurement { Err(_) => { let shell_cmd = if cfg!(windows) { format!("{} /C \"\"", self.shell) @@ -249,8 +259,8 @@ impl Executor for ShellExecutor<'_> { shell_cmd ); } - Ok(r) => { - measurements.push(r); + Ok(result) => { + measurements.push(result); } } @@ -267,7 +277,7 @@ impl Executor for ShellExecutor<'_> { time_wall_clock: measurements.time_wall_clock_mean(), time_user: measurements.time_user_mean(), time_system: measurements.time_system_mean(), - peak_memory_usage: measurements.peak_memory_usage_mean(), + memory_peak_resident: None, exit_status: ExitStatus::default(), }); @@ -275,7 +285,7 @@ impl Executor for ShellExecutor<'_> { } fn time_overhead(&self) -> Time { - self.shell_spawning_time.unwrap().time_wall_clock + self.shell_spawning_time.as_ref().unwrap().time_wall_clock } } @@ -325,7 +335,7 @@ impl Executor for MockExecutor { time_wall_clock: Self::extract_time(command.get_command_line()), time_user: Time::zero(), time_system: Time::zero(), - peak_memory_usage: Information::zero(), + memory_peak_resident: Some(Information::zero()), exit_status, }) } diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index da35e9b..0c2e4ea 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -1,29 +1,62 @@ use std::process::ExitStatus; -use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation}; -use crate::quantity::{Information, Time}; +use serde::Serialize; -/// Performance measurements and exit status from running a single command -#[derive(Debug, Default, Copy, Clone, PartialEq)] +use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation}; +use crate::quantity::{serialize_information, serialize_time, Information, Time}; +use crate::util::exit_code::extract_exit_code; + +fn serialize_exit_status(exit_status: &ExitStatus, serializer: S) -> Result +where + S: serde::Serializer, +{ + match extract_exit_code(*exit_status) { + Some(code) => serializer.serialize_i32(code), + None => serializer.serialize_unit(), + } +} + +fn serialize_optional_information( + value: &Option, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + match value { + Some(value) => serialize_information(value, serializer), + None => serializer.serialize_none(), + } +} + +/// Performance metric measurements and exit code for a single run +#[derive(Debug, Clone, Default, Serialize, PartialEq)] pub struct Measurement { - /// Wall clock time + /// Elapsed wall clock time (real time) + #[serde(serialize_with = "serialize_time")] pub time_wall_clock: Time, /// Time spent in user mode + #[serde(serialize_with = "serialize_time")] pub time_user: Time, /// Time spent in kernel mode + #[serde(serialize_with = "serialize_time")] pub time_system: Time, - /// Maximum amount of memory used - pub peak_memory_usage: Information, + /// Peak resident set size, if available on this platform + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_optional_information" + )] + pub memory_peak_resident: Option, - /// The exit status of the process + // The exit status of the process + #[serde(rename = "exit_code", serialize_with = "serialize_exit_status")] pub exit_status: ExitStatus, } -/// Performance measurements and exit statuses for all runs of a command. -#[derive(Debug, Default, Clone, PartialEq)] +#[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Measurements { pub measurements: Vec, } @@ -49,7 +82,7 @@ impl Measurements { self.measurements.iter().map(|m| m.time_wall_clock) } - /// The average wall clock time. + /// The average wall clock time pub fn time_wall_clock_mean(&self) -> Time { mean(self.wall_clock_times()) } @@ -65,37 +98,33 @@ impl Measurements { } } - /// The median wall clock time. + /// The median wall clock time pub fn median(&self) -> Time { median(self.wall_clock_times()) } - /// The minimum wall clock time. + /// The minimum wall clock time pub fn min(&self) -> Time { min(self.wall_clock_times()) } - /// The maximum wall clock time. + /// The maximum wall clock time pub fn max(&self) -> Time { max(self.wall_clock_times()) } - /// Compute modified Z-scores for the wall clock times. + /// Compute modified Z-scores for the wall clock times pub fn modified_zscores(&self) -> Vec { modified_zscores(&self.wall_clock_times().collect::>()) } - /// The average user time. + /// The average user time pub fn time_user_mean(&self) -> Time { mean(self.measurements.iter().map(|m| m.time_user)) } - /// The average system time. + /// The average system time pub fn time_system_mean(&self) -> Time { mean(self.measurements.iter().map(|m| m.time_system)) } - - pub fn peak_memory_usage_mean(&self) -> Information { - mean(self.measurements.iter().map(|m| m.peak_memory_usage)) - } } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index caa589f..39adf92 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -8,7 +8,9 @@ use std::cmp; use std::io::{self, Write}; use std::time::Instant; +use crate::benchmark::benchmark_result::Parameter; use crate::benchmark::executor::BenchmarkIteration; +use crate::benchmark::measurement::{Measurement, Measurements}; use crate::command::Command; use crate::options::{ CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption, @@ -20,9 +22,8 @@ use crate::output::progress_bar::{ }; use crate::output::warnings::{OutlierWarningOptions, Warnings}; use crate::parameter::ParameterNameAndValue; -use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero}; -use benchmark_result::{BenchmarkResult, Parameter}; -use measurement::{Measurement, Measurements}; +use crate::quantity::{self, const_time_from_seconds, FormatQuantity, Time, Zero}; +use benchmark_result::BenchmarkResult; use anyhow::{anyhow, Result}; use colored::*; @@ -30,7 +31,7 @@ use colored::*; use self::executor::Executor; /// Threshold for warning about fast execution time -pub const MIN_EXECUTION_TIME: Time = const_time_from_seconds(5e-3); +pub const MIN_EXECUTION_TIME: Time = const_time_from_seconds(0.005); pub struct Benchmark<'a> { number: usize, @@ -268,20 +269,20 @@ impl<'a> Benchmark<'a> { if let Some(bar) = progress_bar.as_ref() { start_initial_measurement(bar, Instant::now()); } - let res = self.executor.run_command_and_measure( + let measurement = self.executor.run_command_and_measure( self.command, benchmark_iteration, None, output_policy, )?; - let success = res.exit_status.success(); + let success = measurement.exit_status.success(); if let Some(bar) = progress_bar.as_ref() { let time_unit = self .options .time_unit - .unwrap_or(res.time_wall_clock.suitable_unit()); - let estimate = res.time_wall_clock.format(time_unit); + .unwrap_or(measurement.time_wall_clock.suitable_unit()); + let estimate = measurement.time_wall_clock.format(time_unit); finish_initial_measurement( bar, format!("Current estimate: {}", estimate.to_string().green()), @@ -295,11 +296,11 @@ impl<'a> Benchmark<'a> { // Determine number of benchmark runs let runs_in_min_time = (self.options.min_benchmarking_time - / (res.time_wall_clock + / (measurement.time_wall_clock + self.executor.time_overhead() + preparation_overhead + conclusion_overhead)) - .get::() as u64; + .get::() as u64; let count = { let min = cmp::max(runs_in_min_time, self.options.run_bounds.min); @@ -315,7 +316,7 @@ impl<'a> Benchmark<'a> { let count_remaining = count - 1; // Save the first result - measurements.push(res); + measurements.push(measurement); all_succeeded = all_succeeded && success; @@ -345,15 +346,14 @@ impl<'a> Benchmark<'a> { run_preparation_command(benchmark_iteration)?; - let res = self.executor.run_command_and_measure( + let measurement = self.executor.run_command_and_measure( self.command, benchmark_iteration, None, output_policy, )?; - let success = res.exit_status.success(); - - measurements.push(res); + let success = measurement.exit_status.success(); + measurements.push(measurement); all_succeeded = all_succeeded && success; @@ -368,25 +368,19 @@ impl<'a> Benchmark<'a> { bar.finish_and_clear() } - // Compute statistical quantities - let t_num = measurements.len(); - let t_mean = measurements.time_wall_clock_mean(); - let t_stddev = measurements.stddev(); - let t_min = measurements.min(); - let t_max = measurements.max(); - - let user_mean = measurements.time_user_mean(); - let system_mean = measurements.time_system_mean(); - // Formatting and console output - let time_unit = self.options.time_unit.unwrap_or(t_mean.suitable_unit()); - let mean_str = t_mean.format(time_unit); - let min_str = t_min.format(time_unit); - let max_str = t_max.format(time_unit); - let num_str = format!("{t_num} runs"); + let t_wall_clock_mean = measurements.time_wall_clock_mean(); + let time_unit = self + .options + .time_unit + .unwrap_or(t_wall_clock_mean.suitable_unit()); + let mean_str = t_wall_clock_mean.format(time_unit); + let min_str = measurements.min().format(time_unit); + let max_str = measurements.max().format(time_unit); + let num_str = format!("{num_runs} runs", num_runs = measurements.len()); - let user_str = user_mean.format(time_unit); - let system_str = system_mean.format(time_unit); + let user_str = measurements.time_user_mean().format(time_unit); + let system_str = measurements.time_system_mean().format(time_unit); if self.options.output_style != OutputStyleOption::Disabled { let mut stdout = io::stdout().lock(); @@ -401,7 +395,7 @@ impl<'a> Benchmark<'a> { system_str.blue() )?; } else { - let stddev_str = t_stddev.unwrap().format(time_unit); + let stddev_str = measurements.stddev().unwrap().format(time_unit); console_writeln!( stdout, @@ -481,9 +475,14 @@ impl<'a> Benchmark<'a> { self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?; + let command = self.command.get_command_line(); + let name = self.command.get_name(); + let name = (name != command).then_some(name); + Ok(BenchmarkResult { - command: self.command.get_name(), - command_with_unused_parameters: self.command.get_name_with_unused_parameters(), + command, + name, + display_name: self.command.get_name_with_unused_parameters(), measurements, parameters: self .command @@ -494,7 +493,6 @@ impl<'a> Benchmark<'a> { name.to_string(), Parameter { value: value.to_string(), - is_unused: self.command.is_parameter_unused(name), }, ) }) diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index e62a591..3151a01 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -154,11 +154,12 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; use crate::benchmark::measurement::{Measurement, Measurements}; - use crate::quantity::{second, Time}; + use crate::quantity::second; BenchmarkResult { command: name.into(), - command_with_unused_parameters: name.into(), + display_name: name.into(), + name: None, measurements: Measurements { measurements: vec![Measurement { time_wall_clock: Time::new::(mean), @@ -223,7 +224,7 @@ fn reference_identity_distinguishes_equal_results() { fn reference_ratios_handle_zero_times() { let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; for result in &mut results { - let measurement = result.measurements.measurements[0]; + let measurement = result.measurements.measurements[0].clone(); result.measurements.measurements.push(measurement); } let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index db036d6..c1297f0 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -89,11 +89,7 @@ impl<'a> Scheduler<'a> { let reference = annotated_results.iter().find(|r| r.is_reference).unwrap(); let others = annotated_results.iter().filter(|r| !r.is_reference); - console_writeln!( - stdout, - " {} ran", - reference.result.command_with_unused_parameters.cyan() - )?; + console_writeln!(stdout, " {} ran", reference.result.display_name.cyan())?; for item in others { let stddev = if let Some(stddev) = item.relative_speed_stddev { @@ -122,7 +118,7 @@ impl<'a> Scheduler<'a> { stdout, "{} {}", comparator, - item.result.command_with_unused_parameters.magenta() + &item.result.display_name.magenta() )?; } } @@ -146,7 +142,7 @@ impl<'a> Scheduler<'a> { } else { " ".into() }, - item.result.command_with_unused_parameters, + item.result.display_name, relationship, )?; } @@ -205,39 +201,61 @@ fn generate_results(args: &[&'static str]) -> Result> { fn scheduler_basic() -> Result<()> { insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#" - command: sleep 0.123 - mean: 0.123 - stddev: 0 - median: 0.123 - user: 0 - system: 0 - min: 0.123 - max: 0.123 - times: - - 0.123 - - 0.123 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + measurements: + - time_wall_clock: + value: 0.123 + unit: second + time_user: + value: 0 + unit: second + time_system: + value: 0 + unit: second + memory_peak_resident: + value: 0 + unit: byte + exit_code: 0 + - time_wall_clock: + value: 0.123 + unit: second + time_user: + value: 0 + unit: second + time_system: + value: 0 + unit: second + memory_peak_resident: + value: 0 + unit: byte + exit_code: 0 - command: sleep 0.456 - mean: 0.456 - stddev: 0 - median: 0.456 - user: 0 - system: 0 - min: 0.456 - max: 0.456 - times: - - 0.456 - - 0.456 - memory_usage_byte: - - 0 - - 0 - exit_codes: - - 0 - - 0 + measurements: + - time_wall_clock: + value: 0.456 + unit: second + time_user: + value: 0 + unit: second + time_system: + value: 0 + unit: second + memory_peak_resident: + value: 0 + unit: byte + exit_code: 0 + - time_wall_clock: + value: 0.456 + unit: second + time_user: + value: 0 + unit: second + time_system: + value: 0 + unit: second + memory_peak_resident: + value: 0 + unit: byte + exit_code: 0 "#); Ok(()) diff --git a/src/export/csv.rs b/src/export/csv.rs index 15cc3a5..5a02fdc 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -16,7 +16,7 @@ impl Exporter for CsvExporter { fn serialize( &self, results: &[BenchmarkResult], - _unit: Option, + _time_unit: Option, _sort_order: SortOrder, _reference_index: Option, ) -> Result> { @@ -42,7 +42,7 @@ impl Exporter for CsvExporter { } for res in results { - let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; + let mut fields = vec![Cow::Borrowed(res.get_name().as_bytes())]; for f in &[ res.mean_wall_clock_time(), res.measurements.stddev().unwrap_or_default(), @@ -80,28 +80,29 @@ fn test_csv() { let results = vec![ BenchmarkResult { - command: String::from("command_a"), - command_with_unused_parameters: String::from("command_a"), + command: String::from("echo command_a"), + name: Some(String::from("command_a")), + display_name: String::from("command_a"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(7.0), time_user: Time::new::(7.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(8.0), time_user: Time::new::(8.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(12.0), time_user: Time::new::(12.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -111,14 +112,12 @@ fn test_csv() { "foo".into(), Parameter { value: "one".into(), - is_unused: false, }, ); params.insert( "bar".into(), Parameter { value: "two".into(), - is_unused: false, }, ); params @@ -126,27 +125,28 @@ fn test_csv() { }, BenchmarkResult { command: String::from("command_b"), - command_with_unused_parameters: String::from("command_b"), + name: None, + display_name: String::from("command_b"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(17.0), time_user: Time::new::(17.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(18.0), time_user: Time::new::(18.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(19.0), time_user: Time::new::(19.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -156,14 +156,12 @@ fn test_csv() { "foo".into(), Parameter { value: "one".into(), - is_unused: false, }, ); params.insert( "bar".into(), Parameter { value: "seven".into(), - is_unused: false, }, ); params diff --git a/src/export/json.rs b/src/export/json.rs index c5b6a84..e4c252b 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -1,30 +1,141 @@ +use std::ops::{Add, AddAssign, Div}; + use serde::*; use serde_json::to_vec_pretty; use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; +use crate::benchmark::measurement::Measurements; use crate::options::SortOrder; -use crate::quantity::TimeUnit; +use crate::quantity::statistics::{max, mean, median, min, standard_deviation, UnsafeRawValue}; +use crate::quantity::{byte, second, QuantityInUnit, Ratio, TimeUnit, Zero}; use anyhow::Result; -#[derive(Serialize, Debug)] +mod metadata; +use metadata::Metadata; + +const JSON_SCHEMA_VERSION: u32 = 2; + +#[derive(Serialize)] struct HyperfineSummary<'a> { - results: &'a [BenchmarkResult], + schema_version: u32, + metadata: &'a Metadata, + results: Vec>, +} + +#[derive(Serialize)] +struct JsonBenchmarkResult<'a> { + #[serde(flatten)] + result: &'a BenchmarkResult, + #[serde(skip_serializing_if = "std::ops::Not::not")] + is_reference: bool, + summary: BenchmarkSummary, +} + +#[derive(Serialize)] +struct BenchmarkSummary { + time_wall_clock: StatisticalSummary, + time_user: StatisticalSummary, + time_system: StatisticalSummary, + #[serde(skip_serializing_if = "Option::is_none")] + memory_peak_resident: Option, +} + +impl BenchmarkSummary { + fn from_measurements(measurements: &Measurements) -> Self { + let values = &measurements.measurements; + let mut memory_values = values + .iter() + .filter_map(|m| m.memory_peak_resident) + .peekable(); + Self { + time_wall_clock: StatisticalSummary::from_values( + values.iter().map(|m| m.time_wall_clock), + second, + ), + time_user: StatisticalSummary::from_values(values.iter().map(|m| m.time_user), second), + time_system: StatisticalSummary::from_values( + values.iter().map(|m| m.time_system), + second, + ), + memory_peak_resident: memory_values + .peek() + .is_some() + .then(|| StatisticalSummary::from_values(memory_values, byte)), + } + } +} + +#[derive(Serialize)] +struct StatisticalSummary { + unit: &'static str, + count: usize, + mean: f64, + stddev: Option, + median: f64, + min: f64, + max: f64, +} + +impl StatisticalSummary { + fn from_values(values: impl IntoIterator, _unit: U) -> Self + where + Q: Copy + + PartialOrd + + Add + + AddAssign + + Zero + + Div + + UnsafeRawValue + + QuantityInUnit, + P: Into, + U: uom::si::Unit, + { + let values: Vec<_> = values.into_iter().collect(); + assert!( + !values.is_empty(), + "Statistical summaries require at least one measurement" + ); + Self { + unit: U::singular(), + count: values.len(), + mean: mean(values.iter().copied()).value_in_unit(), + stddev: (values.len() > 1) + .then(|| standard_deviation(values.iter().copied()).value_in_unit()), + median: median(values.iter().copied()).value_in_unit(), + min: min(values.iter().copied()).value_in_unit(), + max: max(values.iter().copied()).value_in_unit(), + } + } } #[derive(Default)] -pub struct JsonExporter {} +pub struct JsonExporter { + metadata: Metadata, +} impl Exporter for JsonExporter { fn serialize( &self, results: &[BenchmarkResult], - _unit: Option, + _time_unit: Option, _sort_order: SortOrder, - _reference_index: Option, + reference_index: Option, ) -> Result> { - let mut output = to_vec_pretty(&HyperfineSummary { results }); + let mut output = to_vec_pretty(&HyperfineSummary { + schema_version: JSON_SCHEMA_VERSION, + metadata: &self.metadata, + results: results + .iter() + .enumerate() + .map(|(index, result)| JsonBenchmarkResult { + result, + is_reference: reference_index == Some(index), + summary: BenchmarkSummary::from_measurements(&result.measurements), + }) + .collect(), + }); if let Ok(ref mut content) = output { content.push(b'\n'); } @@ -32,3 +143,33 @@ impl Exporter for JsonExporter { Ok(output?) } } + +#[test] +fn test_statistical_summary_unit_conversion() { + use crate::quantity::{kibibyte, millisecond, Information, Time}; + + let time = StatisticalSummary::from_values( + [Time::new::(1.0), Time::new::(3.0)], + millisecond, + ); + let information = StatisticalSummary::from_values( + [ + Information::new::(1024.0), + Information::new::(3072.0), + ], + kibibyte, + ); + + for (summary, unit, scale) in [ + (time, "millisecond", 1000.0), + (information, "kibibyte", 1.0), + ] { + assert_eq!(summary.unit, unit); + assert_eq!(summary.count, 2); + approx::assert_relative_eq!(summary.mean, 2.0 * scale); + approx::assert_relative_eq!(summary.median, 2.0 * scale); + approx::assert_relative_eq!(summary.min, scale); + approx::assert_relative_eq!(summary.max, 3.0 * scale); + approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale); + } +} diff --git a/src/export/json/metadata.rs b/src/export/json/metadata.rs new file mode 100644 index 0000000..93b3161 --- /dev/null +++ b/src/export/json/metadata.rs @@ -0,0 +1,51 @@ +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub(super) struct Metadata { + hyperfine_version: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub start_time: Option, + pub platform: Platform, +} + +impl Default for Metadata { + fn default() -> Self { + Self { + hyperfine_version: env!("CARGO_PKG_VERSION"), + start_time: utc_now(), + platform: Platform::detect(), + } + } +} + +fn utc_now() -> Option { + use std::{fmt::Write, time::SystemTime}; + + let now = SystemTime::now(); + now.duration_since(SystemTime::UNIX_EPOCH).ok()?; + let mut timestamp = String::new(); + write!(&mut timestamp, "{}", humantime::format_rfc3339_seconds(now)).ok()?; + Some(timestamp) +} + +#[derive(Debug, Default, Serialize)] +pub(super) struct Platform { + #[serde(skip_serializing_if = "Option::is_none")] + os: Option, + #[serde(skip_serializing_if = "Option::is_none")] + architecture: Option, +} + +impl Platform { + fn detect() -> Self { + use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI}; + + let Ok(info) = PlatformInfo::new() else { + return Self::default(); + }; + Self { + os: Some(info.sysname().to_string_lossy().into_owned()), + architecture: Some(info.machine().to_string_lossy().into_owned()), + } + } +} diff --git a/src/export/markup.rs b/src/export/markup.rs index 0eea533..a506098 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -16,12 +16,12 @@ pub trait MarkupExporter { fn table_results( &self, entries: &[BenchmarkResultWithRelativeSpeed], - unit: TimeUnit, + time_unit: TimeUnit, reference_pending: bool, explicit_reference: bool, ) -> String { // prepare table header strings - let notation = format!("[{}]", unit.short_name()); + let notation = format!("[{}]", time_unit.short_name()); // prepare table cells alignment let cells_alignment = [ @@ -48,25 +48,23 @@ pub trait MarkupExporter { table.push_str(&self.table_divider(&cells_alignment)); for entry in entries { - let measurement = &entry.result; + let result = &entry.result; // prepare data row strings - let cmd_str = measurement - .command_with_unused_parameters - .replace('|', "\\|"); - let mean_str = measurement.mean_wall_clock_time().format_value(unit); - let stddev_str = if let Some(stddev) = measurement.measurements.stddev() { - format!(" ± {}", stddev.format_value(unit)) + let cmd_str = result.display_name.replace('|', "\\|"); + let mean_str = result.mean_wall_clock_time().format_value(time_unit); + let stddev_str = if let Some(stddev) = result.measurements.stddev() { + format!(" ± {}", stddev.format_value(time_unit)) } else { "".into() }; - let min_str = measurement.measurements.min().format_value(unit); - let max_str = measurement.measurements.max().format_value(unit); + let min_str = result.measurements.min().format_value(time_unit); + let max_str = result.measurements.max().format_value(time_unit); // The ratio of two zero times is undefined, even if they compare equal. let relative_unavailable = reference_pending || (explicit_reference && !entry.is_reference && entry.relative_ordering == Ordering::Equal - && measurement.mean_wall_clock_time() == Time::zero()); + && result.mean_wall_clock_time() == Time::zero()); let relative = if relative_unavailable { "N/A".to_string() } else { @@ -127,11 +125,11 @@ impl Exporter for T { fn serialize( &self, results: &[BenchmarkResult], - unit: Option, + time_unit: Option, sort_order: SortOrder, reference_index: Option, ) -> Result> { - let unit = unit.unwrap_or_else(|| determine_unit_from_results(results)); + let unit = time_unit.unwrap_or_else(|| determine_unit_from_results(results)); // Do not report ratios against another benchmark while the selected // reference is still pending in an intermediate export. let reference_pending = reference_index.is_some_and(|i| i >= results.len()); diff --git a/src/export/mod.rs b/src/export/mod.rs index c8dd9a0..0d3c67d 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -49,7 +49,7 @@ trait Exporter { fn serialize( &self, results: &[BenchmarkResult], - unit: Option, + time_unit: Option, sort_order: SortOrder, reference_index: Option, ) -> Result>; diff --git a/src/export/tests.rs b/src/export/tests.rs index abe340a..d99d51a 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -28,27 +28,28 @@ fn test_markup_export_auto_ms() { let results = [ BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), + name: None, + display_name: String::from("sleep 0.1"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.09), time_user: Time::new::(0.09), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.10), time_user: Time::new::(0.10), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.14), time_user: Time::new::(0.14), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -56,27 +57,28 @@ fn test_markup_export_auto_ms() { }, BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), + name: None, + display_name: String::from("sleep 2"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.0), time_user: Time::new::(2.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(3.0), time_user: Time::new::(3.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(4.0), time_user: Time::new::(4.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -129,27 +131,28 @@ fn test_markup_export_auto_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), + name: None, + display_name: String::from("sleep 2"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.1), time_user: Time::new::(2.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.2), time_user: Time::new::(2.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.3), time_user: Time::new::(2.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -157,27 +160,28 @@ fn test_markup_export_auto_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), + name: None, + display_name: String::from("sleep 0.1"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.1), time_user: Time::new::(0.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.2), time_user: Time::new::(0.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.3), time_user: Time::new::(0.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -230,27 +234,28 @@ fn test_markup_export_manual_ms() { let timing_results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), + name: None, + display_name: String::from("sleep 2"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.1), time_user: Time::new::(2.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.2), time_user: Time::new::(2.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.3), time_user: Time::new::(2.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -258,27 +263,28 @@ fn test_markup_export_manual_ms() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), + name: None, + display_name: String::from("sleep 0.1"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.1), time_user: Time::new::(0.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.2), time_user: Time::new::(0.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.3), time_user: Time::new::(0.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -330,27 +336,28 @@ fn test_markup_export_manual_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_with_unused_parameters: String::from("sleep 2"), + name: None, + display_name: String::from("sleep 2"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.01), time_user: Time::new::(2.01), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.02), time_user: Time::new::(2.02), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.03), time_user: Time::new::(2.03), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -358,27 +365,28 @@ fn test_markup_export_manual_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_with_unused_parameters: String::from("sleep 0.1"), + name: None, + display_name: String::from("sleep 0.1"), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.11), time_user: Time::new::(0.11), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.12), time_user: Time::new::(0.12), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.13), time_user: Time::new::(0.13), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + memory_peak_resident: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), diff --git a/src/quantity/mod.rs b/src/quantity/mod.rs index 5844474..98ed20e 100644 --- a/src/quantity/mod.rs +++ b/src/quantity/mod.rs @@ -1,5 +1,8 @@ use std::marker::PhantomData; +use serde::ser::SerializeStruct; +use serde::Serializer; + use uom::si; pub use si::f64::{Information, Ratio, Time}; @@ -13,6 +16,24 @@ pub use units::{InformationUnit, IsUnit, TimeUnit}; pub mod statistics; mod units; +pub trait QuantityInUnit { + fn value_in_unit(self) -> f64; +} + +impl> QuantityInUnit for Time { + fn value_in_unit(self) -> f64 { + self.get::() + } +} + +impl> QuantityInUnit + for Information +{ + fn value_in_unit(self) -> f64 { + self.get::() + } +} + pub trait FormatQuantity { type Unit; @@ -105,6 +126,26 @@ impl FormatQuantity for Information { } } +pub fn serialize_time(t: &Time, s: S) -> Result +where + S: Serializer, +{ + let mut state = s.serialize_struct("Time", 2)?; + state.serialize_field("value", &t.get::())?; + state.serialize_field("unit", "second")?; + state.end() +} + +pub fn serialize_information(i: &Information, s: S) -> Result +where + S: Serializer, +{ + let mut state = s.serialize_struct("Information", 2)?; + state.serialize_field("value", &i.get::())?; + state.serialize_field("unit", "byte")?; + state.end() +} + #[test] fn test_time() { let time = Time::new::(123.4); @@ -181,11 +222,11 @@ fn test_format() { assert_eq!(time.format_auto(), "123.4 ms"); assert_eq!(time.format(TimeUnit::MicroSecond), "123400.0 µs"); - let peak_memory_usage = Information::new::(8.); - assert_eq!(peak_memory_usage.format_auto(), "8.0 KiB"); - assert_eq!(peak_memory_usage.format(InformationUnit::Byte), "8192 B"); + let memory_peak_resident = Information::new::(8.); + assert_eq!(memory_peak_resident.format_auto(), "8.0 KiB"); + assert_eq!(memory_peak_resident.format(InformationUnit::Byte), "8192 B"); assert_eq!( - peak_memory_usage.format(InformationUnit::KibiByte), + memory_peak_resident.format(InformationUnit::KibiByte), "8.0 KiB" ); } diff --git a/src/timer/mod.rs b/src/timer/mod.rs index be00286..4d38a8a 100644 --- a/src/timer/mod.rs +++ b/src/timer/mod.rs @@ -82,14 +82,14 @@ pub fn execute_and_measure(mut command: Command) -> Result { discard(output); } - let (time_user, time_system, peak_memory_usage, exit_status) = cpu_timer.stop(child)?; + let (time_user, time_system, memory_peak_resident, exit_status) = cpu_timer.stop(child)?; let time_wall_clock = wallclock_timer.stop(); Ok(Measurement { time_wall_clock, time_user, time_system, - peak_memory_usage, + memory_peak_resident, exit_status, }) } diff --git a/src/timer/unix_timer.rs b/src/timer/unix_timer.rs index 699a541..63591ee 100644 --- a/src/timer/unix_timer.rs +++ b/src/timer/unix_timer.rs @@ -77,12 +77,12 @@ impl CPUTimer { Self {} } - pub fn stop(&self, child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, child: Child) -> Result<(Time, Time, Option, ExitStatus)> { let (status, usage) = wait4(child)?; Ok(( usage.time_user, usage.time_system, - usage.memory_usage, + Some(usage.memory_usage), status, )) } diff --git a/src/timer/windows_timer.rs b/src/timer/windows_timer.rs index e48e288..3a79e40 100644 --- a/src/timer/windows_timer.rs +++ b/src/timer/windows_timer.rs @@ -84,7 +84,7 @@ impl CPUTimer { Self { job_object } } - pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Option, ExitStatus)> { let status = child.wait()?; let mut job_object_info = @@ -116,9 +116,9 @@ impl CPUTimer { let system_time = Time::new::((job_object_info.TotalKernelTime as f64) * 100.0); - Ok((user_time, system_time, Information::zero(), status)) + Ok((user_time, system_time, None, status)) } else { - Ok((Time::zero(), Time::zero(), Information::zero(), status)) + Ok((Time::zero(), Time::zero(), None, status)) } } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 42c23ab..4b0245a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -24,6 +24,17 @@ fn snapshot_settings() -> insta::Settings { settings } +fn json_snapshot_settings() -> insta::Settings { + let mut settings = snapshot_settings(); + for field in ["hyperfine_version", "start_time", "os", "architecture"] { + settings.add_filter( + &format!(r#"("{field}": ")[^"]+(")"#), + format!(r#"${{1}}[{field}]$2"#), + ); + } + settings +} + #[test] fn runs_successfully() { hyperfine() @@ -916,7 +927,7 @@ fn selects_reference_with_parameterized_prepare() { assert_eq!( results .iter() - .map(|result| result["parameters"]["delay"].as_str().unwrap()) + .map(|result| result["parameters"]["delay"]["value"].as_str().unwrap()) .collect::>(), ["0.2", "0.4", "0.6"] ); @@ -1273,3 +1284,46 @@ fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() { ] ); } + +#[test] +fn json_export_basic() { + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=2", + "--warmup=1", + "--reference-name=one second", + "--reference=sleep 1", + "--command-name=sleep 2", + "sleep 2", + ])); +} + +#[test] +fn json_export_single_run() { + // Make sure that the standard deviation is set to `null` + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=1", + "sleep 1", + ])); +} + +#[test] +fn json_export_parameterized_with_reference() { + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=2", + "-L", + "duration", + "1,2", + "--command-name=sleep for {duration} seconds", + "--reference=sleep for 2 seconds", + "sleep {duration}", + ])); +} diff --git a/tests/snapshots/integration_tests__json_export_basic.snap b/tests/snapshots/integration_tests__json_export_basic.snap new file mode 100644 index 0000000..372f79f --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_basic.snap @@ -0,0 +1,199 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=2" + - "--warmup=1" + - "--reference-name=one second" + - "--reference=sleep 1" + - "--command-name=sleep 2" + - sleep 2 +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "name": "one second", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "is_reference": true, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 1.0, + "stddev": 0.0, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + }, + { + "command": "sleep 2", + "measurements": [ + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 2.0, + "stddev": 0.0, + "median": 2.0, + "min": 2.0, + "max": 2.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr ----- diff --git a/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap b/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap new file mode 100644 index 0000000..dc91fdd --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap @@ -0,0 +1,211 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=2" + - "-L" + - duration + - "1,2" + - "--command-name=sleep for {duration} seconds" + - "--reference=sleep for 2 seconds" + - "sleep {duration}" +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "name": "sleep for 1 seconds", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "parameters": { + "duration": { + "value": "1" + } + }, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 1.0, + "stddev": 0.0, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + }, + { + "command": "sleep 2", + "name": "sleep for 2 seconds", + "measurements": [ + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "parameters": { + "duration": { + "value": "2" + } + }, + "is_reference": true, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 2.0, + "stddev": 0.0, + "median": 2.0, + "min": 2.0, + "max": 2.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr ----- diff --git a/tests/snapshots/integration_tests__json_export_single_run.snap b/tests/snapshots/integration_tests__json_export_single_run.snap new file mode 100644 index 0000000..7f2671c --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_single_run.snap @@ -0,0 +1,93 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=1" + - sleep 1 +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 1, + "mean": 1.0, + "stddev": null, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr -----