vsndrg/hyperfine
1
0 Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-10-28 08:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #790 from sharkdp/new-metrics

Browse Source 
New JSON format
This commit is contained in:
David Peter authored and GitHub committed 2026-10-05 21:20:56 +02:00
2 parents ad6b4e2b1e 3cf3c5d3f2
commit f26c015518
30 files changed
+1157 -299
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cargo.lock
Generated
+18
View File
@@ -369,6 +369,12 @@ version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "humantime"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
[[package]]
name = "hyperfine"
version = "1.21.0"
@@ -380,12 +386,14 @@ dependencies = [
"clap_complete",
"colored",
"csv",
"humantime",
"indicatif",
"insta",
"insta-cmd",
"libc",
"nix",
"once_cell",
"platform-info",
"predicates",
"rand 0.10.3",
"rust_decimal",
@@ -531,6 +539,16 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "platform-info"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba7c3f5af00d3596a148cc80178ac690152b71c283d15a8bcea0cddfea759a36"
dependencies = [
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "portable-atomic"
version = "1.15.0"
Cargo.toml
+2
View File
@@ -29,6 +29,8 @@ shell-words = "1.1"
thiserror = "2.0"
anyhow = "1.0"
uom = { version = "0.36.0", default-features = false, features = ["std", "si", "f64"] }
humantime = "2"
platform-info = "2.2"
[target.'cfg(not(windows))'.dependencies]
libc = "0.2"
README.md
+22
View File
@@ -158,6 +158,28 @@ You can use the `--export-markdown <file>` option to create tables like the foll
#### JSON
The JSON export includes the following metrics for each measured run (excluding warmup runs):
- **`time_wall_clock`**: Time from start to finish, including time spent waiting, in seconds.
- **`time_user`**: CPU time spent running the program's code, summed across threads, in seconds.
- Linux/macOS: Includes child-process time when parents wait for their children to finish.
- Windows: Includes the command and its child processes.
- **`time_system`**: CPU time spent running operating-system code for the program, for example
to read files, summed across threads, in seconds.
- Linux/macOS: Includes child-process time when parents wait for their children to finish.
- Windows: Includes the command and its child processes.
- **`memory_peak_resident`**: Peak memory held in physical RAM, in bytes.
- Linux/macOS: Peak resident set size (RSS). This is the largest per-process peak among the
command and child processes (whose usage is collected when their parents wait for them),
*not the simultaneous total memory of the full process tree*.
- Windows: Currently not supported.
The JSON output is useful if you want to analyze the benchmark results in more detail. The
[`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot
of helpful Python programs to further analyze benchmark results and create helpful
scripts/advanced_statistics.py
+13 -12
View File
@@ -50,11 +50,9 @@ unit_str = str(unit)
with open(args.file) as f:
results = json.load(f)["results"]
commands = [b["command"] for b in results]
times = [b["times"] for b in results]
for command, ts in zip(commands, times):
ts = [t * unit.factor() for t in ts]
for result in results:
summary = result["summary"]["time_wall_clock"]
ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]]
p05 = np.percentile(ts, 5)
p25 = np.percentile(ts, 25)
@@ -63,13 +61,16 @@ for command, ts in zip(commands, times):
iqr = p75 - p25
print(f"Command '{command}'")
print(f" runs: {len(ts):8d}")
print(f" mean: {np.mean(ts):8.3f} {unit_str}")
print(f" stddev: {np.std(ts, ddof=1):8.3f} {unit_str}")
print(f" median: {np.median(ts):8.3f} {unit_str}")
print(f" min: {np.min(ts):8.3f} {unit_str}")
print(f" max: {np.max(ts):8.3f} {unit_str}")
print(f"Command '{result.get('name', result['command'])}'")
print(f" runs: {summary['count']:8d}")
print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}")
if summary["stddev"] is None:
print(f" stddev: {'N/A':>8}")
else:
print(f" stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}")
print(f" median: {summary['median'] * unit.factor():8.3f} {unit_str}")
print(f" min: {summary['min'] * unit.factor():8.3f} {unit_str}")
print(f" max: {summary['max'] * unit.factor():8.3f} {unit_str}")
print()
print(" percentiles:")
print(f" P_05 .. P_95: {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}")
scripts/plot_benchmark_comparison.py
+2 -2
View File
@@ -44,14 +44,14 @@ if args.benchmark_names:
for i, filename in enumerate(args.files):
with open(filename) as f:
results = json.load(f)["results"]
benchmark_commands = [b["command"] for b in results]
benchmark_commands = [b.get("name", b["command"]) for b in results]
if commands is None:
commands = benchmark_commands
else:
assert (
commands == benchmark_commands
), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
data.append([round(b["mean"], 2) for b in results])
data.append([b["summary"]["time_wall_clock"]["mean"] for b in results])
if args.benchmark_names:
inputs.append(args.benchmark_names[i])
else:
scripts/plot_histogram.py
+4 -2
View File
@@ -66,8 +66,10 @@ with open(args.file) as f:
if args.labels:
labels = args.labels.split(",")
else:
labels = [b["command"] for b in results]
all_times = [b["times"] for b in results]
labels = [b.get("name", b["command"]) for b in results]
all_times = [
[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results
]
t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
scripts/plot_parametrized.py
+6 -3
View File
@@ -71,7 +71,7 @@ def unique_parameter(benchmark):
f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
)
[(name, value)] = params_dict.items()
return (name, float(value))
return (name, float(value["value"]))
parameter_name = None
@@ -87,8 +87,11 @@ for filename in args.file:
)
parameter_name = this_parameter_name
times_mean = [b["mean"] for b in results]
times_stddev = [b["stddev"] for b in results]
summaries = [b["summary"]["time_wall_clock"] for b in results]
times_mean = [s["mean"] for s in summaries]
times_stddev = [
s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries
]
plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)
scripts/plot_progression.py
+6 -4
View File
@@ -52,8 +52,8 @@ with open(args.file) as f:
results = json.load(f)["results"]
for result in results:
label = result["command"]
times = result["times"]
label = result.get("name", result["command"])
times = [m["time_wall_clock"]["value"] for m in result["measurements"]]
num = len(times)
nums = range(num)
@@ -63,7 +63,9 @@ for result in results:
if not args.no_moving_average:
moving_average_width = (
num // 5 if args.moving_average_width is None else args.moving_average_width
max(1, num // 5)
if args.moving_average_width is None
else args.moving_average_width
)
average = moving_average(times, moving_average_width)
@@ -74,7 +76,7 @@ if args.title:
legend = []
for result in results:
legend.append(result["command"])
legend.append(result.get("name", result["command"]))
if not args.no_moving_average:
legend.append("moving average")
plt.legend(legend)
scripts/plot_whisker.py
+3 -3
View File
@@ -37,11 +37,11 @@ with open(args.file, encoding="utf-8") as f:
if args.labels:
labels = args.labels.split(",")
else:
labels = [b["command"] for b in results]
times = [b["times"] for b in results]
labels = [b.get("name", b["command"]) for b in results]
times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results]
if args.sort_by == "median":
medians = [b["median"] for b in results]
medians = [b["summary"]["time_wall_clock"]["median"] for b in results]
indices = sorted(range(len(labels)), key=lambda k: medians[k])
labels = [labels[i] for i in indices]
times = [times[i] for i in indices]
scripts/welch_ttest.py
+2 -2
View File
@@ -26,8 +26,8 @@ if len(results) != 2:
print("The input file has to contain exactly two benchmarks")
sys.exit(1)
a, b = (x["command"] for x in results[:2])
X, Y = (x["times"] for x in results[:2])
a, b = (x.get("name", x["command"]) for x in results[:2])
X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2])
print(f"Command 1: {a}")
print(f"Command 2: {b}\n")
src/benchmark/benchmark_result.rs
+28 -70
View File
@@ -1,97 +1,55 @@
use std::collections::BTreeMap;
use serde::ser::SerializeStruct;
use serde::{Serialize, Serializer};
use serde::Serialize;
use crate::benchmark::measurement::Measurements;
use crate::quantity::{byte, second, Time};
use crate::util::exit_code::extract_exit_code;
use crate::quantity::Time;
/// Parameter value and whether it was used in the command line template
/// Parameter value
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
#[serde(transparent)]
pub struct Parameter {
pub value: String,
#[serde(skip)]
pub is_unused: bool,
}
/// Meta data and performance metrics for a single benchmark
#[derive(Debug, Default, Clone, PartialEq)]
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
pub struct BenchmarkResult {
/// The full command line of the program that is being benchmarked
/// The command line being benchmarked, after parameter substitution.
/// For example, `sleep {duration}` with `duration=1` becomes `sleep 1`,
/// regardless of any custom name.
pub command: String,
/// The full command line, including parameters not used in the command template.
pub command_with_unused_parameters: String,
/// The custom name after parameter substitution, if it differs from `command`.
/// For example, `--command-name="wait {duration}s"` with `duration=1`
/// produces `Some("wait 1s")`. Without a custom name, or when the expanded
/// name equals `command`, this is `None`.
#[serde(skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
/// Performance measurements and exit statuses for each run
/// The custom name (or `command`), followed by parameters not used in the
/// command template, in command-line order. For example, `sleep {duration}`
/// with `duration=1`, `branch=main`, and the custom name `wait {duration}s`
/// produces `wait 1s (branch = main)`. Without the custom name, this is
/// `sleep 1 (branch = main)`. Used in summaries and table exports.
#[serde(skip_serializing)]
pub display_name: String,
/// Performance metric measurements and exit codes for each run
#[serde(flatten)]
pub measurements: Measurements,
/// Parameter values for this benchmark
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub parameters: BTreeMap<String, Parameter>,
}
impl BenchmarkResult {
pub fn get_name(&self) -> &str {
self.name.as_deref().unwrap_or(&self.command)
}
/// The average wall clock time
pub fn mean_wall_clock_time(&self) -> Time {
self.measurements.time_wall_clock_mean()
}
}
// Preserve the existing export format while storing typed measurements internally.
impl Serialize for BenchmarkResult {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct(
"BenchmarkResult",
if self.parameters.is_empty() { 11 } else { 12 },
)?;
state.serialize_field("command", &self.command)?;
state.serialize_field("mean", &self.mean_wall_clock_time().get::<second>())?;
state.serialize_field(
"stddev",
&self.measurements.stddev().map(|time| time.get::<second>()),
)?;
state.serialize_field("median", &self.measurements.median().get::<second>())?;
state.serialize_field("user", &self.measurements.time_user_mean().get::<second>())?;
state.serialize_field(
"system",
&self.measurements.time_system_mean().get::<second>(),
)?;
state.serialize_field("min", &self.measurements.min().get::<second>())?;
state.serialize_field("max", &self.measurements.max().get::<second>())?;
state.serialize_field(
"times",
&self
.measurements
.wall_clock_times()
.map(|time| time.get::<second>())
.collect::<Vec<_>>(),
)?;
state.serialize_field(
"memory_usage_byte",
&self
.measurements
.measurements
.iter()
.map(|measurement| measurement.peak_memory_usage.get::<byte>() as u64)
.collect::<Vec<_>>(),
)?;
state.serialize_field(
"exit_codes",
&self
.measurements
.measurements
.iter()
.map(|measurement| extract_exit_code(measurement.exit_status))
.collect::<Vec<_>>(),
)?;
if !self.parameters.is_empty() {
state.serialize_field("parameters", &self.parameters)?;
}
state.end()
}
}
src/benchmark/executor.rs
+31 -21
View File
@@ -2,6 +2,8 @@
use std::os::windows::process::CommandExt;
use std::process::ExitStatus;
use crate::benchmark::measurement::Measurement;
use crate::benchmark::measurement::Measurements;
use crate::command::Command;
use crate::options::{
CmdFailureAction, CommandInputPolicy, CommandOutputPolicy, Options, OutputStyleOption, Shell,
@@ -11,8 +13,6 @@ use crate::quantity::{second, Information, Time, Zero};
use crate::timer::execute_and_measure;
use crate::util::randomized_environment_offset;
use super::measurement::{Measurement, Measurements};
use anyhow::{bail, Context, Result};
#[derive(Clone, Copy)]
@@ -76,10 +76,10 @@ fn run_command_and_measure_common(
command.env("HYPERFINE_ITERATION", value);
}
let result = execute_and_measure(command)
let measurement = execute_and_measure(command)
.with_context(|| format!("Failed to run command '{command_name}'"))?;
if !result.exit_status.success() {
if !measurement.exit_status.success() {
use crate::util::exit_code::extract_exit_code;
let should_fail = match command_failure_action {
@@ -87,7 +87,7 @@ fn run_command_and_measure_common(
CmdFailureAction::IgnoreAllFailures => false,
CmdFailureAction::IgnoreSpecificFailures(ref codes) => {
// Only fail if the exit code is not in the list of codes to ignore
if let Some(exit_code) = extract_exit_code(result.exit_status) {
if let Some(exit_code) = extract_exit_code(measurement.exit_status) {
!codes.contains(&exit_code)
} else {
// If we can't extract an exit code, treat it as a failure
@@ -107,7 +107,7 @@ fn run_command_and_measure_common(
bail!(
"{cause} in {when}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
Alternatively, use the '--show-output' option to debug what went wrong.",
cause=result.exit_status.code().map_or(
cause=measurement.exit_status.code().map_or(
"The process has been terminated by a signal".into(),
|c| format!("Command terminated with non-zero exit code {c}")
@@ -116,7 +116,7 @@ fn run_command_and_measure_common(
}
}
Ok(result)
Ok(measurement)
}
pub struct RawExecutor<'a> {
@@ -192,7 +192,7 @@ impl Executor for ShellExecutor<'_> {
command_builder.arg(command.get_command_line());
}
let mut result = run_command_and_measure_common(
let mut measurement = run_command_and_measure_common(
command_builder,
iteration,
command_failure_action.unwrap_or_else(|| self.options.command_failure_action.clone()),
@@ -202,14 +202,24 @@ impl Executor for ShellExecutor<'_> {
)?;
// Subtract shell spawning time
if let Some(spawning_time) = self.shell_spawning_time {
result.time_wall_clock =
(result.time_wall_clock - spawning_time.time_wall_clock).max(Time::zero());
result.time_user = (result.time_user - spawning_time.time_user).max(Time::zero());
result.time_system = (result.time_system - spawning_time.time_system).max(Time::zero());
fn ensure_non_negative(time: Time) -> Time {
if time < Time::zero() {
Time::zero()
} else {
time
}
}
Ok(result)
if let Some(ref spawning_time) = self.shell_spawning_time {
measurement.time_wall_clock =
ensure_non_negative(measurement.time_wall_clock - spawning_time.time_wall_clock);
measurement.time_user =
ensure_non_negative(measurement.time_user - spawning_time.time_user);
measurement.time_system =
ensure_non_negative(measurement.time_system - spawning_time.time_system);
}
Ok(measurement)
}
/// Measure the average shell spawning time
@@ -229,14 +239,14 @@ impl Executor for ShellExecutor<'_> {
for _ in 0..COUNT {
// Just run the shell without any command
let res = self.run_command_and_measure(
let measurement = self.run_command_and_measure(
&Command::new(None, ""),
BenchmarkIteration::NonBenchmarkRun,
None,
&CommandOutputPolicy::Null,
);
match res {
match measurement {
Err(_) => {
let shell_cmd = if cfg!(windows) {
format!("{} /C \"\"", self.shell)
@@ -249,8 +259,8 @@ impl Executor for ShellExecutor<'_> {
shell_cmd
);
}
Ok(r) => {
measurements.push(r);
Ok(result) => {
measurements.push(result);
}
}
@@ -267,7 +277,7 @@ impl Executor for ShellExecutor<'_> {
time_wall_clock: measurements.time_wall_clock_mean(),
time_user: measurements.time_user_mean(),
time_system: measurements.time_system_mean(),
peak_memory_usage: measurements.peak_memory_usage_mean(),
memory_peak_resident: None,
exit_status: ExitStatus::default(),
});
@@ -275,7 +285,7 @@ impl Executor for ShellExecutor<'_> {
}
fn time_overhead(&self) -> Time {
self.shell_spawning_time.unwrap().time_wall_clock
self.shell_spawning_time.as_ref().unwrap().time_wall_clock
}
}
@@ -325,7 +335,7 @@ impl Executor for MockExecutor {
time_wall_clock: Self::extract_time(command.get_command_line()),
time_user: Time::zero(),
time_system: Time::zero(),
peak_memory_usage: Information::zero(),
memory_peak_resident: Some(Information::zero()),
exit_status,
})
}
src/benchmark/measurement.rs
+50 -21
View File
@@ -1,29 +1,62 @@
use std::process::ExitStatus;
use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation};
use crate::quantity::{Information, Time};
use serde::Serialize;
/// Performance measurements and exit status from running a single command
#[derive(Debug, Default, Copy, Clone, PartialEq)]
use crate::quantity::statistics::{max, mean, median, min, modified_zscores, standard_deviation};
use crate::quantity::{serialize_information, serialize_time, Information, Time};
use crate::util::exit_code::extract_exit_code;
fn serialize_exit_status<S>(exit_status: &ExitStatus, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match extract_exit_code(*exit_status) {
Some(code) => serializer.serialize_i32(code),
None => serializer.serialize_unit(),
}
}
fn serialize_optional_information<S>(
value: &Option<Information>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match value {
Some(value) => serialize_information(value, serializer),
None => serializer.serialize_none(),
}
}
/// Performance metric measurements and exit code for a single run
#[derive(Debug, Clone, Default, Serialize, PartialEq)]
pub struct Measurement {
/// Wall clock time
/// Elapsed wall clock time (real time)
#[serde(serialize_with = "serialize_time")]
pub time_wall_clock: Time,
/// Time spent in user mode
#[serde(serialize_with = "serialize_time")]
pub time_user: Time,
/// Time spent in kernel mode
#[serde(serialize_with = "serialize_time")]
pub time_system: Time,
/// Maximum amount of memory used
pub peak_memory_usage: Information,
/// Peak resident set size, if available on this platform
#[serde(
skip_serializing_if = "Option::is_none",
serialize_with = "serialize_optional_information"
)]
pub memory_peak_resident: Option<Information>,
/// The exit status of the process
// The exit status of the process
#[serde(rename = "exit_code", serialize_with = "serialize_exit_status")]
pub exit_status: ExitStatus,
}
/// Performance measurements and exit statuses for all runs of a command.
#[derive(Debug, Default, Clone, PartialEq)]
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
pub struct Measurements {
pub measurements: Vec<Measurement>,
}
@@ -49,7 +82,7 @@ impl Measurements {
self.measurements.iter().map(|m| m.time_wall_clock)
}
/// The average wall clock time.
/// The average wall clock time
pub fn time_wall_clock_mean(&self) -> Time {
mean(self.wall_clock_times())
}
@@ -65,37 +98,33 @@ impl Measurements {
}
}
/// The median wall clock time.
/// The median wall clock time
pub fn median(&self) -> Time {
median(self.wall_clock_times())
}
/// The minimum wall clock time.
/// The minimum wall clock time
pub fn min(&self) -> Time {
min(self.wall_clock_times())
}
/// The maximum wall clock time.
/// The maximum wall clock time
pub fn max(&self) -> Time {
max(self.wall_clock_times())
}
/// Compute modified Z-scores for the wall clock times.
/// Compute modified Z-scores for the wall clock times
pub fn modified_zscores(&self) -> Vec<f64> {
modified_zscores(&self.wall_clock_times().collect::<Vec<_>>())
}
/// The average user time.
/// The average user time
pub fn time_user_mean(&self) -> Time {
mean(self.measurements.iter().map(|m| m.time_user))
}
/// The average system time.
/// The average system time
pub fn time_system_mean(&self) -> Time {
mean(self.measurements.iter().map(|m| m.time_system))
}
pub fn peak_memory_usage_mean(&self) -> Information {
mean(self.measurements.iter().map(|m| m.peak_memory_usage))
}
}
src/benchmark/mod.rs
+34 -36
View File
@@ -8,7 +8,9 @@ use std::cmp;
use std::io::{self, Write};
use std::time::Instant;
use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::executor::BenchmarkIteration;
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::command::Command;
use crate::options::{
CmdFailureAction, CommandOutputPolicy, ExecutorKind, Options, OutputStyleOption,
@@ -20,9 +22,8 @@ use crate::output::progress_bar::{
};
use crate::output::warnings::{OutlierWarningOptions, Warnings};
use crate::parameter::ParameterNameAndValue;
use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
use benchmark_result::{BenchmarkResult, Parameter};
use measurement::{Measurement, Measurements};
use crate::quantity::{self, const_time_from_seconds, FormatQuantity, Time, Zero};
use benchmark_result::BenchmarkResult;
use anyhow::{anyhow, Result};
use colored::*;
@@ -30,7 +31,7 @@ use colored::*;
use self::executor::Executor;
/// Threshold for warning about fast execution time
pub const MIN_EXECUTION_TIME: Time = const_time_from_seconds(5e-3);
pub const MIN_EXECUTION_TIME: Time = const_time_from_seconds(0.005);
pub struct Benchmark<'a> {
number: usize,
@@ -268,20 +269,20 @@ impl<'a> Benchmark<'a> {
if let Some(bar) = progress_bar.as_ref() {
start_initial_measurement(bar, Instant::now());
}
let res = self.executor.run_command_and_measure(
let measurement = self.executor.run_command_and_measure(
self.command,
benchmark_iteration,
None,
output_policy,
)?;
let success = res.exit_status.success();
let success = measurement.exit_status.success();
if let Some(bar) = progress_bar.as_ref() {
let time_unit = self
.options
.time_unit
.unwrap_or(res.time_wall_clock.suitable_unit());
let estimate = res.time_wall_clock.format(time_unit);
.unwrap_or(measurement.time_wall_clock.suitable_unit());
let estimate = measurement.time_wall_clock.format(time_unit);
finish_initial_measurement(
bar,
format!("Current estimate: {}", estimate.to_string().green()),
@@ -295,11 +296,11 @@ impl<'a> Benchmark<'a> {
// Determine number of benchmark runs
let runs_in_min_time = (self.options.min_benchmarking_time
/ (res.time_wall_clock
/ (measurement.time_wall_clock
+ self.executor.time_overhead()
+ preparation_overhead
+ conclusion_overhead))
.get::<ratio>() as u64;
.get::<quantity::ratio>() as u64;
let count = {
let min = cmp::max(runs_in_min_time, self.options.run_bounds.min);
@@ -315,7 +316,7 @@ impl<'a> Benchmark<'a> {
let count_remaining = count - 1;
// Save the first result
measurements.push(res);
measurements.push(measurement);
all_succeeded = all_succeeded && success;
@@ -345,15 +346,14 @@ impl<'a> Benchmark<'a> {
run_preparation_command(benchmark_iteration)?;
let res = self.executor.run_command_and_measure(
let measurement = self.executor.run_command_and_measure(
self.command,
benchmark_iteration,
None,
output_policy,
)?;
let success = res.exit_status.success();
measurements.push(res);
let success = measurement.exit_status.success();
measurements.push(measurement);
all_succeeded = all_succeeded && success;
@@ -368,25 +368,19 @@ impl<'a> Benchmark<'a> {
bar.finish_and_clear()
}
// Compute statistical quantities
let t_num = measurements.len();
let t_mean = measurements.time_wall_clock_mean();
let t_stddev = measurements.stddev();
let t_min = measurements.min();
let t_max = measurements.max();
let user_mean = measurements.time_user_mean();
let system_mean = measurements.time_system_mean();
// Formatting and console output
let time_unit = self.options.time_unit.unwrap_or(t_mean.suitable_unit());
let mean_str = t_mean.format(time_unit);
let min_str = t_min.format(time_unit);
let max_str = t_max.format(time_unit);
let num_str = format!("{t_num} runs");
let t_wall_clock_mean = measurements.time_wall_clock_mean();
let time_unit = self
.options
.time_unit
.unwrap_or(t_wall_clock_mean.suitable_unit());
let mean_str = t_wall_clock_mean.format(time_unit);
let min_str = measurements.min().format(time_unit);
let max_str = measurements.max().format(time_unit);
let num_str = format!("{num_runs} runs", num_runs = measurements.len());
let user_str = user_mean.format(time_unit);
let system_str = system_mean.format(time_unit);
let user_str = measurements.time_user_mean().format(time_unit);
let system_str = measurements.time_system_mean().format(time_unit);
if self.options.output_style != OutputStyleOption::Disabled {
let mut stdout = io::stdout().lock();
@@ -401,7 +395,7 @@ impl<'a> Benchmark<'a> {
system_str.blue()
)?;
} else {
let stddev_str = t_stddev.unwrap().format(time_unit);
let stddev_str = measurements.stddev().unwrap().format(time_unit);
console_writeln!(
stdout,
@@ -481,9 +475,14 @@ impl<'a> Benchmark<'a> {
self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?;
let command = self.command.get_command_line();
let name = self.command.get_name();
let name = (name != command).then_some(name);
Ok(BenchmarkResult {
command: self.command.get_name(),
command_with_unused_parameters: self.command.get_name_with_unused_parameters(),
command,
name,
display_name: self.command.get_name_with_unused_parameters(),
measurements,
parameters: self
.command
@@ -494,7 +493,6 @@ impl<'a> Benchmark<'a> {
name.to_string(),
Parameter {
value: value.to_string(),
is_unused: self.command.is_parameter_unused(name),
},
)
})
src/benchmark/relative_speed.rs
+4 -3
View File
@@ -154,11 +154,12 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult {
use std::collections::BTreeMap;
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::quantity::{second, Time};
use crate::quantity::second;
BenchmarkResult {
command: name.into(),
command_with_unused_parameters: name.into(),
display_name: name.into(),
name: None,
measurements: Measurements {
measurements: vec![Measurement {
time_wall_clock: Time::new::<second>(mean),
@@ -223,7 +224,7 @@ fn reference_identity_distinguishes_equal_results() {
fn reference_ratios_handle_zero_times() {
let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
for result in &mut results {
let measurement = result.measurements.measurements[0];
let measurement = result.measurements.measurements[0].clone();
result.measurements.measurements.push(measurement);
}
let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
src/benchmark/scheduler.rs
+57 -39
View File
@@ -89,11 +89,7 @@ impl<'a> Scheduler<'a> {
let reference = annotated_results.iter().find(|r| r.is_reference).unwrap();
let others = annotated_results.iter().filter(|r| !r.is_reference);
console_writeln!(
stdout,
" {} ran",
reference.result.command_with_unused_parameters.cyan()
)?;
console_writeln!(stdout, " {} ran", reference.result.display_name.cyan())?;
for item in others {
let stddev = if let Some(stddev) = item.relative_speed_stddev {
@@ -122,7 +118,7 @@ impl<'a> Scheduler<'a> {
stdout,
"{} {}",
comparator,
item.result.command_with_unused_parameters.magenta()
&item.result.display_name.magenta()
)?;
}
}
@@ -146,7 +142,7 @@ impl<'a> Scheduler<'a> {
} else {
" ".into()
},
item.result.command_with_unused_parameters,
item.result.display_name,
relationship,
)?;
}
@@ -205,39 +201,61 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
fn scheduler_basic() -> Result<()> {
insta::assert_yaml_snapshot!(generate_results(&["--runs=2", "sleep 0.123", "sleep 0.456"])?, @r#"
- command: sleep 0.123
mean: 0.123
stddev: 0
median: 0.123
user: 0
system: 0
min: 0.123
max: 0.123
times:
- 0.123
- 0.123
memory_usage_byte:
- 0
- 0
exit_codes:
- 0
- 0
measurements:
- time_wall_clock:
value: 0.123
unit: second
time_user:
value: 0
unit: second
time_system:
value: 0
unit: second
memory_peak_resident:
value: 0
unit: byte
exit_code: 0
- time_wall_clock:
value: 0.123
unit: second
time_user:
value: 0
unit: second
time_system:
value: 0
unit: second
memory_peak_resident:
value: 0
unit: byte
exit_code: 0
- command: sleep 0.456
mean: 0.456
stddev: 0
median: 0.456
user: 0
system: 0
min: 0.456
max: 0.456
times:
- 0.456
- 0.456
memory_usage_byte:
- 0
- 0
exit_codes:
- 0
- 0
measurements:
- time_wall_clock:
value: 0.456
unit: second
time_user:
value: 0
unit: second
time_system:
value: 0
unit: second
memory_peak_resident:
value: 0
unit: byte
exit_code: 0
- time_wall_clock:
value: 0.456
unit: second
time_user:
value: 0
unit: second
time_system:
value: 0
unit: second
memory_peak_resident:
value: 0
unit: byte
exit_code: 0
"#);
Ok(())
src/export/csv.rs
+13 -15
View File
@@ -16,7 +16,7 @@ impl Exporter for CsvExporter {
fn serialize(
&self,
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_time_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference_index: Option<usize>,
) -> Result<Vec<u8>> {
@@ -42,7 +42,7 @@ impl Exporter for CsvExporter {
}
for res in results {
let mut fields = vec![Cow::Borrowed(res.command.as_bytes())];
let mut fields = vec![Cow::Borrowed(res.get_name().as_bytes())];
for f in &[
res.mean_wall_clock_time(),
res.measurements.stddev().unwrap_or_default(),
@@ -80,28 +80,29 @@ fn test_csv() {
let results = vec![
BenchmarkResult {
command: String::from("command_a"),
command_with_unused_parameters: String::from("command_a"),
command: String::from("echo command_a"),
name: Some(String::from("command_a")),
display_name: String::from("command_a"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(7.0),
time_user: Time::new::<second>(7.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(8.0),
time_user: Time::new::<second>(8.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(12.0),
time_user: Time::new::<second>(12.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -111,14 +112,12 @@ fn test_csv() {
"foo".into(),
Parameter {
value: "one".into(),
is_unused: false,
},
);
params.insert(
"bar".into(),
Parameter {
value: "two".into(),
is_unused: false,
},
);
params
@@ -126,27 +125,28 @@ fn test_csv() {
},
BenchmarkResult {
command: String::from("command_b"),
command_with_unused_parameters: String::from("command_b"),
name: None,
display_name: String::from("command_b"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(17.0),
time_user: Time::new::<second>(17.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(18.0),
time_user: Time::new::<second>(18.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(19.0),
time_user: Time::new::<second>(19.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -156,14 +156,12 @@ fn test_csv() {
"foo".into(),
Parameter {
value: "one".into(),
is_unused: false,
},
);
params.insert(
"bar".into(),
Parameter {
value: "seven".into(),
is_unused: false,
},
);
params
src/export/json.rs
+148 -7
View File
@@ -1,30 +1,141 @@
use std::ops::{Add, AddAssign, Div};
use serde::*;
use serde_json::to_vec_pretty;
use super::Exporter;
use crate::benchmark::benchmark_result::BenchmarkResult;
use crate::benchmark::measurement::Measurements;
use crate::options::SortOrder;
use crate::quantity::TimeUnit;
use crate::quantity::statistics::{max, mean, median, min, standard_deviation, UnsafeRawValue};
use crate::quantity::{byte, second, QuantityInUnit, Ratio, TimeUnit, Zero};
use anyhow::Result;
#[derive(Serialize, Debug)]
mod metadata;
use metadata::Metadata;
const JSON_SCHEMA_VERSION: u32 = 2;
#[derive(Serialize)]
struct HyperfineSummary<'a> {
results: &'a [BenchmarkResult],
schema_version: u32,
metadata: &'a Metadata,
results: Vec<JsonBenchmarkResult<'a>>,
}
#[derive(Serialize)]
struct JsonBenchmarkResult<'a> {
#[serde(flatten)]
result: &'a BenchmarkResult,
#[serde(skip_serializing_if = "std::ops::Not::not")]
is_reference: bool,
summary: BenchmarkSummary,
}
#[derive(Serialize)]
struct BenchmarkSummary {
time_wall_clock: StatisticalSummary,
time_user: StatisticalSummary,
time_system: StatisticalSummary,
#[serde(skip_serializing_if = "Option::is_none")]
memory_peak_resident: Option<StatisticalSummary>,
}
impl BenchmarkSummary {
fn from_measurements(measurements: &Measurements) -> Self {
let values = &measurements.measurements;
let mut memory_values = values
.iter()
.filter_map(|m| m.memory_peak_resident)
.peekable();
Self {
time_wall_clock: StatisticalSummary::from_values(
values.iter().map(|m| m.time_wall_clock),
second,
),
time_user: StatisticalSummary::from_values(values.iter().map(|m| m.time_user), second),
time_system: StatisticalSummary::from_values(
values.iter().map(|m| m.time_system),
second,
),
memory_peak_resident: memory_values
.peek()
.is_some()
.then(|| StatisticalSummary::from_values(memory_values, byte)),
}
}
}
#[derive(Serialize)]
struct StatisticalSummary {
unit: &'static str,
count: usize,
mean: f64,
stddev: Option<f64>,
median: f64,
min: f64,
max: f64,
}
impl StatisticalSummary {
fn from_values<Q, P, U>(values: impl IntoIterator<Item = Q>, _unit: U) -> Self
where
Q: Copy
+ PartialOrd
+ Add<Output = Q>
+ AddAssign
+ Zero
+ Div<Ratio, Output = P>
+ UnsafeRawValue
+ QuantityInUnit<U>,
P: Into<Q>,
U: uom::si::Unit,
{
let values: Vec<_> = values.into_iter().collect();
assert!(
!values.is_empty(),
"Statistical summaries require at least one measurement"
);
Self {
unit: U::singular(),
count: values.len(),
mean: mean(values.iter().copied()).value_in_unit(),
stddev: (values.len() > 1)
.then(|| standard_deviation(values.iter().copied()).value_in_unit()),
median: median(values.iter().copied()).value_in_unit(),
min: min(values.iter().copied()).value_in_unit(),
max: max(values.iter().copied()).value_in_unit(),
}
}
}
#[derive(Default)]
pub struct JsonExporter {}
pub struct JsonExporter {
metadata: Metadata,
}
impl Exporter for JsonExporter {
fn serialize(
&self,
results: &[BenchmarkResult],
_unit: Option<TimeUnit>,
_time_unit: Option<TimeUnit>,
_sort_order: SortOrder,
_reference_index: Option<usize>,
reference_index: Option<usize>,
) -> Result<Vec<u8>> {
let mut output = to_vec_pretty(&HyperfineSummary { results });
let mut output = to_vec_pretty(&HyperfineSummary {
schema_version: JSON_SCHEMA_VERSION,
metadata: &self.metadata,
results: results
.iter()
.enumerate()
.map(|(index, result)| JsonBenchmarkResult {
result,
is_reference: reference_index == Some(index),
summary: BenchmarkSummary::from_measurements(&result.measurements),
})
.collect(),
});
if let Ok(ref mut content) = output {
content.push(b'\n');
}
@@ -32,3 +143,33 @@ impl Exporter for JsonExporter {
Ok(output?)
}
}
#[test]
fn test_statistical_summary_unit_conversion() {
use crate::quantity::{kibibyte, millisecond, Information, Time};
let time = StatisticalSummary::from_values(
[Time::new::<second>(1.0), Time::new::<second>(3.0)],
millisecond,
);
let information = StatisticalSummary::from_values(
[
Information::new::<byte>(1024.0),
Information::new::<byte>(3072.0),
],
kibibyte,
);
for (summary, unit, scale) in [
(time, "millisecond", 1000.0),
(information, "kibibyte", 1.0),
] {
assert_eq!(summary.unit, unit);
assert_eq!(summary.count, 2);
approx::assert_relative_eq!(summary.mean, 2.0 * scale);
approx::assert_relative_eq!(summary.median, 2.0 * scale);
approx::assert_relative_eq!(summary.min, scale);
approx::assert_relative_eq!(summary.max, 3.0 * scale);
approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale);
}
}
src/export/json/metadata.rs
+51
View File
@@ -0,0 +1,51 @@
use serde::Serialize;
#[derive(Debug, Serialize)]
pub(super) struct Metadata {
hyperfine_version: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub start_time: Option<String>,
pub platform: Platform,
}
impl Default for Metadata {
fn default() -> Self {
Self {
hyperfine_version: env!("CARGO_PKG_VERSION"),
start_time: utc_now(),
platform: Platform::detect(),
}
}
}
fn utc_now() -> Option<String> {
use std::{fmt::Write, time::SystemTime};
let now = SystemTime::now();
now.duration_since(SystemTime::UNIX_EPOCH).ok()?;
let mut timestamp = String::new();
write!(&mut timestamp, "{}", humantime::format_rfc3339_seconds(now)).ok()?;
Some(timestamp)
}
#[derive(Debug, Default, Serialize)]
pub(super) struct Platform {
#[serde(skip_serializing_if = "Option::is_none")]
os: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
architecture: Option<String>,
}
impl Platform {
fn detect() -> Self {
use platform_info::{PlatformInfo, PlatformInfoAPI, UNameAPI};
let Ok(info) = PlatformInfo::new() else {
return Self::default();
};
Self {
os: Some(info.sysname().to_string_lossy().into_owned()),
architecture: Some(info.machine().to_string_lossy().into_owned()),
}
}
}
src/export/markup.rs
+12 -14
View File
@@ -16,12 +16,12 @@ pub trait MarkupExporter {
fn table_results(
&self,
entries: &[BenchmarkResultWithRelativeSpeed],
unit: TimeUnit,
time_unit: TimeUnit,
reference_pending: bool,
explicit_reference: bool,
) -> String {
// prepare table header strings
let notation = format!("[{}]", unit.short_name());
let notation = format!("[{}]", time_unit.short_name());
// prepare table cells alignment
let cells_alignment = [
@@ -48,25 +48,23 @@ pub trait MarkupExporter {
table.push_str(&self.table_divider(&cells_alignment));
for entry in entries {
let measurement = &entry.result;
let result = &entry.result;
// prepare data row strings
let cmd_str = measurement
.command_with_unused_parameters
.replace('|', "\\|");
let mean_str = measurement.mean_wall_clock_time().format_value(unit);
let stddev_str = if let Some(stddev) = measurement.measurements.stddev() {
format!(" ± {}", stddev.format_value(unit))
let cmd_str = result.display_name.replace('|', "\\|");
let mean_str = result.mean_wall_clock_time().format_value(time_unit);
let stddev_str = if let Some(stddev) = result.measurements.stddev() {
format!(" ± {}", stddev.format_value(time_unit))
} else {
"".into()
};
let min_str = measurement.measurements.min().format_value(unit);
let max_str = measurement.measurements.max().format_value(unit);
let min_str = result.measurements.min().format_value(time_unit);
let max_str = result.measurements.max().format_value(time_unit);
// The ratio of two zero times is undefined, even if they compare equal.
let relative_unavailable = reference_pending
|| (explicit_reference
&& !entry.is_reference
&& entry.relative_ordering == Ordering::Equal
&& measurement.mean_wall_clock_time() == Time::zero());
&& result.mean_wall_clock_time() == Time::zero());
let relative = if relative_unavailable {
"N/A".to_string()
} else {
@@ -127,11 +125,11 @@ impl<T: MarkupExporter> Exporter for T {
fn serialize(
&self,
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
time_unit: Option<TimeUnit>,
sort_order: SortOrder,
reference_index: Option<usize>,
) -> Result<Vec<u8>> {
let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
let unit = time_unit.unwrap_or_else(|| determine_unit_from_results(results));
// Do not report ratios against another benchmark while the selected
// reference is still pending in an intermediate export.
let reference_pending = reference_index.is_some_and(|i| i >= results.len());
src/export/mod.rs
+1 -1
View File
@@ -49,7 +49,7 @@ trait Exporter {
fn serialize(
&self,
results: &[BenchmarkResult],
unit: Option<TimeUnit>,
time_unit: Option<TimeUnit>,
sort_order: SortOrder,
reference_index: Option<usize>,
) -> Result<Vec<u8>>;
src/export/tests.rs
+40 -32
View File
@@ -28,27 +28,28 @@ fn test_markup_export_auto_ms() {
let results = [
BenchmarkResult {
command: String::from("sleep 0.1"),
command_with_unused_parameters: String::from("sleep 0.1"),
name: None,
display_name: String::from("sleep 0.1"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(0.09),
time_user: Time::new::<second>(0.09),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.10),
time_user: Time::new::<second>(0.10),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.14),
time_user: Time::new::<second>(0.14),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -56,27 +57,28 @@ fn test_markup_export_auto_ms() {
},
BenchmarkResult {
command: String::from("sleep 2"),
command_with_unused_parameters: String::from("sleep 2"),
name: None,
display_name: String::from("sleep 2"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(2.0),
time_user: Time::new::<second>(2.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(3.0),
time_user: Time::new::<second>(3.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(4.0),
time_user: Time::new::<second>(4.0),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -129,27 +131,28 @@ fn test_markup_export_auto_s() {
let results = [
BenchmarkResult {
command: String::from("sleep 2"),
command_with_unused_parameters: String::from("sleep 2"),
name: None,
display_name: String::from("sleep 2"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(2.1),
time_user: Time::new::<second>(2.1),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.2),
time_user: Time::new::<second>(2.2),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.3),
time_user: Time::new::<second>(2.3),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -157,27 +160,28 @@ fn test_markup_export_auto_s() {
},
BenchmarkResult {
command: String::from("sleep 0.1"),
command_with_unused_parameters: String::from("sleep 0.1"),
name: None,
display_name: String::from("sleep 0.1"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(0.1),
time_user: Time::new::<second>(0.1),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.2),
time_user: Time::new::<second>(0.2),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.3),
time_user: Time::new::<second>(0.3),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -230,27 +234,28 @@ fn test_markup_export_manual_ms() {
let timing_results = [
BenchmarkResult {
command: String::from("sleep 2"),
command_with_unused_parameters: String::from("sleep 2"),
name: None,
display_name: String::from("sleep 2"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(2.1),
time_user: Time::new::<second>(2.1),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.2),
time_user: Time::new::<second>(2.2),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.3),
time_user: Time::new::<second>(2.3),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -258,27 +263,28 @@ fn test_markup_export_manual_ms() {
},
BenchmarkResult {
command: String::from("sleep 0.1"),
command_with_unused_parameters: String::from("sleep 0.1"),
name: None,
display_name: String::from("sleep 0.1"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(0.1),
time_user: Time::new::<second>(0.1),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.2),
time_user: Time::new::<second>(0.2),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.3),
time_user: Time::new::<second>(0.3),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -330,27 +336,28 @@ fn test_markup_export_manual_s() {
let results = [
BenchmarkResult {
command: String::from("sleep 2"),
command_with_unused_parameters: String::from("sleep 2"),
name: None,
display_name: String::from("sleep 2"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(2.01),
time_user: Time::new::<second>(2.01),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.02),
time_user: Time::new::<second>(2.02),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(2.03),
time_user: Time::new::<second>(2.03),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
@@ -358,27 +365,28 @@ fn test_markup_export_manual_s() {
},
BenchmarkResult {
command: String::from("sleep 0.1"),
command_with_unused_parameters: String::from("sleep 0.1"),
name: None,
display_name: String::from("sleep 0.1"),
measurements: Measurements::new(vec![
Measurement {
time_wall_clock: Time::new::<second>(0.11),
time_user: Time::new::<second>(0.11),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.12),
time_user: Time::new::<second>(0.12),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
Measurement {
time_wall_clock: Time::new::<second>(0.13),
time_user: Time::new::<second>(0.13),
time_system: Time::zero(),
peak_memory_usage: Information::new::<byte>(1024.),
memory_peak_resident: Some(Information::new::<byte>(1024.)),
exit_status: ExitStatus::default(),
},
]),
src/quantity/mod.rs
+45 -4
View File
@@ -1,5 +1,8 @@
use std::marker::PhantomData;
use serde::ser::SerializeStruct;
use serde::Serializer;
use uom::si;
pub use si::f64::{Information, Ratio, Time};
@@ -13,6 +16,24 @@ pub use units::{InformationUnit, IsUnit, TimeUnit};
pub mod statistics;
mod units;
pub trait QuantityInUnit<U> {
fn value_in_unit(self) -> f64;
}
impl<U: uom::si::time::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U> for Time {
fn value_in_unit(self) -> f64 {
self.get::<U>()
}
}
impl<U: uom::si::information::Unit + uom::Conversion<f64, T = f64>> QuantityInUnit<U>
for Information
{
fn value_in_unit(self) -> f64 {
self.get::<U>()
}
}
pub trait FormatQuantity {
type Unit;
@@ -105,6 +126,26 @@ impl FormatQuantity for Information {
}
}
pub fn serialize_time<S>(t: &Time, s: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = s.serialize_struct("Time", 2)?;
state.serialize_field("value", &t.get::<second>())?;
state.serialize_field("unit", "second")?;
state.end()
}
pub fn serialize_information<S>(i: &Information, s: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = s.serialize_struct("Information", 2)?;
state.serialize_field("value", &i.get::<byte>())?;
state.serialize_field("unit", "byte")?;
state.end()
}
#[test]
fn test_time() {
let time = Time::new::<millisecond>(123.4);
@@ -181,11 +222,11 @@ fn test_format() {
assert_eq!(time.format_auto(), "123.4 ms");
assert_eq!(time.format(TimeUnit::MicroSecond), "123400.0 µs");
let peak_memory_usage = Information::new::<kibibyte>(8.);
assert_eq!(peak_memory_usage.format_auto(), "8.0 KiB");
assert_eq!(peak_memory_usage.format(InformationUnit::Byte), "8192 B");
let memory_peak_resident = Information::new::<kibibyte>(8.);
assert_eq!(memory_peak_resident.format_auto(), "8.0 KiB");
assert_eq!(memory_peak_resident.format(InformationUnit::Byte), "8192 B");
assert_eq!(
peak_memory_usage.format(InformationUnit::KibiByte),
memory_peak_resident.format(InformationUnit::KibiByte),
"8.0 KiB"
);
}
src/timer/mod.rs
+2 -2
View File
@@ -82,14 +82,14 @@ pub fn execute_and_measure(mut command: Command) -> Result<Measurement> {
discard(output);
}
let (time_user, time_system, peak_memory_usage, exit_status) = cpu_timer.stop(child)?;
let (time_user, time_system, memory_peak_resident, exit_status) = cpu_timer.stop(child)?;
let time_wall_clock = wallclock_timer.stop();
Ok(Measurement {
time_wall_clock,
time_user,
time_system,
peak_memory_usage,
memory_peak_resident,
exit_status,
})
}
src/timer/unix_timer.rs
+2 -2
View File
@@ -77,12 +77,12 @@ impl CPUTimer {
Self {}
}
pub fn stop(&self, child: Child) -> Result<(Time, Time, Information, ExitStatus)> {
pub fn stop(&self, child: Child) -> Result<(Time, Time, Option<Information>, ExitStatus)> {
let (status, usage) = wait4(child)?;
Ok((
usage.time_user,
usage.time_system,
usage.memory_usage,
Some(usage.memory_usage),
status,
))
}
src/timer/windows_timer.rs
+3 -3
View File
@@ -84,7 +84,7 @@ impl CPUTimer {
Self { job_object }
}
pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> {
pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Option<Information>, ExitStatus)> {
let status = child.wait()?;
let mut job_object_info =
@@ -116,9 +116,9 @@ impl CPUTimer {
let system_time =
Time::new::<nanosecond>((job_object_info.TotalKernelTime as f64) * 100.0);
Ok((user_time, system_time, Information::zero(), status))
Ok((user_time, system_time, None, status))
} else {
Ok((Time::zero(), Time::zero(), Information::zero(), status))
Ok((Time::zero(), Time::zero(), None, status))
}
}
}
tests/integration_tests.rs
+55 -1
View File
@@ -24,6 +24,17 @@ fn snapshot_settings() -> insta::Settings {
settings
}
fn json_snapshot_settings() -> insta::Settings {
let mut settings = snapshot_settings();
for field in ["hyperfine_version", "start_time", "os", "architecture"] {
settings.add_filter(
&format!(r#"("{field}": ")[^"]+(")"#),
format!(r#"${{1}}[{field}]$2"#),
);
}
settings
}
#[test]
fn runs_successfully() {
hyperfine()
@@ -916,7 +927,7 @@ fn selects_reference_with_parameterized_prepare() {
assert_eq!(
results
.iter()
.map(|result| result["parameters"]["delay"].as_str().unwrap())
.map(|result| result["parameters"]["delay"]["value"].as_str().unwrap())
.collect::<Vec<_>>(),
["0.2", "0.4", "0.6"]
);
@@ -1273,3 +1284,46 @@ fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
]
);
}
#[test]
fn json_export_basic() {
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=2",
"--warmup=1",
"--reference-name=one second",
"--reference=sleep 1",
"--command-name=sleep 2",
"sleep 2",
]));
}
#[test]
fn json_export_single_run() {
// Make sure that the standard deviation is set to `null`
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=1",
"sleep 1",
]));
}
#[test]
fn json_export_parameterized_with_reference() {
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=2",
"-L",
"duration",
"1,2",
"--command-name=sleep for {duration} seconds",
"--reference=sleep for 2 seconds",
"sleep {duration}",
]));
}
tests/snapshots/integration_tests__json_export_basic.snap
+199
View File
@@ -0,0 +1,199 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=2"
- "--warmup=1"
- "--reference-name=one second"
- "--reference=sleep 1"
- "--command-name=sleep 2"
- sleep 2
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"name": "one second",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"is_reference": true,
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 1.0,
"stddev": 0.0,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
{
"command": "sleep 2",
"measurements": [
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 2.0,
"stddev": 0.0,
"median": 2.0,
"min": 2.0,
"max": 2.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----
tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap
+211
View File
@@ -0,0 +1,211 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=2"
- "-L"
- duration
- "1,2"
- "--command-name=sleep for {duration} seconds"
- "--reference=sleep for 2 seconds"
- "sleep {duration}"
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"name": "sleep for 1 seconds",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"parameters": {
"duration": {
"value": "1"
}
},
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 1.0,
"stddev": 0.0,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
{
"command": "sleep 2",
"name": "sleep for 2 seconds",
"measurements": [
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"parameters": {
"duration": {
"value": "2"
}
},
"is_reference": true,
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 2.0,
"stddev": 0.0,
"median": 2.0,
"min": 2.0,
"max": 2.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----
tests/snapshots/integration_tests__json_export_single_run.snap
+93
View File
@@ -0,0 +1,93 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=1"
- sleep 1
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 1,
"mean": 1.0,
"stddev": null,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----