vsndrg/hyperfine
1
0 Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-11-01 08:50:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #969 from sharkdp/codex/result-measurements

Browse Source 
Store typed Measurements in BenchmarkResult
This commit is contained in:
David Peter authored and GitHub committed 2026-10-05 12:51:22 +02:00
2 parents 30a2b61d4c b23372c5e2
commit e689520fb5
7 files changed
+432 -257
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/benchmark/benchmark_result.rs
+75 -40
View File
@@ -1,6 +1,11 @@
use std::collections::BTreeMap;
use serde::Serialize;
use serde::ser::SerializeStruct;
use serde::{Serialize, Serializer};
use crate::benchmark::measurement::Measurements;
use crate::quantity::{byte, second, Time};
use crate::util::exit_code::extract_exit_code;
/// Parameter value and whether it was used in the command line template
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
@@ -11,52 +16,82 @@ pub struct Parameter {
pub is_unused: bool,
}
/// Set of values that will be exported.
// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
/// Meta data and performance metrics for a single benchmark
#[derive(Debug, Default, Clone, PartialEq)]
pub struct BenchmarkResult {
/// The full command line of the program that is being benchmarked
pub command: String,
/// The full command line of the program that is being benchmarked, possibly including a list of
/// parameters that were not used in the command line template.
#[serde(skip_serializing)]
/// The full command line, including parameters not used in the command template.
pub command_with_unused_parameters: String,
/// The average run time
pub mean: f64,
/// The standard deviation of all run times. Not available if only one run has been performed
pub stddev: Option<f64>,
/// The median run time
pub median: f64,
/// Time spent in user mode
pub user: f64,
/// Time spent in kernel mode
pub system: f64,
/// Minimum of all measured times
pub min: f64,
/// Maximum of all measured times
pub max: f64,
/// All run time measurements
#[serde(skip_serializing_if = "Option::is_none")]
pub times: Option<Vec<f64>>,
/// Maximum memory usage of the process, in bytes
#[serde(skip_serializing_if = "Option::is_none")]
pub memory_usage_byte: Option<Vec<u64>>,
/// Exit codes of all command invocations
pub exit_codes: Vec<Option<i32>>,
/// Performance measurements and exit statuses for each run
pub measurements: Measurements,
/// Parameter values for this benchmark
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub parameters: BTreeMap<String, Parameter>,
}
impl BenchmarkResult {
/// The average wall clock time
pub fn mean_wall_clock_time(&self) -> Time {
self.measurements.time_wall_clock_mean()
}
}
// Preserve the existing export format while storing typed measurements internally.
impl Serialize for BenchmarkResult {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct(
"BenchmarkResult",
if self.parameters.is_empty() { 11 } else { 12 },
)?;
state.serialize_field("command", &self.command)?;
state.serialize_field("mean", &self.mean_wall_clock_time().get::<second>())?;
state.serialize_field(
"stddev",
&self.measurements.stddev().map(|time| time.get::<second>()),
)?;
state.serialize_field("median", &self.measurements.median().get::<second>())?;
state.serialize_field("user", &self.measurements.time_user_mean().get::<second>())?;
state.serialize_field(
"system",
&self.measurements.time_system_mean().get::<second>(),
)?;
state.serialize_field("min", &self.measurements.min().get::<second>())?;
state.serialize_field("max", &self.measurements.max().get::<second>())?;
state.serialize_field(
"times",
&self
.measurements
.wall_clock_times()
.map(|time| time.get::<second>())
.collect::<Vec<_>>(),
)?;
state.serialize_field(
"memory_usage_byte",
&self
.measurements
.measurements
.iter()
.map(|measurement| measurement.peak_memory_usage.get::<byte>() as u64)
.collect::<Vec<_>>(),
)?;
state.serialize_field(
"exit_codes",
&self
.measurements
.measurements
.iter()
.map(|measurement| extract_exit_code(measurement.exit_status))
.collect::<Vec<_>>(),
)?;
if !self.parameters.is_empty() {
state.serialize_field("parameters", &self.parameters)?;
}
state.end()
}
}
src/benchmark/measurement.rs
+6 -2
View File
@@ -4,7 +4,7 @@ use crate::quantity::statistics::{max, mean, median, min, modified_zscores, stan
use crate::quantity::{Information, Time};
/// Performance measurements and exit status from running a single command
#[derive(Debug, Default, Copy, Clone)]
#[derive(Debug, Default, Copy, Clone, PartialEq)]
pub struct Measurement {
/// Wall clock time
pub time_wall_clock: Time,
@@ -23,12 +23,16 @@ pub struct Measurement {
}
/// Performance measurements and exit statuses for all runs of a command.
#[derive(Debug, Default, Clone)]
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Measurements {
pub measurements: Vec<Measurement>,
}
impl Measurements {
pub fn new(measurements: Vec<Measurement>) -> Self {
Self { measurements }
}
pub fn len(&self) -> usize {
self.measurements.len()
}
src/benchmark/mod.rs
+2 -28
View File
@@ -17,8 +17,7 @@ use crate::output::console_writeln;
use crate::output::progress_bar::get_progress_bar;
use crate::output::warnings::{OutlierWarningOptions, Warnings};
use crate::parameter::ParameterNameAndValue;
use crate::quantity::{byte, const_time_from_seconds, ratio, second, FormatQuantity, Time, Zero};
use crate::util::exit_code::extract_exit_code;
use crate::quantity::{const_time_from_seconds, ratio, FormatQuantity, Time, Zero};
use benchmark_result::{BenchmarkResult, Parameter};
use measurement::{Measurement, Measurements};
@@ -352,7 +351,6 @@ impl<'a> Benchmark<'a> {
let t_num = measurements.len();
let t_mean = measurements.time_wall_clock_mean();
let t_stddev = measurements.stddev();
let t_median = measurements.median();
let t_min = measurements.min();
let t_max = measurements.max();
@@ -465,31 +463,7 @@ impl<'a> Benchmark<'a> {
Ok(BenchmarkResult {
command: self.command.get_name(),
command_with_unused_parameters: self.command.get_name_with_unused_parameters(),
mean: t_mean.get::<second>(),
stddev: t_stddev.map(|t| t.get::<second>()),
median: t_median.get::<second>(),
user: user_mean.get::<second>(),
system: system_mean.get::<second>(),
min: t_min.get::<second>(),
max: t_max.get::<second>(),
times: Some(
measurements
.wall_clock_times()
.map(|t| t.get::<second>())
.collect(),
),
memory_usage_byte: Some(
measurements
.measurements
.iter()
.map(|m| m.peak_memory_usage.get::<byte>() as u64)
.collect(),
),
exit_codes: measurements
.measurements
.iter()
.map(|m| extract_exit_code(m.exit_status))
.collect(),
measurements,
parameters: self
.command
.get_parameters()
src/benchmark/relative_speed.rs
+41 -19
View File
@@ -2,6 +2,7 @@ use std::cmp::Ordering;
use super::benchmark_result::BenchmarkResult;
use crate::options::SortOrder;
use crate::quantity::second;
#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> {
@@ -14,7 +15,10 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
}
pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
l.mean_wall_clock_time()
.get::<second>()
.partial_cmp(&r.mean_wall_clock_time().get::<second>())
.unwrap_or(Ordering::Equal)
}
pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
@@ -35,7 +39,7 @@ fn compute_relative_speeds<'a>(
let is_reference = result == reference;
let relative_ordering = compare_mean_time(result, reference);
if result.mean == 0.0 {
if result.mean_wall_clock_time().get::<second>() == 0.0 {
return BenchmarkResultWithRelativeSpeed {
result,
relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
@@ -46,18 +50,34 @@ fn compute_relative_speeds<'a>(
}
let ratio = match relative_ordering {
Ordering::Less => reference.mean / result.mean,
Ordering::Less => {
reference.mean_wall_clock_time().get::<second>()
/ result.mean_wall_clock_time().get::<second>()
}
Ordering::Equal => 1.0,
Ordering::Greater => result.mean / reference.mean,
Ordering::Greater => {
result.mean_wall_clock_time().get::<second>()
/ reference.mean_wall_clock_time().get::<second>()
}
};
// https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
// Covariance assumed to be 0, i.e. variables are assumed to be independent
let ratio_stddev = match (result.stddev, reference.stddev) {
let ratio_stddev = match (
result
.measurements
.stddev()
.map(|time| time.get::<second>()),
reference
.measurements
.stddev()
.map(|time| time.get::<second>()),
) {
(Some(result_stddev), Some(fastest_stddev)) => Some(
ratio
* ((result_stddev / result.mean).powi(2)
+ (fastest_stddev / reference.mean).powi(2))
* ((result_stddev / result.mean_wall_clock_time().get::<second>()).powi(2)
+ (fastest_stddev / reference.mean_wall_clock_time().get::<second>())
.powi(2))
.sqrt(),
),
_ => None,
@@ -88,7 +108,9 @@ pub fn compute_with_check_from_reference<'a>(
reference: &'a BenchmarkResult,
sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
if fastest_of(results).mean == 0.0 || reference.mean == 0.0 {
if fastest_of(results).mean_wall_clock_time().get::<second>() == 0.0
|| reference.mean_wall_clock_time().get::<second>() == 0.0
{
return None;
}
@@ -101,7 +123,7 @@ pub fn compute_with_check(
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
let fastest = fastest_of(results);
if fastest.mean == 0.0 {
if fastest.mean_wall_clock_time().get::<second>() == 0.0 {
return None;
}
@@ -122,19 +144,19 @@ pub fn compute(
fn create_result(name: &str, mean: f64) -> BenchmarkResult {
use std::collections::BTreeMap;
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::quantity::{second, Time};
BenchmarkResult {
command: name.into(),
command_with_unused_parameters: name.into(),
mean,
stddev: Some(1.0),
median: mean,
user: mean,
system: 0.0,
min: mean,
max: mean,
times: None,
memory_usage_byte: None,
exit_codes: Vec::new(),
measurements: Measurements {
measurements: vec![Measurement {
time_wall_clock: Time::new::<second>(mean),
time_user: Time::new::<second>(mean),
..Default::default()
}],
},
parameters: BTreeMap::new(),
}
}