From 4cd0e245865f998ffb66b77ad91926b9ec2e003b Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 15:32:27 +0200 Subject: [PATCH] Document JSON measurements and omit unavailable memory metrics --- README.md | 20 +++++++++++++++ src/benchmark/executor.rs | 4 +-- src/benchmark/measurement.rs | 26 +++++++++++++------ src/export/csv.rs | 12 ++++----- src/export/json.rs | 47 +++++++++++++++++++++++++++++++++++ src/export/tests.rs | 48 ++++++++++++++++++------------------ src/timer/unix_timer.rs | 4 +-- src/timer/windows_timer.rs | 6 ++--- tests/integration_tests.rs | 24 ++++++++++++++++++ 9 files changed, 147 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index dc6d655..d7a5806 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,26 @@ You can use the `--export-markdown ` option to create tables like the foll #### JSON +The JSON export contains a `results` array with one entry per benchmark. Each entry has a +`measurements` array with one object per measured run (excluding warmup runs): + +| Field | Format | Meaning and platform differences | +|:---|:---|:---| +| `time_wall_clock` | `{ "value": number, "unit": "second" }` | Elapsed time, including time spent waiting or descheduled. Available on all platforms. | +| `time_user` | `{ "value": number, "unit": "second" }` | CPU time spent in user mode, accumulated across threads. On Unix, includes descendants whose usage propagates through parents waiting for them. On Windows, includes processes associated with the benchmark's job object. | +| `time_system` | `{ "value": number, "unit": "second" }` | CPU time spent in kernel mode, with the same process accounting as `time_user`. | +| `peak_memory_usage` | `{ "value": number, "unit": "byte" }` | Peak resident set size (RSS) on Linux and macOS; omitted on Windows. This is the largest per-process peak among the command and descendants whose usage is collected by waiting for them, **not the simultaneous total memory of the process tree**. Linux's KiB counter and macOS's byte counter are both exported in bytes. On macOS, this is RSS, not Apple's separate memory-footprint metric. | +| `exit_code` | Integer or `null` | Process exit code; `0` indicates success. On Unix, termination by a signal is encoded as `128 + signal number`. `null` means no exit code could be determined. Failed runs are included when allowed by `--ignore-failure`. | + +Metrics unavailable on a platform are omitted, rather than reported as zero. Consumers should +accept missing metrics and additional fields as new metrics are introduced. Time values are always +exported in seconds, regardless of `--time-unit`. When using a shell, the estimated shell startup +CPU and wall-clock times are subtracted; memory usage is not adjusted. + +For example, two concurrent worker processes that each use 128 MiB can produce a +`peak_memory_usage` of about 128 MiB, even though their combined memory use is about 256 MiB. +Threads share a process's address space and are covered by its RSS measurement. + The JSON output is useful if you want to analyze the benchmark results in more detail. The [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot of helpful Python programs to further analyze benchmark results and create helpful diff --git a/src/benchmark/executor.rs b/src/benchmark/executor.rs index 5ffd27a..dce30cd 100644 --- a/src/benchmark/executor.rs +++ b/src/benchmark/executor.rs @@ -277,7 +277,7 @@ impl Executor for ShellExecutor<'_> { time_wall_clock: measurements.time_wall_clock_mean(), time_user: measurements.time_user_mean(), time_system: measurements.time_system_mean(), - peak_memory_usage: measurements.peak_memory_usage_mean(), + peak_memory_usage: None, exit_status: ExitStatus::default(), }); @@ -335,7 +335,7 @@ impl Executor for MockExecutor { time_wall_clock: Self::extract_time(command.get_command_line()), time_user: Time::zero(), time_system: Time::zero(), - peak_memory_usage: Information::zero(), + peak_memory_usage: Some(Information::zero()), exit_status, }) } diff --git a/src/benchmark/measurement.rs b/src/benchmark/measurement.rs index 186d4f4..faff776 100644 --- a/src/benchmark/measurement.rs +++ b/src/benchmark/measurement.rs @@ -16,6 +16,19 @@ where } } +fn serialize_optional_information( + value: &Option, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + match value { + Some(value) => serialize_information(value, serializer), + None => serializer.serialize_none(), + } +} + /// Performance metric measurements and exit code for a single run #[derive(Debug, Clone, Default, Serialize, PartialEq)] pub struct Measurement { @@ -31,9 +44,12 @@ pub struct Measurement { #[serde(serialize_with = "serialize_time")] pub time_system: Time, - /// Maximum memory usage of the process - #[serde(serialize_with = "serialize_information")] - pub peak_memory_usage: Information, + /// Peak resident set size, if available on this platform + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_optional_information" + )] + pub peak_memory_usage: Option, // The exit status of the process #[serde(rename = "exit_code", serialize_with = "serialize_exit_status")] @@ -111,8 +127,4 @@ impl Measurements { pub fn time_system_mean(&self) -> Time { mean(self.measurements.iter().map(|m| m.time_system)) } - - pub fn peak_memory_usage_mean(&self) -> Information { - mean(self.measurements.iter().map(|m| m.peak_memory_usage)) - } } diff --git a/src/export/csv.rs b/src/export/csv.rs index b7baf14..16071ee 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -86,21 +86,21 @@ fn test_csv() { time_wall_clock: Time::new::(7.0), time_user: Time::new::(7.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(8.0), time_user: Time::new::(8.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(12.0), time_user: Time::new::(12.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -130,21 +130,21 @@ fn test_csv() { time_wall_clock: Time::new::(17.0), time_user: Time::new::(17.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(18.0), time_user: Time::new::(18.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(19.0), time_user: Time::new::(19.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), diff --git a/src/export/json.rs b/src/export/json.rs index 358b419..9336013 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -32,3 +32,50 @@ impl Exporter for JsonExporter { Ok(output?) } } + +#[test] +fn test_json_optional_memory() { + use crate::benchmark::measurement::{Measurement, Measurements}; + use crate::quantity::{byte, Information}; + use serde_json::json; + + for (memory, expected_memory) in [ + (None, None), + ( + Some(Information::new::(0.0)), + Some(json!({"value": 0.0, "unit": "byte"})), + ), + ( + Some(Information::new::(1024.0)), + Some(json!({"value": 1024.0, "unit": "byte"})), + ), + ] { + let result = BenchmarkResult { + command: "example".into(), + measurements: Measurements::new(vec![Measurement { + peak_memory_usage: memory, + ..Measurement::default() + }]), + ..BenchmarkResult::default() + }; + let output = JsonExporter::default() + .serialize(&[result], None, SortOrder::Command) + .unwrap(); + let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); + let mut expected = json!({ + "results": [{ + "command": "example", + "measurements": [{ + "time_wall_clock": {"value": 0.0, "unit": "second"}, + "time_user": {"value": 0.0, "unit": "second"}, + "time_system": {"value": 0.0, "unit": "second"}, + "exit_code": 0 + }] + }] + }); + if let Some(memory) = expected_memory { + expected["results"][0]["measurements"][0]["peak_memory_usage"] = memory; + } + assert_eq!(actual, expected); + } +} diff --git a/src/export/tests.rs b/src/export/tests.rs index ba4d4fa..eeb0804 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -33,21 +33,21 @@ fn test_markup_export_auto_ms() { time_wall_clock: Time::new::(0.09), time_user: Time::new::(0.09), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.10), time_user: Time::new::(0.10), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.14), time_user: Time::new::(0.14), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -60,21 +60,21 @@ fn test_markup_export_auto_ms() { time_wall_clock: Time::new::(2.0), time_user: Time::new::(2.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(3.0), time_user: Time::new::(3.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(4.0), time_user: Time::new::(4.0), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -132,21 +132,21 @@ fn test_markup_export_auto_s() { time_wall_clock: Time::new::(2.1), time_user: Time::new::(2.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.2), time_user: Time::new::(2.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.3), time_user: Time::new::(2.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -159,21 +159,21 @@ fn test_markup_export_auto_s() { time_wall_clock: Time::new::(0.1), time_user: Time::new::(0.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.2), time_user: Time::new::(0.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.3), time_user: Time::new::(0.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -231,21 +231,21 @@ fn test_markup_export_manual_ms() { time_wall_clock: Time::new::(2.1), time_user: Time::new::(2.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.2), time_user: Time::new::(2.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.3), time_user: Time::new::(2.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -258,21 +258,21 @@ fn test_markup_export_manual_ms() { time_wall_clock: Time::new::(0.1), time_user: Time::new::(0.1), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.2), time_user: Time::new::(0.2), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.3), time_user: Time::new::(0.3), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -329,21 +329,21 @@ fn test_markup_export_manual_s() { time_wall_clock: Time::new::(2.01), time_user: Time::new::(2.01), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.02), time_user: Time::new::(2.02), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(2.03), time_user: Time::new::(2.03), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), @@ -356,21 +356,21 @@ fn test_markup_export_manual_s() { time_wall_clock: Time::new::(0.11), time_user: Time::new::(0.11), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.12), time_user: Time::new::(0.12), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, Measurement { time_wall_clock: Time::new::(0.13), time_user: Time::new::(0.13), time_system: Time::zero(), - peak_memory_usage: Information::new::(1024.), + peak_memory_usage: Some(Information::new::(1024.)), exit_status: ExitStatus::default(), }, ]), diff --git a/src/timer/unix_timer.rs b/src/timer/unix_timer.rs index 699a541..63591ee 100644 --- a/src/timer/unix_timer.rs +++ b/src/timer/unix_timer.rs @@ -77,12 +77,12 @@ impl CPUTimer { Self {} } - pub fn stop(&self, child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, child: Child) -> Result<(Time, Time, Option, ExitStatus)> { let (status, usage) = wait4(child)?; Ok(( usage.time_user, usage.time_system, - usage.memory_usage, + Some(usage.memory_usage), status, )) } diff --git a/src/timer/windows_timer.rs b/src/timer/windows_timer.rs index e48e288..3a79e40 100644 --- a/src/timer/windows_timer.rs +++ b/src/timer/windows_timer.rs @@ -84,7 +84,7 @@ impl CPUTimer { Self { job_object } } - pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Information, ExitStatus)> { + pub fn stop(&self, mut child: Child) -> Result<(Time, Time, Option, ExitStatus)> { let status = child.wait()?; let mut job_object_info = @@ -116,9 +116,9 @@ impl CPUTimer { let system_time = Time::new::((job_object_info.TotalKernelTime as f64) * 100.0); - Ok((user_time, system_time, Information::zero(), status)) + Ok((user_time, system_time, None, status)) } else { - Ok((Time::zero(), Time::zero(), Information::zero(), status)) + Ok((Time::zero(), Time::zero(), None, status)) } } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 42c23ab..f35505b 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -42,6 +42,30 @@ fn one_run_is_supported() { .success(); } +#[test] +fn json_memory_availability_matches_platform() { + let output = hyperfine_raw_command() + .args([ + "--runs=1", + "--style=none", + "--export-json=-", + "echo benchmark", + ]) + .output() + .unwrap(); + assert!(output.status.success()); + let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let measurement = &export["results"][0]["measurements"][0]; + let memory = measurement.get("peak_memory_usage"); + if cfg!(windows) { + assert!(memory.is_none(), "unavailable memory must be omitted"); + } else { + let memory = memory.expect("Unix memory measurement must be present"); + assert_eq!(memory["unit"], "byte"); + assert!(memory["value"].as_f64().unwrap() >= 0.0); + } +} + /// Regression test: hyperfine must not panic when writing to a closed /// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process /// that terminates early). A `BrokenPipe` error should result in a quiet