vsndrg/hyperfine
1
0 Fork 0
mirror of https://github.com/sharkdp/hyperfine.git synced 2026-10-31 00:40:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cover JSON exports with integration snapshots

Browse Source
This commit is contained in:
David Peter committed 2026-10-05 20:42:13 +02:00
1 parent cb021e64b9
commit eedec07812
6 files changed
+557 -264
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/export/json.rs
-176
View File
@@ -173,179 +173,3 @@ fn test_statistical_summary_unit_conversion() {
approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale);
}
}
#[test]
fn test_json_optional_memory() {
use crate::benchmark::measurement::{Measurement, Measurements};
use crate::quantity::{byte, Information};
use serde_json::json;
for (memory, expected_memory) in [
(None, None),
(
Some(Information::new::<byte>(0.0)),
Some(json!({"value": 0.0, "unit": "byte"})),
),
(
Some(Information::new::<byte>(1024.0)),
Some(json!({"value": 1024.0, "unit": "byte"})),
),
] {
let result = BenchmarkResult {
command: "example".into(),
name: None,
measurements: Measurements::new(vec![Measurement {
memory_peak_resident: memory,
..Measurement::default()
}]),
..BenchmarkResult::default()
};
let mut exporter = JsonExporter::default();
exporter.metadata.start_time = Some("2000-02-29T12:34:56Z".to_owned());
let output = exporter
.serialize(&[result], None, SortOrder::Command, None)
.unwrap();
let actual: serde_json::Value = serde_json::from_slice(&output).unwrap();
let mut expected = json!({
"schema_version": JSON_SCHEMA_VERSION,
"metadata": {
"hyperfine_version": env!("CARGO_PKG_VERSION"),
"start_time": "2000-02-29T12:34:56Z",
"platform": exporter.metadata.platform
},
"results": [{
"command": "example",
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
},
"measurements": [{
"time_wall_clock": {"value": 0.0, "unit": "second"},
"time_user": {"value": 0.0, "unit": "second"},
"time_system": {"value": 0.0, "unit": "second"},
"exit_code": 0
}]
}]
});
for field in ["time_user", "time_system"] {
expected["results"][0]["summary"][field] =
expected["results"][0]["summary"]["time_wall_clock"].clone();
}
if let Some(memory) = expected_memory {
let value = memory["value"].clone();
expected["results"][0]["summary"]["memory_peak_resident"] = json!({
"unit": "byte", "count": 1, "mean": value,
"stddev": null, "median": value, "min": value, "max": value
});
expected["results"][0]["measurements"][0]["memory_peak_resident"] = memory;
}
assert_eq!(actual, expected);
}
}
#[test]
fn test_json_summaries_for_parameterized_results() {
use crate::benchmark::benchmark_result::Parameter;
use crate::benchmark::measurement::Measurement;
use crate::quantity::{second, Time};
use serde_json::json;
let results: Vec<_> = vec![("small", vec![1.0, 3.0]), ("large", vec![10.0])]
.into_iter()
.map(|(size, times)| BenchmarkResult {
command: "example".into(),
name: None,
parameters: [(
"size".into(),
Parameter {
value: size.into(),
is_unused: size == "small",
},
)]
.into(),
measurements: Measurements::new(
times
.into_iter()
.map(|time| Measurement {
time_wall_clock: Time::new::<second>(time),
..Measurement::default()
})
.collect(),
),
})
.collect();
let output = JsonExporter::default()
.serialize(&results, None, SortOrder::Command, None)
.unwrap();
let actual: serde_json::Value = serde_json::from_slice(&output).unwrap();
let results = actual["results"].as_array().unwrap();
assert_eq!(results.len(), 2);
assert_eq!(results[0]["parameters"]["size"], json!({"value": "small"}));
assert_eq!(results[1]["parameters"]["size"], json!({"value": "large"}));
assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2);
assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1);
assert_eq!(
results[0]["summary"]["time_wall_clock"],
json!({
"unit": "second", "count": 2, "mean": 2.0,
"stddev": 2.0_f64.sqrt(), "median": 2.0, "min": 1.0, "max": 3.0
})
);
assert_eq!(
results[1]["summary"]["time_wall_clock"],
json!({
"unit": "second", "count": 1, "mean": 10.0,
"stddev": null, "median": 10.0, "min": 10.0, "max": 10.0
})
);
}
#[test]
fn test_json_summary_metrics_and_missing_samples() {
use crate::benchmark::measurement::Measurement;
use crate::quantity::{byte, second, Information, Time};
use serde_json::json;
let measurements = Measurements::new(
vec![(1.0, Some(1024.0)), (2.0, None), (3.0, Some(3072.0))]
.into_iter()
.map(|(time, memory)| Measurement {
time_wall_clock: Time::new::<second>(time),
time_user: Time::new::<second>(time * 2.0),
time_system: Time::new::<second>(time * 3.0),
memory_peak_resident: memory.map(Information::new::<byte>),
..Measurement::default()
})
.collect(),
);
let actual = serde_json::to_value(BenchmarkSummary::from_measurements(&measurements)).unwrap();
assert_eq!(
actual,
json!({
"time_wall_clock": {
"unit": "second", "count": 3, "mean": 2.0, "stddev": 1.0,
"median": 2.0, "min": 1.0, "max": 3.0
},
"time_user": {
"unit": "second", "count": 3, "mean": 4.0, "stddev": 2.0,
"median": 4.0, "min": 2.0, "max": 6.0
},
"time_system": {
"unit": "second", "count": 3, "mean": 6.0, "stddev": 3.0,
"median": 6.0, "min": 3.0, "max": 9.0
},
"memory_peak_resident": {
"unit": "byte", "count": 2, "mean": 2048.0,
"stddev": 1024.0 * 2.0_f64.sqrt(),
"median": 2048.0, "min": 1024.0, "max": 3072.0
}
})
);
}
src/export/json/metadata.rs
-8
View File
@@ -49,11 +49,3 @@ impl Platform {
}
}
}
#[test]
fn unavailable_platform_fields_are_omitted() {
assert_eq!(
serde_json::to_value(Platform::default()).unwrap(),
serde_json::json!({})
);
}
tests/integration_tests.rs
+54 -80
View File
@@ -24,6 +24,17 @@ fn snapshot_settings() -> insta::Settings {
settings
}
fn json_snapshot_settings() -> insta::Settings {
let mut settings = snapshot_settings();
for field in ["hyperfine_version", "start_time", "os", "architecture"] {
settings.add_filter(
&format!(r#"("{field}": ")[^"]+(")"#),
format!(r#"${{1}}[{field}]$2"#),
);
}
settings
}
#[test]
fn runs_successfully() {
hyperfine()
@@ -42,86 +53,6 @@ fn one_run_is_supported() {
.success();
}
#[test]
fn json_includes_name_only_when_different() {
for (args, expected) in [
(vec!["sleep 0.01"], vec![("sleep 0.01", None)]),
(
vec!["--command-name=example", "sleep 0.01"],
vec![("sleep 0.01", Some("example"))],
),
(
vec!["--command-name=sleep 0.01", "sleep 0.01"],
vec![("sleep 0.01", None)],
),
(
vec![
"-L",
"delay",
"0.01,0.02",
"--command-name=delay {delay}",
"sleep {delay}",
],
vec![
("sleep 0.01", Some("delay 0.01")),
("sleep 0.02", Some("delay 0.02")),
],
),
(
vec![
"-L",
"delay",
"0.01,0.02",
"--command-name=sleep {delay}",
"sleep {delay}",
],
vec![("sleep 0.01", None), ("sleep 0.02", None)],
),
] {
let output = hyperfine_debug()
.args(["--runs=1", "--style=none", "--export-json=-"])
.args(args)
.output()
.unwrap();
assert!(output.status.success());
let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
let results = export["results"].as_array().unwrap();
assert_eq!(results.len(), expected.len());
for (result, (command, name)) in results.iter().zip(expected) {
assert_eq!(result["command"], command);
assert!(result.get("command_raw").is_none());
assert_eq!(
result.get("name"),
name.map(serde_json::Value::from).as_ref()
);
}
}
}
#[test]
fn json_memory_availability_matches_platform() {
let output = hyperfine_raw_command()
.args([
"--runs=1",
"--style=none",
"--export-json=-",
"echo benchmark",
])
.output()
.unwrap();
assert!(output.status.success());
let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
let measurement = &export["results"][0]["measurements"][0];
let memory = measurement.get("memory_peak_resident");
if cfg!(windows) {
assert!(memory.is_none(), "unavailable memory must be omitted");
} else {
let memory = memory.expect("Unix memory measurement must be present");
assert_eq!(memory["unit"], "byte");
assert!(memory["value"].as_f64().unwrap() >= 0.0);
}
}
/// Regression test: hyperfine must not panic when writing to a closed
/// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process
/// that terminates early). A `BrokenPipe` error should result in a quiet
@@ -1353,3 +1284,46 @@ fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() {
]
);
}
#[test]
fn json_export_basic() {
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=2",
"--warmup=1",
"--reference-name=one second",
"--reference=sleep 1",
"--command-name=sleep 2",
"sleep 2",
]));
}
#[test]
fn json_export_single_run() {
// Make sure that the standard deviation is set to `null`
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=1",
"sleep 1",
]));
}
#[test]
fn json_export_parameterized_with_reference() {
let _settings = json_snapshot_settings().bind_to_scope();
assert_cmd_snapshot!(hyperfine_debug().args([
"--style=none",
"--export-json=-",
"--runs=2",
"-L",
"duration",
"1,2",
"--command-name=sleep for {duration} seconds",
"--reference=sleep for 2 seconds",
"sleep {duration}",
]));
}
tests/snapshots/integration_tests__json_export_basic.snap
+199
View File
@@ -0,0 +1,199 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=2"
- "--warmup=1"
- "--reference-name=one second"
- "--reference=sleep 1"
- "--command-name=sleep 2"
- sleep 2
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"name": "one second",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"is_reference": true,
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 1.0,
"stddev": 0.0,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
{
"command": "sleep 2",
"measurements": [
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 2.0,
"stddev": 0.0,
"median": 2.0,
"min": 2.0,
"max": 2.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----
tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap
+211
View File
@@ -0,0 +1,211 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=2"
- "-L"
- duration
- "1,2"
- "--command-name=sleep for {duration} seconds"
- "--reference=sleep for 2 seconds"
- "sleep {duration}"
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"name": "sleep for 1 seconds",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"parameters": {
"duration": {
"value": "1"
}
},
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 1.0,
"stddev": 0.0,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
},
{
"command": "sleep 2",
"name": "sleep for 2 seconds",
"measurements": [
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
},
{
"time_wall_clock": {
"value": 2.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"parameters": {
"duration": {
"value": "2"
}
},
"is_reference": true,
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 2,
"mean": 2.0,
"stddev": 0.0,
"median": 2.0,
"min": 2.0,
"max": 2.0
},
"time_user": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 2,
"mean": 0.0,
"stddev": 0.0,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----
tests/snapshots/integration_tests__json_export_single_run.snap
+93
View File
@@ -0,0 +1,93 @@
---
source: tests/integration_tests.rs
info:
program: hyperfine
args:
- "--debug-mode"
- "--style=none"
- "--export-json=-"
- "--runs=1"
- sleep 1
---
success: true
exit_code: 0
----- stdout -----
{
"schema_version": 2,
"metadata": {
"hyperfine_version": "[hyperfine_version]",
"start_time": "[start_time]",
"platform": {
"os": "[os]",
"architecture": "[architecture]"
}
},
"results": [
{
"command": "sleep 1",
"measurements": [
{
"time_wall_clock": {
"value": 1.0,
"unit": "second"
},
"time_user": {
"value": 0.0,
"unit": "second"
},
"time_system": {
"value": 0.0,
"unit": "second"
},
"memory_peak_resident": {
"value": 0.0,
"unit": "byte"
},
"exit_code": 0
}
],
"summary": {
"time_wall_clock": {
"unit": "second",
"count": 1,
"mean": 1.0,
"stddev": null,
"median": 1.0,
"min": 1.0,
"max": 1.0
},
"time_user": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"time_system": {
"unit": "second",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
},
"memory_peak_resident": {
"unit": "byte",
"count": 1,
"mean": 0.0,
"stddev": null,
"median": 0.0,
"min": 0.0,
"max": 0.0
}
}
}
]
}
----- stderr -----