From eedec078127a2dcbe725077132e3d21090b35fc9 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 20:42:13 +0200 Subject: [PATCH] Cover JSON exports with integration snapshots --- src/export/json.rs | 176 --------------- src/export/json/metadata.rs | 8 - tests/integration_tests.rs | 134 +++++------ .../integration_tests__json_export_basic.snap | 199 +++++++++++++++++ ...n_export_parameterized_with_reference.snap | 211 ++++++++++++++++++ ...gration_tests__json_export_single_run.snap | 93 ++++++++ 6 files changed, 557 insertions(+), 264 deletions(-) create mode 100644 tests/snapshots/integration_tests__json_export_basic.snap create mode 100644 tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap create mode 100644 tests/snapshots/integration_tests__json_export_single_run.snap diff --git a/src/export/json.rs b/src/export/json.rs index 4ea123b..e4c252b 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -173,179 +173,3 @@ fn test_statistical_summary_unit_conversion() { approx::assert_relative_eq!(summary.stddev.unwrap(), 2.0_f64.sqrt() * scale); } } - -#[test] -fn test_json_optional_memory() { - use crate::benchmark::measurement::{Measurement, Measurements}; - use crate::quantity::{byte, Information}; - use serde_json::json; - - for (memory, expected_memory) in [ - (None, None), - ( - Some(Information::new::(0.0)), - Some(json!({"value": 0.0, "unit": "byte"})), - ), - ( - Some(Information::new::(1024.0)), - Some(json!({"value": 1024.0, "unit": "byte"})), - ), - ] { - let result = BenchmarkResult { - command: "example".into(), - name: None, - measurements: Measurements::new(vec![Measurement { - memory_peak_resident: memory, - ..Measurement::default() - }]), - ..BenchmarkResult::default() - }; - let mut exporter = JsonExporter::default(); - exporter.metadata.start_time = Some("2000-02-29T12:34:56Z".to_owned()); - let output = exporter - .serialize(&[result], None, SortOrder::Command, None) - .unwrap(); - let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); - let mut expected = json!({ - "schema_version": JSON_SCHEMA_VERSION, - "metadata": { - "hyperfine_version": env!("CARGO_PKG_VERSION"), - "start_time": "2000-02-29T12:34:56Z", - "platform": exporter.metadata.platform - }, - "results": [{ - "command": "example", - "summary": { - "time_wall_clock": { - "unit": "second", - "count": 1, - "mean": 0.0, - "stddev": null, - "median": 0.0, - "min": 0.0, - "max": 0.0 - } - }, - "measurements": [{ - "time_wall_clock": {"value": 0.0, "unit": "second"}, - "time_user": {"value": 0.0, "unit": "second"}, - "time_system": {"value": 0.0, "unit": "second"}, - "exit_code": 0 - }] - }] - }); - for field in ["time_user", "time_system"] { - expected["results"][0]["summary"][field] = - expected["results"][0]["summary"]["time_wall_clock"].clone(); - } - if let Some(memory) = expected_memory { - let value = memory["value"].clone(); - expected["results"][0]["summary"]["memory_peak_resident"] = json!({ - "unit": "byte", "count": 1, "mean": value, - "stddev": null, "median": value, "min": value, "max": value - }); - expected["results"][0]["measurements"][0]["memory_peak_resident"] = memory; - } - assert_eq!(actual, expected); - } -} - -#[test] -fn test_json_summaries_for_parameterized_results() { - use crate::benchmark::benchmark_result::Parameter; - use crate::benchmark::measurement::Measurement; - use crate::quantity::{second, Time}; - use serde_json::json; - - let results: Vec<_> = vec![("small", vec![1.0, 3.0]), ("large", vec![10.0])] - .into_iter() - .map(|(size, times)| BenchmarkResult { - command: "example".into(), - name: None, - parameters: [( - "size".into(), - Parameter { - value: size.into(), - is_unused: size == "small", - }, - )] - .into(), - measurements: Measurements::new( - times - .into_iter() - .map(|time| Measurement { - time_wall_clock: Time::new::(time), - ..Measurement::default() - }) - .collect(), - ), - }) - .collect(); - let output = JsonExporter::default() - .serialize(&results, None, SortOrder::Command, None) - .unwrap(); - let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); - let results = actual["results"].as_array().unwrap(); - assert_eq!(results.len(), 2); - assert_eq!(results[0]["parameters"]["size"], json!({"value": "small"})); - assert_eq!(results[1]["parameters"]["size"], json!({"value": "large"})); - assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2); - assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1); - assert_eq!( - results[0]["summary"]["time_wall_clock"], - json!({ - "unit": "second", "count": 2, "mean": 2.0, - "stddev": 2.0_f64.sqrt(), "median": 2.0, "min": 1.0, "max": 3.0 - }) - ); - assert_eq!( - results[1]["summary"]["time_wall_clock"], - json!({ - "unit": "second", "count": 1, "mean": 10.0, - "stddev": null, "median": 10.0, "min": 10.0, "max": 10.0 - }) - ); -} - -#[test] -fn test_json_summary_metrics_and_missing_samples() { - use crate::benchmark::measurement::Measurement; - use crate::quantity::{byte, second, Information, Time}; - use serde_json::json; - - let measurements = Measurements::new( - vec![(1.0, Some(1024.0)), (2.0, None), (3.0, Some(3072.0))] - .into_iter() - .map(|(time, memory)| Measurement { - time_wall_clock: Time::new::(time), - time_user: Time::new::(time * 2.0), - time_system: Time::new::(time * 3.0), - memory_peak_resident: memory.map(Information::new::), - ..Measurement::default() - }) - .collect(), - ); - let actual = serde_json::to_value(BenchmarkSummary::from_measurements(&measurements)).unwrap(); - assert_eq!( - actual, - json!({ - "time_wall_clock": { - "unit": "second", "count": 3, "mean": 2.0, "stddev": 1.0, - "median": 2.0, "min": 1.0, "max": 3.0 - }, - "time_user": { - "unit": "second", "count": 3, "mean": 4.0, "stddev": 2.0, - "median": 4.0, "min": 2.0, "max": 6.0 - }, - "time_system": { - "unit": "second", "count": 3, "mean": 6.0, "stddev": 3.0, - "median": 6.0, "min": 3.0, "max": 9.0 - }, - "memory_peak_resident": { - "unit": "byte", "count": 2, "mean": 2048.0, - "stddev": 1024.0 * 2.0_f64.sqrt(), - "median": 2048.0, "min": 1024.0, "max": 3072.0 - } - }) - ); -} diff --git a/src/export/json/metadata.rs b/src/export/json/metadata.rs index eca3efe..93b3161 100644 --- a/src/export/json/metadata.rs +++ b/src/export/json/metadata.rs @@ -49,11 +49,3 @@ impl Platform { } } } - -#[test] -fn unavailable_platform_fields_are_omitted() { - assert_eq!( - serde_json::to_value(Platform::default()).unwrap(), - serde_json::json!({}) - ); -} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e07adf2..4b0245a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -24,6 +24,17 @@ fn snapshot_settings() -> insta::Settings { settings } +fn json_snapshot_settings() -> insta::Settings { + let mut settings = snapshot_settings(); + for field in ["hyperfine_version", "start_time", "os", "architecture"] { + settings.add_filter( + &format!(r#"("{field}": ")[^"]+(")"#), + format!(r#"${{1}}[{field}]$2"#), + ); + } + settings +} + #[test] fn runs_successfully() { hyperfine() @@ -42,86 +53,6 @@ fn one_run_is_supported() { .success(); } -#[test] -fn json_includes_name_only_when_different() { - for (args, expected) in [ - (vec!["sleep 0.01"], vec![("sleep 0.01", None)]), - ( - vec!["--command-name=example", "sleep 0.01"], - vec![("sleep 0.01", Some("example"))], - ), - ( - vec!["--command-name=sleep 0.01", "sleep 0.01"], - vec![("sleep 0.01", None)], - ), - ( - vec![ - "-L", - "delay", - "0.01,0.02", - "--command-name=delay {delay}", - "sleep {delay}", - ], - vec![ - ("sleep 0.01", Some("delay 0.01")), - ("sleep 0.02", Some("delay 0.02")), - ], - ), - ( - vec![ - "-L", - "delay", - "0.01,0.02", - "--command-name=sleep {delay}", - "sleep {delay}", - ], - vec![("sleep 0.01", None), ("sleep 0.02", None)], - ), - ] { - let output = hyperfine_debug() - .args(["--runs=1", "--style=none", "--export-json=-"]) - .args(args) - .output() - .unwrap(); - assert!(output.status.success()); - let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let results = export["results"].as_array().unwrap(); - assert_eq!(results.len(), expected.len()); - for (result, (command, name)) in results.iter().zip(expected) { - assert_eq!(result["command"], command); - assert!(result.get("command_raw").is_none()); - assert_eq!( - result.get("name"), - name.map(serde_json::Value::from).as_ref() - ); - } - } -} - -#[test] -fn json_memory_availability_matches_platform() { - let output = hyperfine_raw_command() - .args([ - "--runs=1", - "--style=none", - "--export-json=-", - "echo benchmark", - ]) - .output() - .unwrap(); - assert!(output.status.success()); - let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let measurement = &export["results"][0]["measurements"][0]; - let memory = measurement.get("memory_peak_resident"); - if cfg!(windows) { - assert!(memory.is_none(), "unavailable memory must be omitted"); - } else { - let memory = memory.expect("Unix memory measurement must be present"); - assert_eq!(memory["unit"], "byte"); - assert!(memory["value"].as_f64().unwrap() >= 0.0); - } -} - /// Regression test: hyperfine must not panic when writing to a closed /// stdout pipe (e.g. `hyperfine ... | head -n 1` or a downstream process /// that terminates early). A `BrokenPipe` error should result in a quiet @@ -1353,3 +1284,46 @@ fn hyperfine_iteration_env_var_in_prepare_and_conclude_commands() { ] ); } + +#[test] +fn json_export_basic() { + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=2", + "--warmup=1", + "--reference-name=one second", + "--reference=sleep 1", + "--command-name=sleep 2", + "sleep 2", + ])); +} + +#[test] +fn json_export_single_run() { + // Make sure that the standard deviation is set to `null` + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=1", + "sleep 1", + ])); +} + +#[test] +fn json_export_parameterized_with_reference() { + let _settings = json_snapshot_settings().bind_to_scope(); + assert_cmd_snapshot!(hyperfine_debug().args([ + "--style=none", + "--export-json=-", + "--runs=2", + "-L", + "duration", + "1,2", + "--command-name=sleep for {duration} seconds", + "--reference=sleep for 2 seconds", + "sleep {duration}", + ])); +} diff --git a/tests/snapshots/integration_tests__json_export_basic.snap b/tests/snapshots/integration_tests__json_export_basic.snap new file mode 100644 index 0000000..372f79f --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_basic.snap @@ -0,0 +1,199 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=2" + - "--warmup=1" + - "--reference-name=one second" + - "--reference=sleep 1" + - "--command-name=sleep 2" + - sleep 2 +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "name": "one second", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "is_reference": true, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 1.0, + "stddev": 0.0, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + }, + { + "command": "sleep 2", + "measurements": [ + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 2.0, + "stddev": 0.0, + "median": 2.0, + "min": 2.0, + "max": 2.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr ----- diff --git a/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap b/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap new file mode 100644 index 0000000..dc91fdd --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_parameterized_with_reference.snap @@ -0,0 +1,211 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=2" + - "-L" + - duration + - "1,2" + - "--command-name=sleep for {duration} seconds" + - "--reference=sleep for 2 seconds" + - "sleep {duration}" +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "name": "sleep for 1 seconds", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "parameters": { + "duration": { + "value": "1" + } + }, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 1.0, + "stddev": 0.0, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + }, + { + "command": "sleep 2", + "name": "sleep for 2 seconds", + "measurements": [ + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + }, + { + "time_wall_clock": { + "value": 2.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "parameters": { + "duration": { + "value": "2" + } + }, + "is_reference": true, + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 2, + "mean": 2.0, + "stddev": 0.0, + "median": 2.0, + "min": 2.0, + "max": 2.0 + }, + "time_user": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 2, + "mean": 0.0, + "stddev": 0.0, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr ----- diff --git a/tests/snapshots/integration_tests__json_export_single_run.snap b/tests/snapshots/integration_tests__json_export_single_run.snap new file mode 100644 index 0000000..7f2671c --- /dev/null +++ b/tests/snapshots/integration_tests__json_export_single_run.snap @@ -0,0 +1,93 @@ +--- +source: tests/integration_tests.rs +info: + program: hyperfine + args: + - "--debug-mode" + - "--style=none" + - "--export-json=-" + - "--runs=1" + - sleep 1 +--- +success: true +exit_code: 0 +----- stdout ----- + +{ + "schema_version": 2, + "metadata": { + "hyperfine_version": "[hyperfine_version]", + "start_time": "[start_time]", + "platform": { + "os": "[os]", + "architecture": "[architecture]" + } + }, + "results": [ + { + "command": "sleep 1", + "measurements": [ + { + "time_wall_clock": { + "value": 1.0, + "unit": "second" + }, + "time_user": { + "value": 0.0, + "unit": "second" + }, + "time_system": { + "value": 0.0, + "unit": "second" + }, + "memory_peak_resident": { + "value": 0.0, + "unit": "byte" + }, + "exit_code": 0 + } + ], + "summary": { + "time_wall_clock": { + "unit": "second", + "count": 1, + "mean": 1.0, + "stddev": null, + "median": 1.0, + "min": 1.0, + "max": 1.0 + }, + "time_user": { + "unit": "second", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "time_system": { + "unit": "second", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + }, + "memory_peak_resident": { + "unit": "byte", + "count": 1, + "mean": 0.0, + "stddev": null, + "median": 0.0, + "min": 0.0, + "max": 0.0 + } + } + } + ] +} + + +----- stderr -----