diff --git a/README.md b/README.md index 1032ccd..d0418d5 100644 --- a/README.md +++ b/README.md @@ -158,23 +158,16 @@ You can use the `--export-markdown ` option to create tables like the foll #### JSON -The JSON export contains a top-level `"schema_version": 2` and a `results` array with one entry -per benchmark. Earlier exports without `schema_version` use the legacy format (version 1). -The schema version is independent of the hyperfine release version and changes when the format -changes incompatibly; adding optional metrics does not require a new schema version. +The JSON export includes the following metrics for each measured run (excluding warmup runs): -Each result has a `measurements` array with one object per measured run (excluding warmup runs): - -| Field | Format | Meaning and platform differences | +| Metric | Unit | Meaning and platform differences | |:---|:---|:---| -| `time_wall_clock` | `{ "value": number, "unit": "second" }` | Elapsed time, including time spent waiting or descheduled. Available on all platforms. | -| `time_user` | `{ "value": number, "unit": "second" }` | CPU time spent in user mode, accumulated across threads. On Unix, includes descendants whose usage propagates through parents waiting for them. On Windows, includes processes associated with the benchmark's job object. | -| `time_system` | `{ "value": number, "unit": "second" }` | CPU time spent in kernel mode, with the same process accounting as `time_user`. | -| `peak_memory_usage` | `{ "value": number, "unit": "byte" }` | Peak resident set size (RSS) on Linux and macOS; omitted on Windows. This is the largest per-process peak among the command and descendants whose usage is collected by waiting for them, **not the simultaneous total memory of the process tree**. Linux's KiB counter and macOS's byte counter are both exported in bytes. On macOS, this is RSS, not Apple's separate memory-footprint metric. | -| `exit_code` | Integer or `null` | Process exit code; `0` indicates success. On Unix, termination by a signal is encoded as `128 + signal number`. `null` means no exit code could be determined. Failed runs are included when allowed by `--ignore-failure`. | +| `time_wall_clock` | second | Elapsed time, including time spent waiting or descheduled. Available on all platforms. | +| `time_user` | second | CPU time spent in user mode, accumulated across threads. On Unix, includes descendants whose usage propagates through parents waiting for them. On Windows, includes processes associated with the benchmark's job object. | +| `time_system` | second | CPU time spent in kernel mode, with the same process accounting as `time_user`. | +| `peak_memory_usage` | byte | Peak resident set size (RSS) on Linux and macOS; omitted on Windows. This is the largest per-process peak among the command and descendants whose usage is collected by waiting for them, **not the simultaneous total memory of the process tree**. Linux's KiB counter and macOS's byte counter are both exported in bytes. On macOS, this is RSS, not Apple's separate memory-footprint metric. | -Metrics unavailable on a platform are omitted, rather than reported as zero. Consumers should -accept missing metrics and additional fields as new metrics are introduced. Time values are always +Metrics unavailable on a platform are omitted, rather than reported as zero. Time values are always exported in seconds, regardless of `--time-unit`. When using a shell, the estimated shell startup CPU and wall-clock times are subtracted; memory usage is not adjusted. diff --git a/scripts/advanced_statistics.py b/scripts/advanced_statistics.py index 310bd53..99394eb 100755 --- a/scripts/advanced_statistics.py +++ b/scripts/advanced_statistics.py @@ -61,7 +61,7 @@ for result in results: iqr = p75 - p25 - print(f"Command '{result['command']}'") + print(f"Command '{result.get('name', result['command'])}'") print(f" runs: {summary['count']:8d}") print(f" mean: {summary['mean'] * unit.factor():8.3f} {unit_str}") if summary["stddev"] is None: diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index e30f15d..754ee17 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -44,7 +44,7 @@ if args.benchmark_names: for i, filename in enumerate(args.files): with open(filename) as f: results = json.load(f)["results"] - benchmark_commands = [b["command"] for b in results] + benchmark_commands = [b.get("name", b["command"]) for b in results] if commands is None: commands = benchmark_commands else: diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index 4ce1bd1..13d51c4 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -66,7 +66,7 @@ with open(args.file) as f: if args.labels: labels = args.labels.split(",") else: - labels = [b["command"] for b in results] + labels = [b.get("name", b["command"]) for b in results] all_times = [ [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results ] diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index 1932934..9cd7808 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -52,7 +52,7 @@ with open(args.file) as f: results = json.load(f)["results"] for result in results: - label = result["command"] + label = result.get("name", result["command"]) times = [m["time_wall_clock"]["value"] for m in result["measurements"]] num = len(times) nums = range(num) @@ -76,7 +76,7 @@ if args.title: legend = [] for result in results: - legend.append(result["command"]) + legend.append(result.get("name", result["command"])) if not args.no_moving_average: legend.append("moving average") plt.legend(legend) diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index ab27a23..baaf175 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -37,7 +37,7 @@ with open(args.file, encoding="utf-8") as f: if args.labels: labels = args.labels.split(",") else: - labels = [b["command"] for b in results] + labels = [b.get("name", b["command"]) for b in results] times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results] if args.sort_by == "median": diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index 4d87d8a..f6e48b3 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -26,7 +26,7 @@ if len(results) != 2: print("The input file has to contain exactly two benchmarks") sys.exit(1) -a, b = (x["command"] for x in results[:2]) +a, b = (x.get("name", x["command"]) for x in results[:2]) X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2]) print(f"Command 1: {a}") diff --git a/src/benchmark/benchmark_result.rs b/src/benchmark/benchmark_result.rs index 7e32d15..cbd552d 100644 --- a/src/benchmark/benchmark_result.rs +++ b/src/benchmark/benchmark_result.rs @@ -9,18 +9,19 @@ use crate::quantity::Time; #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct Parameter { pub value: String, + #[serde(skip_serializing)] pub is_unused: bool, } /// Meta data and performance metrics for a single benchmark #[derive(Debug, Default, Clone, Serialize, PartialEq)] pub struct BenchmarkResult { - /// The display name of the command being benchmarked + /// The expanded command line being benchmarked pub command: String, - /// The expanded command line, when it differs from the display name + /// The display name, when it differs from the command line #[serde(skip_serializing_if = "Option::is_none")] - pub command_raw: Option, + pub name: Option, /// Performance metric measurements and exit codes for each run #[serde(flatten)] @@ -32,12 +33,16 @@ pub struct BenchmarkResult { } impl BenchmarkResult { + pub fn get_name(&self) -> &str { + self.name.as_deref().unwrap_or(&self.command) + } + /// The average wall clock time pub fn mean_wall_clock_time(&self) -> Time { self.measurements.time_wall_clock_mean() } - /// The full command line of the program that is being benchmarked, possibly including a list of + /// The display name of the command, possibly including a list of /// parameters that were not used in the command line template. pub fn command_with_unused_parameters(&self) -> String { let parameters = self @@ -54,6 +59,6 @@ impl BenchmarkResult { format!(" ({parameters})") }; - format!("{}{}", self.command, parameters) + format!("{}{}", self.get_name(), parameters) } } diff --git a/src/benchmark/mod.rs b/src/benchmark/mod.rs index 5c074b2..3fb8709 100644 --- a/src/benchmark/mod.rs +++ b/src/benchmark/mod.rs @@ -475,13 +475,13 @@ impl<'a> Benchmark<'a> { self.run_cleanup_command(self.command.get_parameters().iter().cloned(), output_policy)?; - let command = self.command.get_name(); - let command_raw = self.command.get_command_line(); - let command_raw = (command_raw != command).then_some(command_raw); + let command = self.command.get_command_line(); + let name = self.command.get_name(); + let name = (name != command).then_some(name); Ok(BenchmarkResult { command, - command_raw, + name, measurements, parameters: self .command diff --git a/src/benchmark/relative_speed.rs b/src/benchmark/relative_speed.rs index 333814c..b8edc45 100644 --- a/src/benchmark/relative_speed.rs +++ b/src/benchmark/relative_speed.rs @@ -158,7 +158,7 @@ fn create_result(name: &str, mean: f64) -> BenchmarkResult { BenchmarkResult { command: name.into(), - command_raw: None, + name: None, measurements: Measurements { measurements: vec![Measurement { time_wall_clock: Time::new::(mean), diff --git a/src/export/csv.rs b/src/export/csv.rs index 31f783f..c7900b3 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -42,7 +42,7 @@ impl Exporter for CsvExporter { } for res in results { - let mut fields = vec![Cow::Borrowed(res.command.as_bytes())]; + let mut fields = vec![Cow::Borrowed(res.get_name().as_bytes())]; for f in &[ res.mean_wall_clock_time(), res.measurements.stddev().unwrap_or_default(), @@ -80,8 +80,8 @@ fn test_csv() { let results = vec![ BenchmarkResult { - command: String::from("command_a"), - command_raw: None, + command: String::from("echo command_a"), + name: Some(String::from("command_a")), measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(7.0), @@ -126,7 +126,7 @@ fn test_csv() { }, BenchmarkResult { command: String::from("command_b"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(17.0), diff --git a/src/export/json.rs b/src/export/json.rs index 5d93d02..6911ecb 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -15,8 +15,11 @@ use anyhow::Result; mod metadata; use metadata::Metadata; +const JSON_SCHEMA_VERSION: u32 = 2; + #[derive(Serialize)] struct HyperfineSummary<'a> { + schema_version: u32, metadata: &'a Metadata, results: Vec>, } @@ -116,6 +119,7 @@ impl Exporter for JsonExporter { _reference_index: Option, ) -> Result> { let mut output = to_vec_pretty(&HyperfineSummary { + schema_version: JSON_SCHEMA_VERSION, metadata: &self.metadata, results: results .iter() @@ -182,7 +186,7 @@ fn test_json_optional_memory() { ] { let result = BenchmarkResult { command: "example".into(), - command_raw: None, + name: None, measurements: Measurements::new(vec![Measurement { peak_memory_usage: memory, ..Measurement::default() @@ -196,8 +200,8 @@ fn test_json_optional_memory() { .unwrap(); let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); let mut expected = json!({ + "schema_version": JSON_SCHEMA_VERSION, "metadata": { - "json_schema_version": metadata::JSON_SCHEMA_VERSION, "hyperfine_version": env!("CARGO_PKG_VERSION"), "start_time": "2000-02-29T12:34:56Z", "platform": exporter.metadata.platform @@ -250,12 +254,12 @@ fn test_json_summaries_for_parameterized_results() { .into_iter() .map(|(size, times)| BenchmarkResult { command: "example".into(), - command_raw: None, + name: None, parameters: [( "size".into(), Parameter { value: size.into(), - is_unused: true, + is_unused: size == "small", }, )] .into(), @@ -276,8 +280,8 @@ fn test_json_summaries_for_parameterized_results() { let actual: serde_json::Value = serde_json::from_slice(&output).unwrap(); let results = actual["results"].as_array().unwrap(); assert_eq!(results.len(), 2); - assert_eq!(results[0]["parameters"]["size"]["value"], "small"); - assert_eq!(results[1]["parameters"]["size"]["value"], "large"); + assert_eq!(results[0]["parameters"]["size"], json!({"value": "small"})); + assert_eq!(results[1]["parameters"]["size"], json!({"value": "large"})); assert_eq!(results[0]["measurements"].as_array().unwrap().len(), 2); assert_eq!(results[1]["measurements"].as_array().unwrap().len(), 1); assert_eq!( diff --git a/src/export/json/metadata.rs b/src/export/json/metadata.rs index dec87f5..eca3efe 100644 --- a/src/export/json/metadata.rs +++ b/src/export/json/metadata.rs @@ -1,10 +1,7 @@ use serde::Serialize; -pub(super) const JSON_SCHEMA_VERSION: u32 = 2; - #[derive(Debug, Serialize)] pub(super) struct Metadata { - json_schema_version: u32, hyperfine_version: &'static str, #[serde(skip_serializing_if = "Option::is_none")] pub start_time: Option, @@ -14,7 +11,6 @@ pub(super) struct Metadata { impl Default for Metadata { fn default() -> Self { Self { - json_schema_version: JSON_SCHEMA_VERSION, hyperfine_version: env!("CARGO_PKG_VERSION"), start_time: utc_now(), platform: Platform::detect(), diff --git a/src/export/tests.rs b/src/export/tests.rs index ab21be3..cc67c09 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -28,7 +28,7 @@ fn test_markup_export_auto_ms() { let results = [ BenchmarkResult { command: String::from("sleep 0.1"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.09), @@ -56,7 +56,7 @@ fn test_markup_export_auto_ms() { }, BenchmarkResult { command: String::from("sleep 2"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.0), @@ -129,7 +129,7 @@ fn test_markup_export_auto_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.1), @@ -157,7 +157,7 @@ fn test_markup_export_auto_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.1), @@ -230,7 +230,7 @@ fn test_markup_export_manual_ms() { let timing_results = [ BenchmarkResult { command: String::from("sleep 2"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.1), @@ -258,7 +258,7 @@ fn test_markup_export_manual_ms() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.1), @@ -330,7 +330,7 @@ fn test_markup_export_manual_s() { let results = [ BenchmarkResult { command: String::from("sleep 2"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(2.01), @@ -358,7 +358,7 @@ fn test_markup_export_manual_s() { }, BenchmarkResult { command: String::from("sleep 0.1"), - command_raw: None, + name: None, measurements: Measurements::new(vec![ Measurement { time_wall_clock: Time::new::(0.11), diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 58290c7..cbfe33a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -43,12 +43,12 @@ fn one_run_is_supported() { } #[test] -fn json_includes_raw_command_only_when_different() { +fn json_includes_name_only_when_different() { for (args, expected) in [ (vec!["sleep 0.01"], vec![("sleep 0.01", None)]), ( vec!["--command-name=example", "sleep 0.01"], - vec![("example", Some("sleep 0.01"))], + vec![("sleep 0.01", Some("example"))], ), ( vec!["--command-name=sleep 0.01", "sleep 0.01"], @@ -63,8 +63,8 @@ fn json_includes_raw_command_only_when_different() { "sleep {delay}", ], vec![ - ("delay 0.01", Some("sleep 0.01")), - ("delay 0.02", Some("sleep 0.02")), + ("sleep 0.01", Some("delay 0.01")), + ("sleep 0.02", Some("delay 0.02")), ], ), ( @@ -87,11 +87,12 @@ fn json_includes_raw_command_only_when_different() { let export: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); let results = export["results"].as_array().unwrap(); assert_eq!(results.len(), expected.len()); - for (result, (command, command_raw)) in results.iter().zip(expected) { + for (result, (command, name)) in results.iter().zip(expected) { assert_eq!(result["command"], command); + assert!(result.get("command_raw").is_none()); assert_eq!( - result.get("command_raw"), - command_raw.map(serde_json::Value::from).as_ref() + result.get("name"), + name.map(serde_json::Value::from).as_ref() ); } }