From 7cf08c30dcce394700ae5b4fcb324c9ca42a4b04 Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 7 Oct 2026 21:17:25 +0200 Subject: [PATCH] Export selected metrics side by side in CSV --- README.md | 13 ++++- doc/hyperfine.1 | 10 ++-- src/benchmark/scheduler.rs | 2 +- src/cli.rs | 7 +-- src/export/csv.rs | 103 +++++++++++++++++++------------------ src/export/json.rs | 7 ++- src/export/markup.rs | 7 ++- src/export/mod.rs | 16 +++--- src/export/tests.rs | 4 +- src/main.rs | 2 +- 10 files changed, 101 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index eefa583..7059fad 100644 --- a/README.md +++ b/README.md @@ -239,8 +239,9 @@ You can also use a preset to select a group of metrics: ### Exporting results -Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats -contain only the primary metric, which is the first metric selected by `--metrics`. +Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. CSV exports all +selected metrics, while Markdown, AsciiDoc, and org-mode contain only the primary metric, +which is the first metric selected by `--metrics`. #### Markdown @@ -276,6 +277,14 @@ uvx --from 'git+https://github.com/sharkdp/hyperfine' \ |---:|---:| +#### CSV + +Use `--export-csv ` to export one row per command. Each metric selected by `--metrics` +contributes a group of columns, in selection order: `_unit`, `_mean`, +`_stddev`, `_median`, `_min`, and `_max`. + +CSV uses explicitly selected units, or base units (seconds, bytes, or counts) when omitted. + ### Detailed benchmark flowchart The following chart explains the execution order of various benchmark runs when using options diff --git a/doc/hyperfine.1 b/doc/hyperfine.1 index f91b656..8ef77d5 100644 --- a/doc/hyperfine.1 +++ b/doc/hyperfine.1 @@ -232,7 +232,7 @@ of the tool. .HP \fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|time|all\fP .IP -Select the metrics to display, in order, with an optional unit for each. +Select the metrics to measure, in order, with an optional unit for each. Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR on Windows, where peak resident memory is not yet supported. .IP @@ -266,9 +266,11 @@ Export the primary metric's summary statistics as an AsciiDoc table to the given .HP \fB\-\-export\-csv\fR \fIFILE\fP .IP -Export the primary metric's summary statistics as CSV to the given \fIFILE\fP, -identifying the metric and unit explicitly. Use the explicitly selected unit, -or the base unit (seconds, bytes, or unscaled counts) when omitted. +Export summary statistics for all selected metrics as CSV to the given \fIFILE\fP, +one row per command. Each metric contributes columns for unit, mean, stddev, median, +min, and max, prefixed with the metric name, in selection order. The command column +comes first and parameter columns come last. Use the explicitly selected units, +or base units (seconds, bytes, or unscaled counts) when omitted. .HP \fB\-\-export\-json\fR \fIFILE\fP .IP diff --git a/src/benchmark/scheduler.rs b/src/benchmark/scheduler.rs index d47dcbc..1335fcb 100644 --- a/src/benchmark/scheduler.rs +++ b/src/benchmark/scheduler.rs @@ -69,7 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result> { assert_eq!(options.executor_kind, ExecutorKind::Mock(None)); let commands = Commands::from_cli_arguments(&cli_arguments)?; - let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?; + let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?; options.validate_against_command_list(&commands)?; diff --git a/src/cli.rs b/src/cli.rs index e611e14..0f714d9 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -58,7 +58,7 @@ fn build_command() -> Command { .hide_default_value(true) .help("Performance metrics to measure, in order.\n\n\ This is either a preset name, or a comma-separated list of metrics with optional units. \ - The first metric is used for non-JSON exports, \ + CSV exports all selected metrics; markup exports use the first metric. \ JSON always includes all collected metrics.\n\n\ Presets (use one on its own):\n \ default Wall-clock time and peak RSS\n \ @@ -303,9 +303,10 @@ fn build_command() -> Command { .action(ArgAction::Set) .value_name("FILE") .value_hint(ValueHint::FilePath) - .help("Export the primary metric summary statistics as CSV to the given FILE. If you need \ + .help("Export summary statistics for all selected metrics as CSV to the given FILE. If you need \ all metrics for each individual run, use the JSON export format. \ - Use the explicitly selected unit, or the base unit when unspecified."), + Column names are prefixed with the metric name. \ + Use the explicitly selected units, or base units when unspecified."), ) .arg( Arg::new("export-json") diff --git a/src/export/csv.rs b/src/export/csv.rs index 4dd8c31..f42bb1b 100644 --- a/src/export/csv.rs +++ b/src/export/csv.rs @@ -4,23 +4,34 @@ use super::Exporter; use crate::benchmark::benchmark_result::BenchmarkResult; use crate::metric::MetricSelection; -use anyhow::{Context, Result}; +use anyhow::Result; #[derive(Default)] pub struct CsvExporter {} impl Exporter for CsvExporter { - fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result> { - // CSV never chooses units automatically: an omitted suffix means base units. - let unit = primary.csv_unit(); + fn serialize( + &self, + results: &[BenchmarkResult], + metrics: &[MetricSelection], + ) -> Result> { + // Omit metrics unavailable for all commands (possible with `--metrics all`). + let metrics: Vec<_> = metrics + .iter() + .filter(|selection| { + results + .iter() + .any(|result| selection.metric.summarize(&result.measurements).is_some()) + }) + .collect(); let mut writer = WriterBuilder::new().from_writer(vec![]); - let mut headers: Vec = [ - "command", "metric", "unit", "mean", "stddev", "median", "min", "max", - ] - .iter() - .copied() - .map(str::to_owned) - .collect(); + let mut headers = vec!["command".to_owned()]; + for selection in &metrics { + headers.extend( + ["unit", "mean", "stddev", "median", "min", "max"] + .map(|field| format!("{}_{field}", selection.metric.name())), + ); + } if let Some(result) = results.first() { headers.extend( result @@ -32,36 +43,28 @@ impl Exporter for CsvExporter { writer.write_record(headers)?; for result in results { - let stats = primary - .metric - .summarize(&result.measurements) - .with_context(|| { - format!( - "Metric '{}' is unavailable for '{}'", - primary.metric.name(), - result.get_name() - ) - })?; - let mut fields = vec![ - result.get_name().to_owned(), - primary.metric.name().to_owned(), - unit.symbol.to_owned(), - ]; - // Preserve numeric precision rather than using rounded display values. - fields.extend( - [ - Some(stats.mean), - stats.stddev, - Some(stats.median), - Some(stats.min), - Some(stats.max), - ] - .map(|value| { - value - .map(|value| (value / unit.scale).to_string()) - .unwrap_or_default() - }), - ); + let mut fields = vec![result.get_name().to_owned()]; + for selection in &metrics { + // CSV never chooses units automatically: an omitted suffix means base units. + let unit = selection.csv_unit(); + let stats = selection.metric.summarize(&result.measurements); + fields.push(unit.symbol.to_owned()); + // Preserve numeric precision rather than using rounded display values. + fields.extend( + [ + stats.map(|stats| stats.mean), + stats.and_then(|stats| stats.stddev), + stats.map(|stats| stats.median), + stats.map(|stats| stats.min), + stats.map(|stats| stats.max), + ] + .map(|value| { + value + .map(|value| (value / unit.scale).to_string()) + .unwrap_or_default() + }), + ); + } fields.extend( result .parameters @@ -191,15 +194,18 @@ fn test_csv() { let actual = String::from_utf8( exporter - .serialize(&results, MetricSelection::default()) + .serialize( + &results, + &MetricSelection::parse_list("time_wall_clock,memory_peak_resident").unwrap(), + ) .unwrap(), ) .unwrap(); insta::assert_snapshot!(actual, @r#" - command,metric,unit,mean,stddev,median,min,max,parameter_bar,parameter_foo - command_a,time_wall_clock,s,9,2.6457513110645907,8,7,12,two,one - command_b,time_wall_clock,s,18,1,18,17,19,seven,one + command,time_wall_clock_unit,time_wall_clock_mean,time_wall_clock_stddev,time_wall_clock_median,time_wall_clock_min,time_wall_clock_max,memory_peak_resident_unit,memory_peak_resident_mean,memory_peak_resident_stddev,memory_peak_resident_median,memory_peak_resident_min,memory_peak_resident_max,parameter_bar,parameter_foo + command_a,s,9,2.6457513110645907,8,7,12,B,1024,0,1024,1024,1024,two,one + command_b,s,18,1,18,17,19,B,1024,0,1024,1024,1024,seven,one "#); for (selection, unit, mean) in [ @@ -207,12 +213,11 @@ fn test_csv() { ("memory_peak_resident:KiB", "KiB", "1"), ] { let output = exporter - .serialize(&results, MetricSelection::parse_list(selection).unwrap()[0]) + .serialize(&results, &MetricSelection::parse_list(selection).unwrap()) .unwrap(); let mut reader = csv::Reader::from_reader(output.as_slice()); let row = reader.records().next().unwrap().unwrap(); - assert_eq!(&row[1], "memory_peak_resident"); - assert_eq!(&row[2], unit); - assert_eq!(&row[3], mean); + assert_eq!(&row[1], unit); + assert_eq!(&row[2], mean); } } diff --git a/src/export/json.rs b/src/export/json.rs index 2dad1b0..52cd563 100644 --- a/src/export/json.rs +++ b/src/export/json.rs @@ -65,7 +65,12 @@ pub struct JsonExporter { } impl Exporter for JsonExporter { - fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result> { + fn serialize( + &self, + results: &[BenchmarkResult], + metrics: &[MetricSelection], + ) -> Result> { + let primary = metrics[0]; let mut output = to_vec_pretty(&HyperfineSummary { schema_version: JSON_SCHEMA_VERSION, metadata: &self.metadata, diff --git a/src/export/markup.rs b/src/export/markup.rs index 338cecb..c061fcb 100644 --- a/src/export/markup.rs +++ b/src/export/markup.rs @@ -77,7 +77,12 @@ pub trait MarkupExporter { } impl Exporter for T { - fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result> { + fn serialize( + &self, + results: &[BenchmarkResult], + metrics: &[MetricSelection], + ) -> Result> { + let primary = metrics[0]; let stats: Vec<_> = results .iter() .map(|result| { diff --git a/src/export/mod.rs b/src/export/mod.rs index 02d0e26..06c28c9 100644 --- a/src/export/mod.rs +++ b/src/export/mod.rs @@ -44,8 +44,12 @@ pub enum ExportType { /// Interface for different exporters. trait Exporter { - /// Export the given entries in the serialized form. - fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result>; + /// Export the given entries using a nonempty list of selected metrics, primary first. + fn serialize( + &self, + results: &[BenchmarkResult], + metrics: &[MetricSelection], + ) -> Result>; } pub enum ExportTarget { @@ -61,16 +65,16 @@ struct ExporterWithTarget { /// Handles the management of multiple file exporters. pub struct ExportManager { exporters: Vec, - primary: MetricSelection, + metrics: Vec, } impl ExportManager { /// Build the ExportManager that will export the results specified /// in the given ArgMatches - pub fn from_cli_arguments(matches: &ArgMatches, primary: MetricSelection) -> Result { + pub fn from_cli_arguments(matches: &ArgMatches, metrics: &[MetricSelection]) -> Result { let mut export_manager = Self { exporters: vec![], - primary, + metrics: metrics.to_vec(), }; { let mut add_exporter = |flag, exporttype| -> Result<()> { @@ -120,7 +124,7 @@ impl ExportManager { /// order not to clutter the output of hyperfine with intermediate results). pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> { for e in &self.exporters { - let content = || e.exporter.serialize(results, self.primary); + let content = || e.exporter.serialize(results, &self.metrics); match e.target { ExportTarget::File(ref filename) => { diff --git a/src/export/tests.rs b/src/export/tests.rs index 651c8db..3191c5c 100644 --- a/src/export/tests.rs +++ b/src/export/tests.rs @@ -15,10 +15,10 @@ fn get_output(results: &[BenchmarkResult], unit: Option Result<()> { } } - let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?; + let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?; let mut scheduler = Scheduler::new(&commands, &options, &export_manager); scheduler.run_benchmarks()?;