Export selected metrics side by side in CSV

README.md

@@ -239,8 +239,9 @@ You can also use a preset to select a group of metrics:
 
 ### Exporting results
 
-Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. Non-JSON formats
-contain only the primary metric, which is the first metric selected by `--metrics`.
+Hyperfine can export results to CSV, JSON, Markdown, AsciiDoc, and org-mode. CSV exports all
+selected metrics, while Markdown, AsciiDoc, and org-mode contain only the primary metric,
+which is the first metric selected by `--metrics`.
 
 #### Markdown
 
@@ -276,6 +277,14 @@ uvx --from 'git+https://github.com/sharkdp/hyperfine' \
 |---:|---:|
 
 
+#### CSV
+
+Use `--export-csv <file>` to export one row per command. Each metric selected by `--metrics`
+contributes a group of columns, in selection order: `<metric>_unit`, `<metric>_mean`,
+`<metric>_stddev`, `<metric>_median`, `<metric>_min`, and `<metric>_max`.
+
+CSV uses explicitly selected units, or base units (seconds, bytes, or counts) when omitted.
+
 ### Detailed benchmark flowchart
 
 The following chart explains the execution order of various benchmark runs when using options

doc/hyperfine.1

@@ -232,7 +232,7 @@ of the tool.
 .HP
 \fB\-\-metrics\fR \fIMETRIC[:UNIT],...|default|time|all\fP
 .IP
-Select the metrics to display, in order, with an optional unit for each.
+Select the metrics to measure, in order, with an optional unit for each.
 Default: \fBtime_wall_clock,memory_peak_resident\fR, or \fBtime_wall_clock\fR
 on Windows, where peak resident memory is not yet supported.
 .IP
@@ -266,9 +266,11 @@ Export the primary metric's summary statistics as an AsciiDoc table to the given
 .HP
 \fB\-\-export\-csv\fR \fIFILE\fP
 .IP
-Export the primary metric's summary statistics as CSV to the given \fIFILE\fP,
-identifying the metric and unit explicitly. Use the explicitly selected unit,
-or the base unit (seconds, bytes, or unscaled counts) when omitted.
+Export summary statistics for all selected metrics as CSV to the given \fIFILE\fP,
+one row per command. Each metric contributes columns for unit, mean, stddev, median,
+min, and max, prefixed with the metric name, in selection order. The command column
+comes first and parameter columns come last. Use the explicitly selected units,
+or base units (seconds, bytes, or unscaled counts) when omitted.
 .HP
 \fB\-\-export\-json\fR \fIFILE\fP
 .IP

src/benchmark/scheduler.rs

@@ -69,7 +69,7 @@ fn generate_results(args: &[&'static str]) -> Result<Vec<BenchmarkResult>> {
     assert_eq!(options.executor_kind, ExecutorKind::Mock(None));
 
     let commands = Commands::from_cli_arguments(&cli_arguments)?;
-    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
 
     options.validate_against_command_list(&commands)?;
 

src/cli.rs

@@ -58,7 +58,7 @@ fn build_command() -> Command {
                 .hide_default_value(true)
                 .help("Performance metrics to measure, in order.\n\n\
                        This is either a preset name, or a comma-separated list of metrics with optional units. \
-                       The first metric is used for non-JSON exports, \
+                       CSV exports all selected metrics; markup exports use the first metric. \
                        JSON always includes all collected metrics.\n\n\
                        Presets (use one on its own):\n  \
                          default  Wall-clock time and peak RSS\n  \
@@ -303,9 +303,10 @@ fn build_command() -> Command {
                 .action(ArgAction::Set)
                 .value_name("FILE")
                 .value_hint(ValueHint::FilePath)
-                .help("Export the primary metric summary statistics as CSV to the given FILE. If you need \
+                .help("Export summary statistics for all selected metrics as CSV to the given FILE. If you need \
                        all metrics for each individual run, use the JSON export format. \
-                       Use the explicitly selected unit, or the base unit when unspecified."),
+                       Column names are prefixed with the metric name. \
+                       Use the explicitly selected units, or base units when unspecified."),
         )
         .arg(
             Arg::new("export-json")

src/export/csv.rs

@@ -4,23 +4,34 @@ use super::Exporter;
 use crate::benchmark::benchmark_result::BenchmarkResult;
 use crate::metric::MetricSelection;
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 
 #[derive(Default)]
 pub struct CsvExporter {}
 
 impl Exporter for CsvExporter {
-    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
-        // CSV never chooses units automatically: an omitted suffix means base units.
-        let unit = primary.csv_unit();
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        metrics: &[MetricSelection],
+    ) -> Result<Vec<u8>> {
+        // Omit metrics unavailable for all commands (possible with `--metrics all`).
+        let metrics: Vec<_> = metrics
+            .iter()
+            .filter(|selection| {
+                results
+                    .iter()
+                    .any(|result| selection.metric.summarize(&result.measurements).is_some())
+            })
+            .collect();
         let mut writer = WriterBuilder::new().from_writer(vec![]);
-        let mut headers: Vec<String> = [
-            "command", "metric", "unit", "mean", "stddev", "median", "min", "max",
-        ]
-        .iter()
-        .copied()
-        .map(str::to_owned)
-        .collect();
+        let mut headers = vec!["command".to_owned()];
+        for selection in &metrics {
+            headers.extend(
+                ["unit", "mean", "stddev", "median", "min", "max"]
+                    .map(|field| format!("{}_{field}", selection.metric.name())),
+            );
+        }
         if let Some(result) = results.first() {
             headers.extend(
                 result
@@ -32,36 +43,28 @@ impl Exporter for CsvExporter {
         writer.write_record(headers)?;
 
         for result in results {
-            let stats = primary
-                .metric
-                .summarize(&result.measurements)
-                .with_context(|| {
-                    format!(
-                        "Metric '{}' is unavailable for '{}'",
-                        primary.metric.name(),
-                        result.get_name()
-                    )
-                })?;
-            let mut fields = vec![
-                result.get_name().to_owned(),
-                primary.metric.name().to_owned(),
-                unit.symbol.to_owned(),
-            ];
-            // Preserve numeric precision rather than using rounded display values.
-            fields.extend(
-                [
-                    Some(stats.mean),
-                    stats.stddev,
-                    Some(stats.median),
-                    Some(stats.min),
-                    Some(stats.max),
-                ]
-                .map(|value| {
-                    value
-                        .map(|value| (value / unit.scale).to_string())
-                        .unwrap_or_default()
-                }),
-            );
+            let mut fields = vec![result.get_name().to_owned()];
+            for selection in &metrics {
+                // CSV never chooses units automatically: an omitted suffix means base units.
+                let unit = selection.csv_unit();
+                let stats = selection.metric.summarize(&result.measurements);
+                fields.push(unit.symbol.to_owned());
+                // Preserve numeric precision rather than using rounded display values.
+                fields.extend(
+                    [
+                        stats.map(|stats| stats.mean),
+                        stats.and_then(|stats| stats.stddev),
+                        stats.map(|stats| stats.median),
+                        stats.map(|stats| stats.min),
+                        stats.map(|stats| stats.max),
+                    ]
+                    .map(|value| {
+                        value
+                            .map(|value| (value / unit.scale).to_string())
+                            .unwrap_or_default()
+                    }),
+                );
+            }
             fields.extend(
                 result
                     .parameters
@@ -191,15 +194,18 @@ fn test_csv() {
 
     let actual = String::from_utf8(
         exporter
-            .serialize(&results, MetricSelection::default())
+            .serialize(
+                &results,
+                &MetricSelection::parse_list("time_wall_clock,memory_peak_resident").unwrap(),
+            )
             .unwrap(),
     )
     .unwrap();
 
     insta::assert_snapshot!(actual, @r#"
-    command,metric,unit,mean,stddev,median,min,max,parameter_bar,parameter_foo
-    command_a,time_wall_clock,s,9,2.6457513110645907,8,7,12,two,one
-    command_b,time_wall_clock,s,18,1,18,17,19,seven,one
+    command,time_wall_clock_unit,time_wall_clock_mean,time_wall_clock_stddev,time_wall_clock_median,time_wall_clock_min,time_wall_clock_max,memory_peak_resident_unit,memory_peak_resident_mean,memory_peak_resident_stddev,memory_peak_resident_median,memory_peak_resident_min,memory_peak_resident_max,parameter_bar,parameter_foo
+    command_a,s,9,2.6457513110645907,8,7,12,B,1024,0,1024,1024,1024,two,one
+    command_b,s,18,1,18,17,19,B,1024,0,1024,1024,1024,seven,one
     "#);
 
     for (selection, unit, mean) in [
@@ -207,12 +213,11 @@ fn test_csv() {
         ("memory_peak_resident:KiB", "KiB", "1"),
     ] {
         let output = exporter
-            .serialize(&results, MetricSelection::parse_list(selection).unwrap()[0])
+            .serialize(&results, &MetricSelection::parse_list(selection).unwrap())
             .unwrap();
         let mut reader = csv::Reader::from_reader(output.as_slice());
         let row = reader.records().next().unwrap().unwrap();
-        assert_eq!(&row[1], "memory_peak_resident");
-        assert_eq!(&row[2], unit);
-        assert_eq!(&row[3], mean);
+        assert_eq!(&row[1], unit);
+        assert_eq!(&row[2], mean);
     }
 }

src/export/json.rs

@@ -65,7 +65,12 @@ pub struct JsonExporter {
 }
 
 impl Exporter for JsonExporter {
-    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        metrics: &[MetricSelection],
+    ) -> Result<Vec<u8>> {
+        let primary = metrics[0];
         let mut output = to_vec_pretty(&HyperfineSummary {
             schema_version: JSON_SCHEMA_VERSION,
             metadata: &self.metadata,

src/export/markup.rs

@@ -77,7 +77,12 @@ pub trait MarkupExporter {
 }
 
 impl<T: MarkupExporter> Exporter for T {
-    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>> {
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        metrics: &[MetricSelection],
+    ) -> Result<Vec<u8>> {
+        let primary = metrics[0];
         let stats: Vec<_> = results
             .iter()
             .map(|result| {

src/export/mod.rs

@@ -44,8 +44,12 @@ pub enum ExportType {
 
 /// Interface for different exporters.
 trait Exporter {
-    /// Export the given entries in the serialized form.
-    fn serialize(&self, results: &[BenchmarkResult], primary: MetricSelection) -> Result<Vec<u8>>;
+    /// Export the given entries using a nonempty list of selected metrics, primary first.
+    fn serialize(
+        &self,
+        results: &[BenchmarkResult],
+        metrics: &[MetricSelection],
+    ) -> Result<Vec<u8>>;
 }
 
 pub enum ExportTarget {
@@ -61,16 +65,16 @@ struct ExporterWithTarget {
 /// Handles the management of multiple file exporters.
 pub struct ExportManager {
     exporters: Vec<ExporterWithTarget>,
-    primary: MetricSelection,
+    metrics: Vec<MetricSelection>,
 }
 
 impl ExportManager {
     /// Build the ExportManager that will export the results specified
     /// in the given ArgMatches
-    pub fn from_cli_arguments(matches: &ArgMatches, primary: MetricSelection) -> Result<Self> {
+    pub fn from_cli_arguments(matches: &ArgMatches, metrics: &[MetricSelection]) -> Result<Self> {
         let mut export_manager = Self {
             exporters: vec![],
-            primary,
+            metrics: metrics.to_vec(),
         };
         {
             let mut add_exporter = |flag, exporttype| -> Result<()> {
@@ -120,7 +124,7 @@ impl ExportManager {
     /// order not to clutter the output of hyperfine with intermediate results).
     pub fn write_results(&self, results: &[BenchmarkResult], intermediate: bool) -> Result<()> {
         for e in &self.exporters {
-            let content = || e.exporter.serialize(results, self.primary);
+            let content = || e.exporter.serialize(results, &self.metrics);
 
             match e.target {
                 ExportTarget::File(ref filename) => {

src/export/tests.rs

@@ -15,10 +15,10 @@ fn get_output<E: Exporter + Default>(results: &[BenchmarkResult], unit: Option<U
         exporter
             .serialize(
                 results,
-                MetricSelection {
+                &[MetricSelection {
                     unit,
                     ..MetricSelection::default()
-                },
+                }],
             )
             .unwrap(),
     )

src/main.rs

@@ -62,7 +62,7 @@ fn run() -> Result<()> {
         }
     }
 
-    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, options.metrics[0])?;
+    let export_manager = ExportManager::from_cli_arguments(&cli_arguments, &options.metrics)?;
 
     let mut scheduler = Scheduler::new(&commands, &options, &export_manager);
     scheduler.run_benchmarks()?;