Alternative performance metrics in the terminal (#986)

scripts/README.md

@@ -7,9 +7,7 @@ hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
 uv run plot_whisker.py sleep.json
 ```
 
-All plotting scripts accept `--metric` to select a metric. Hardware counters
-(`cpu_cycles`, `instructions`, `cache_references`, `cache_misses`, and `branch_misses`)
-are plotted as unscaled counts when available in the JSON export. For example:
+All plotting and analysis scripts use the JSON export's `primary_metric` by default.
 
 ```bash
 uv run plot_histogram.py benchmark.json --metric instructions

scripts/advanced_statistics.py

@@ -7,74 +7,56 @@
 # ///
 
 import argparse
-import json
-from enum import Enum
 
 import numpy as np
 
-
-class Unit(Enum):
-    SECOND = 1
-    MILLISECOND = 2
-
-    def factor(self):
-        match self:
-            case Unit.SECOND:
-                return 1
-            case Unit.MILLISECOND:
-                return 1e3
-
-    def __str__(self):
-        match self:
-            case Unit.SECOND:
-                return "s"
-            case Unit.MILLISECOND:
-                return "ms"
-
+from plot_utils import METRICS, add_metric_argument, load_results
 
 parser = argparse.ArgumentParser()
 parser.add_argument("file", help="JSON file with benchmark results")
+add_metric_argument(parser)
 parser.add_argument(
     "--time-unit",
-    help="The unit of time.",
-    default="second",
-    action="store",
+    help="Display unit for time metrics (default: second).",
     choices=["second", "millisecond"],
-    dest="unit",
 )
 args = parser.parse_args()
 
-unit = Unit.MILLISECOND if args.unit == "millisecond" else Unit.SECOND
-unit_str = str(unit)
+metric, [results] = load_results(parser, [args.file], args.metric)
+description = METRICS[metric]
+unit, scale = description.unit, description.scale
+if args.time_unit is not None:
+    if not metric.startswith("time_"):
+        parser.error("--time-unit can only be used with a time metric")
+    if args.time_unit == "millisecond":
+        unit, scale = "ms", 0.001
 
-with open(args.file) as f:
-    results = json.load(f)["results"]
 
+def format_value(value):
+    suffix = f" {unit}" if unit else ""
+    return f"{value:.3f}{suffix}"
+
+
+print(f"Metric: {description.label}\n")
 for result in results:
-    summary = result["summary"]["time_wall_clock"]
-    ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]]
+    summary = result["summary"][metric]
+    values = [m[metric]["value"] / scale for m in result["measurements"]]
 
-    p05 = np.percentile(ts, 5)
-    p25 = np.percentile(ts, 25)
-    p75 = np.percentile(ts, 75)
-    p95 = np.percentile(ts, 95)
+    p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95])
 
     iqr = p75 - p25
 
     print(f"Command '{result.get('name', result['command'])}'")
     print(f"  runs:   {summary['count']:8d}")
-    print(f"  mean:   {summary['mean'] * unit.factor():8.3f} {unit_str}")
-    if summary["stddev"] is None:
-        print(f"  stddev: {'N/A':>8}")
-    else:
-        print(f"  stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}")
-    print(f"  median: {summary['median'] * unit.factor():8.3f} {unit_str}")
-    print(f"  min:    {summary['min'] * unit.factor():8.3f} {unit_str}")
-    print(f"  max:    {summary['max'] * unit.factor():8.3f} {unit_str}")
+    for statistic in ["mean", "stddev", "median", "min", "max"]:
+        value = summary[statistic]
+        formatted = "N/A" if value is None else format_value(value / scale)
+        print(f"  {statistic + ':':<8}{formatted:>8}")
     print()
     print("  percentiles:")
-    print(f"     P_05 .. P_95:    {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}")
+    print(f"     P_05 .. P_95:    {format_value(p05)} .. {format_value(p95)}")
     print(
-        f"     P_25 .. P_75:    {p25:.3f} {unit_str} .. {p75:.3f} {unit_str}  (IQR = {iqr:.3f} {unit_str})"
+        f"     P_25 .. P_75:    {format_value(p25)} .. {format_value(p75)}"
+        f"  (IQR = {format_value(iqr)})"
     )
     print()

scripts/plot_benchmark_comparison.py

@@ -14,13 +14,12 @@ Note all the input files must contain results for all commands.
 """
 
 import argparse
-import json
 import pathlib
 
 import matplotlib.pyplot as plt
 import numpy as np
 
-from plot_utils import METRICS, add_metric_argument, validate_metric
+from plot_utils import METRICS, add_metric_argument, load_results
 
 parser = argparse.ArgumentParser(description=__doc__)
 add_metric_argument(parser)
@@ -34,7 +33,8 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename")
 
 args = parser.parse_args()
-metric_label, metric_scale = METRICS[args.metric]
+metric, datasets = load_results(parser, args.files, args.metric)
+metric_info = METRICS[metric]
 
 commands = None
 data = []
@@ -45,10 +45,7 @@ if args.benchmark_names:
         "Number of benchmark names must match the number of input files."
     )
 
-for i, filename in enumerate(args.files):
-    with open(filename) as f:
-        results = json.load(f)["results"]
-    validate_metric(parser, results, args.metric)
+for i, (filename, results) in enumerate(zip(args.files, datasets)):
     benchmark_commands = [b.get("name", b["command"]) for b in results]
     if commands is None:
         commands = benchmark_commands
@@ -56,7 +53,7 @@ for i, filename in enumerate(args.files):
         assert commands == benchmark_commands, (
             f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
         )
-    data.append([b["summary"][args.metric]["mean"] / metric_scale for b in results])
+    data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
     if args.benchmark_names:
         inputs.append(args.benchmark_names[i])
     else:
@@ -79,7 +76,7 @@ ax.grid(visible=True, axis="y")
 if args.title:
     plt.title(args.title)
 plt.xlabel("Benchmark")
-plt.ylabel(metric_label)
+plt.ylabel(metric_info.axis_label)
 plt.legend(title="Command")
 
 if args.output:

scripts/plot_histogram.py

@@ -11,12 +11,11 @@
 """This program shows `hyperfine` benchmark results as a histogram."""
 
 import argparse
-import json
 
 import matplotlib.pyplot as plt
 import numpy as np
 
-from plot_utils import METRICS, add_metric_argument, validate_metric
+from plot_utils import METRICS, add_metric_argument, load_results
 
 parser = argparse.ArgumentParser(description=__doc__)
 add_metric_argument(parser)
@@ -70,18 +69,15 @@ parser.add_argument(
 )
 
 args = parser.parse_args()
-metric_label, metric_scale = METRICS[args.metric]
-
-with open(args.file) as f:
-    results = json.load(f)["results"]
-validate_metric(parser, results, args.metric)
+metric, [results] = load_results(parser, [args.file], args.metric)
+metric_info = METRICS[metric]
 
 if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b.get("name", b["command"]) for b in results]
 all_values = [
-    [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results
+    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
 ]
 
 value_min = (
@@ -113,7 +109,7 @@ plt.legend(
     prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
 )
 
-plt.xlabel(metric_label)
+plt.xlabel(metric_info.axis_label)
 if args.title:
     plt.title(args.title)
 

scripts/plot_parametrized.py

@@ -11,12 +11,11 @@
 errorbar plot."""
 
 import argparse
-import json
 import sys
 
 import matplotlib.pyplot as plt
 
-from plot_utils import METRICS, add_metric_argument, validate_metric
+from plot_utils import METRICS, add_metric_argument, load_results
 
 parser = argparse.ArgumentParser(description=__doc__)
 add_metric_argument(parser)
@@ -43,7 +42,8 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
 args = parser.parse_args()
-metric_label, metric_scale = METRICS[args.metric]
+metric, datasets = load_results(parser, args.file, args.metric)
+metric_info = METRICS[metric]
 if args.parameter_name is not None:
     sys.stderr.write(
         "warning: --parameter-name is deprecated; names are inferred from "
@@ -84,11 +84,7 @@ def unique_parameter(benchmark):
 
 parameter_name = None
 
-for filename in args.file:
-    with open(filename) as f:
-        results = json.load(f)["results"]
-    validate_metric(parser, results, args.metric)
-
+for results in datasets:
     (this_parameter_name, parameter_values) = extract_parameters(results)
     if parameter_name is not None and this_parameter_name != parameter_name:
         die(
@@ -96,17 +92,17 @@ for filename in args.file:
         )
     parameter_name = this_parameter_name
 
-    summaries = [b["summary"][args.metric] for b in results]
-    means = [s["mean"] / metric_scale for s in summaries]
+    summaries = [b["summary"][metric] for b in results]
+    means = [s["mean"] / metric_info.scale for s in summaries]
     stddevs = [
-        s["stddev"] / metric_scale if s["stddev"] is not None else float("nan")
+        s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan")
         for s in summaries
     ]
 
     plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
 
 plt.xlabel(parameter_name)
-plt.ylabel(metric_label)
+plt.ylabel(metric_info.axis_label)
 
 if args.log_y:
     plt.yscale("log")

scripts/plot_progression.py

@@ -14,12 +14,11 @@ thermal throttling and similar effects.
 """
 
 import argparse
-import json
 
 import matplotlib.pyplot as plt
 import numpy as np
 
-from plot_utils import METRICS, add_metric_argument, validate_metric
+from plot_utils import METRICS, add_metric_argument, load_results
 
 
 def moving_average(values, num_runs):
@@ -50,15 +49,12 @@ parser.add_argument(
 
 
 args = parser.parse_args()
-metric_label, metric_scale = METRICS[args.metric]
-
-with open(args.file) as f:
-    results = json.load(f)["results"]
-validate_metric(parser, results, args.metric)
+metric, [results] = load_results(parser, [args.file], args.metric)
+metric_info = METRICS[metric]
 
 for result in results:
     label = result.get("name", result["command"])
-    values = [m[args.metric]["value"] / metric_scale for m in result["measurements"]]
+    values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]]
     num = len(values)
     nums = range(num)
 
@@ -86,7 +82,7 @@ for result in results:
         legend.append("moving average")
 plt.legend(legend)
 
-plt.ylabel(metric_label)
+plt.ylabel(metric_info.axis_label)
 
 if args.output:
     plt.savefig(args.output)

scripts/plot_utils.py

@@ -1,15 +1,30 @@
-"""Shared metric options for the plotting scripts."""
+"""Shared metric selection and units for analysis and plotting scripts."""
+
+import json
+from typing import NamedTuple
+
+
+class Metric(NamedTuple):
+    label: str
+    unit: str = ""
+    scale: float = 1
+
+    @property
+    def axis_label(self):
+        return f"{self.label} [{self.unit}]" if self.unit else self.label
+
 
 METRICS = {
-    "time_wall_clock": ("Time [s]", 1),
-    "time_user": ("User CPU time [s]", 1),
-    "time_system": ("System CPU time [s]", 1),
-    "memory_peak_resident": ("Peak resident memory [MiB]", 1024**2),
-    "cpu_cycles": ("CPU cycles", 1),
-    "instructions": ("Completed CPU instructions", 1),
-    "cache_references": ("Cache references", 1),
-    "cache_misses": ("Cache misses", 1),
-    "branch_misses": ("Mispredicted branches", 1),
+    "time_wall_clock": Metric("Wall-clock time", "s"),
+    "time_cpu": Metric("Total CPU time", "s"),
+    "time_user": Metric("User CPU time", "s"),
+    "time_system": Metric("System CPU time", "s"),
+    "memory_peak_resident": Metric("Peak resident memory", "MiB", 1024**2),
+    "cpu_cycles": Metric("CPU cycles"),
+    "instructions": Metric("Completed CPU instructions"),
+    "cache_references": Metric("Cache references"),
+    "cache_misses": Metric("Cache misses"),
+    "branch_misses": Metric("Mispredicted branches"),
 }
 
 
@@ -17,8 +32,8 @@ def add_metric_argument(parser):
     parser.add_argument(
         "--metric",
         choices=METRICS,
-        default="time_wall_clock",
-        help="Metric to plot (default: time_wall_clock). "
+        help="Metric to analyze (default: primary_metric from the JSON export, "
+        "or time_wall_clock if unspecified). "
         "Times are in seconds; memory is in MiB; hardware counters are unscaled counts.",
     )
 
@@ -30,3 +45,30 @@ def validate_metric(parser, results, metric):
                 f"metric {metric!r} is unavailable for "
                 f"{result.get('name', result['command'])!r}"
             )
+
+
+def load_results(parser, filenames, metric=None):
+    exports = []
+    for filename in filenames:
+        with open(filename, encoding="utf-8") as f:
+            export = json.load(f)
+        if not export["results"]:
+            parser.error(f"no benchmark results in {filename}")
+        exports.append(export)
+
+    if metric is None:
+        primary_metrics = {
+            export.get("primary_metric", "time_wall_clock") for export in exports
+        }
+        if len(primary_metrics) != 1:
+            parser.error(
+                "input files use different primary metrics; select one with --metric"
+            )
+        metric = primary_metrics.pop()
+    if metric not in METRICS:
+        parser.error(f"unknown metric {metric!r}; select one with --metric")
+
+    results = [export["results"] for export in exports]
+    for benchmarks in results:
+        validate_metric(parser, benchmarks, metric)
+    return metric, results

scripts/plot_whisker.py

@@ -16,11 +16,10 @@ Quoting from the matplotlib documentation:
 """
 
 import argparse
-import json
 
 import matplotlib.pyplot as plt
 
-from plot_utils import METRICS, add_metric_argument, validate_metric
+from plot_utils import METRICS, add_metric_argument, load_results
 
 parser = argparse.ArgumentParser(description=__doc__)
 add_metric_argument(parser)
@@ -33,22 +32,19 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
 args = parser.parse_args()
-metric_label, metric_scale = METRICS[args.metric]
-
-with open(args.file, encoding="utf-8") as f:
-    results = json.load(f)["results"]
-validate_metric(parser, results, args.metric)
+metric, [results] = load_results(parser, [args.file], args.metric)
+metric_info = METRICS[metric]
 
 if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b.get("name", b["command"]) for b in results]
 values = [
-    [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results
+    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
 ]
 
 if args.sort_by == "median":
-    medians = [b["summary"][args.metric]["median"] for b in results]
+    medians = [b["summary"][metric]["median"] for b in results]
     indices = sorted(range(len(labels)), key=lambda k: medians[k])
     labels = [labels[i] for i in indices]
     values = [values[i] for i in indices]
@@ -64,7 +60,7 @@ for patch, color in zip(boxplot["boxes"], colors):
 if args.title:
     plt.title(args.title)
 plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
-plt.ylabel(metric_label)
+plt.ylabel(metric_info.axis_label)
 plt.ylim(0, None)
 plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
 if args.output:

scripts/welch_ttest.py

@@ -7,32 +7,37 @@
 # ///
 
 """This script performs Welch's t-test on a JSON export file with two
-benchmark results to test for a difference in their population mean runtimes."""
+benchmark results to test for a difference in their population means."""
 
 import argparse
-import json
-import sys
+import math
 
 from scipy import stats
 
+from plot_utils import METRICS, add_metric_argument, load_results
+
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument("file", help="JSON file with two benchmark results")
+add_metric_argument(parser)
 args = parser.parse_args()
 
-with open(args.file) as f:
-    results = json.load(f)["results"]
+metric, [results] = load_results(parser, [args.file], args.metric)
 
 if len(results) != 2:
-    print("The input file has to contain exactly two benchmarks")
-    sys.exit(1)
+    parser.error("the input file must contain exactly two benchmarks")
 
 a, b = (x.get("name", x["command"]) for x in results[:2])
-X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2])
+X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2])
+if min(len(X), len(Y)) < 2:
+    parser.error("Welch's t-test requires at least two measurements per benchmark")
 
+print(f"Metric: {METRICS[metric].label}")
 print(f"Command 1: {a}")
 print(f"Command 2: {b}\n")
 
 t, p = stats.ttest_ind(X, Y, equal_var=False)
+if not math.isfinite(p):
+    parser.error("Welch's t-test is undefined for these measurements")
 th = 0.05
 dispose = p < th
 print(f"t = {t:.3}, p = {p:.3}")
@@ -40,9 +45,9 @@ print()
 
 if dispose:
     print(
-        f"A statistically significant difference in mean runtime was detected (p < {th})."
+        f"A statistically significant difference in mean value was detected (p < {th})."
     )
 else:
     print(
-        f"No statistically significant difference in mean runtime was detected (p >= {th})."
+        f"No statistically significant difference in mean value was detected (p >= {th})."
     )