Support all exported metrics in plotting scripts (#981)

scripts/plot_benchmark_comparison.py

@@ -20,7 +20,10 @@ import pathlib
 import matplotlib.pyplot as plt
 import numpy as np
 
+from plot_utils import METRICS, add_metric_argument, validate_metric
+
 parser = argparse.ArgumentParser(description=__doc__)
+add_metric_argument(parser)
 parser.add_argument(
     "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
 )
@@ -31,27 +34,29 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename")
 
 args = parser.parse_args()
+metric_label, metric_scale = METRICS[args.metric]
 
 commands = None
 data = []
 inputs = []
 
 if args.benchmark_names:
-    assert len(args.files) == len(
-        args.benchmark_names
-    ), "Number of benchmark names must match the number of input files."
+    assert len(args.files) == len(args.benchmark_names), (
+        "Number of benchmark names must match the number of input files."
+    )
 
 for i, filename in enumerate(args.files):
     with open(filename) as f:
         results = json.load(f)["results"]
+    validate_metric(parser, results, args.metric)
     benchmark_commands = [b.get("name", b["command"]) for b in results]
     if commands is None:
         commands = benchmark_commands
     else:
-        assert (
-            commands == benchmark_commands
-        ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
-    data.append([b["summary"]["time_wall_clock"]["mean"] for b in results])
+        assert commands == benchmark_commands, (
+            f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
+        )
+    data.append([b["summary"][args.metric]["mean"] / metric_scale for b in results])
     if args.benchmark_names:
         inputs.append(args.benchmark_names[i])
     else:
@@ -74,7 +79,7 @@ ax.grid(visible=True, axis="y")
 if args.title:
     plt.title(args.title)
 plt.xlabel("Benchmark")
-plt.ylabel("Time [s]")
+plt.ylabel(metric_label)
 plt.legend(title="Command")
 
 if args.output:

scripts/plot_histogram.py

@@ -16,7 +16,10 @@ import json
 import matplotlib.pyplot as plt
 import numpy as np
 
+from plot_utils import METRICS, add_metric_argument, validate_metric
+
 parser = argparse.ArgumentParser(description=__doc__)
+add_metric_argument(parser)
 parser.add_argument("file", help="JSON file with benchmark results")
 parser.add_argument("--title", help="Plot title")
 parser.add_argument(
@@ -47,10 +50,18 @@ parser.add_argument(
 )
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
 parser.add_argument(
-    "--t-min", metavar="T", help="Minimum time to be displayed (seconds)"
+    "--min",
+    "--t-min",
+    dest="value_min",
+    type=float,
+    help="Minimum value to display (seconds, or MiB for memory)",
 )
 parser.add_argument(
-    "--t-max", metavar="T", help="Maximum time to be displayed (seconds)"
+    "--max",
+    "--t-max",
+    dest="value_max",
+    type=float,
+    help="Maximum value to display (seconds, or MiB for memory)",
 )
 parser.add_argument(
     "--log-count",
@@ -59,31 +70,41 @@ parser.add_argument(
 )
 
 args = parser.parse_args()
+metric_label, metric_scale = METRICS[args.metric]
 
 with open(args.file) as f:
     results = json.load(f)["results"]
+validate_metric(parser, results, args.metric)
 
 if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b.get("name", b["command"]) for b in results]
-all_times = [
-    [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results
+all_values = [
+    [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results
 ]
 
-t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
-t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))
+value_min = (
+    args.value_min
+    if args.value_min is not None
+    else np.min(list(map(np.min, all_values)))
+)
+value_max = (
+    args.value_max
+    if args.value_max is not None
+    else np.max(list(map(np.max, all_values)))
+)
 
 bins = int(args.bins) if args.bins else "auto"
 histtype = args.type if args.type else "bar"
 
 plt.figure(figsize=(10, 5))
 plt.hist(
-    all_times,
+    all_values,
     label=labels,
     bins=bins,
     histtype=histtype,
-    range=(t_min, t_max),
+    range=(value_min, value_max),
 )
 plt.legend(
     loc=args.legend_location,
@@ -92,7 +113,7 @@ plt.legend(
     prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
 )
 
-plt.xlabel("Time [s]")
+plt.xlabel(metric_label)
 if args.title:
     plt.title(args.title)
 

scripts/plot_parametrized.py

@@ -16,7 +16,10 @@ import sys
 
 import matplotlib.pyplot as plt
 
+from plot_utils import METRICS, add_metric_argument, validate_metric
+
 parser = argparse.ArgumentParser(description=__doc__)
+add_metric_argument(parser)
 parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
 parser.add_argument(
     "--parameter-name",
@@ -28,7 +31,11 @@ parser.add_argument(
     "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
 )
 parser.add_argument(
-    "--log-time", help="Use a logarithmic time axis", action="store_true"
+    "--log-y",
+    "--log-time",
+    dest="log_y",
+    help="Use a logarithmic metric axis",
+    action="store_true",
 )
 parser.add_argument(
     "--titles", help="Comma-separated list of titles for the plot legend"
@@ -36,6 +43,7 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
 args = parser.parse_args()
+metric_label, metric_scale = METRICS[args.metric]
 if args.parameter_name is not None:
     sys.stderr.write(
         "warning: --parameter-name is deprecated; names are inferred from "
@@ -79,6 +87,7 @@ parameter_name = None
 for filename in args.file:
     with open(filename) as f:
         results = json.load(f)["results"]
+    validate_metric(parser, results, args.metric)
 
     (this_parameter_name, parameter_values) = extract_parameters(results)
     if parameter_name is not None and this_parameter_name != parameter_name:
@@ -87,18 +96,19 @@ for filename in args.file:
         )
     parameter_name = this_parameter_name
 
-    summaries = [b["summary"]["time_wall_clock"] for b in results]
-    times_mean = [s["mean"] for s in summaries]
-    times_stddev = [
-        s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries
+    summaries = [b["summary"][args.metric] for b in results]
+    means = [s["mean"] / metric_scale for s in summaries]
+    stddevs = [
+        s["stddev"] / metric_scale if s["stddev"] is not None else float("nan")
+        for s in summaries
     ]
 
-    plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)
+    plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
 
 plt.xlabel(parameter_name)
-plt.ylabel("Time [s]")
+plt.ylabel(metric_label)
 
-if args.log_time:
+if args.log_y:
     plt.yscale("log")
 else:
     plt.ylim(0, None)

scripts/plot_progression.py

@@ -19,16 +19,19 @@ import json
 import matplotlib.pyplot as plt
 import numpy as np
 
+from plot_utils import METRICS, add_metric_argument, validate_metric
 
-def moving_average(times, num_runs):
-    times_padded = np.pad(
-        times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
+
+def moving_average(values, num_runs):
+    values_padded = np.pad(
+        values, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge"
     )
     kernel = np.ones(num_runs) / num_runs
-    return np.convolve(times_padded, kernel, mode="valid")
+    return np.convolve(values_padded, kernel, mode="valid")
 
 
 parser = argparse.ArgumentParser(description=__doc__)
+add_metric_argument(parser)
 parser.add_argument("file", help="JSON file with benchmark results")
 parser.add_argument("--title", help="Plot Title")
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
@@ -47,17 +50,19 @@ parser.add_argument(
 
 
 args = parser.parse_args()
+metric_label, metric_scale = METRICS[args.metric]
 
 with open(args.file) as f:
     results = json.load(f)["results"]
+validate_metric(parser, results, args.metric)
 
 for result in results:
     label = result.get("name", result["command"])
-    times = [m["time_wall_clock"]["value"] for m in result["measurements"]]
-    num = len(times)
+    values = [m[args.metric]["value"] / metric_scale for m in result["measurements"]]
+    num = len(values)
     nums = range(num)
 
-    plt.scatter(x=nums, y=times, marker=".")
+    plt.scatter(x=nums, y=values, marker=".")
     plt.ylim([0, None])
     plt.xlim([-1, num])
 
@@ -68,7 +73,7 @@ for result in results:
             else args.moving_average_width
         )
 
-        average = moving_average(times, moving_average_width)
+        average = moving_average(values, moving_average_width)
         plt.plot(nums, average, "-")
 
 if args.title:
@@ -81,7 +86,7 @@ for result in results:
         legend.append("moving average")
 plt.legend(legend)
 
-plt.ylabel("Time [s]")
+plt.ylabel(metric_label)
 
 if args.output:
     plt.savefig(args.output)

scripts/plot_utils.py

@@ -0,0 +1,27 @@
+"""Shared metric options for the plotting scripts."""
+
+METRICS = {
+    "time_wall_clock": ("Time [s]", 1),
+    "time_user": ("User CPU time [s]", 1),
+    "time_system": ("System CPU time [s]", 1),
+    "memory_peak_resident": ("Peak resident memory [MiB]", 1024**2),
+}
+
+
+def add_metric_argument(parser):
+    parser.add_argument(
+        "--metric",
+        choices=METRICS,
+        default="time_wall_clock",
+        help="Metric to plot (default: time_wall_clock). "
+        "Times are in seconds; memory is in MiB.",
+    )
+
+
+def validate_metric(parser, results, metric):
+    for result in results:
+        if metric not in result["summary"]:
+            parser.error(
+                f"metric {metric!r} is unavailable for "
+                f"{result.get('name', result['command'])!r}"
+            )

scripts/plot_whisker.py

@@ -20,7 +20,10 @@ import json
 
 import matplotlib.pyplot as plt
 
+from plot_utils import METRICS, add_metric_argument, validate_metric
+
 parser = argparse.ArgumentParser(description=__doc__)
+add_metric_argument(parser)
 parser.add_argument("file", help="JSON file with benchmark results")
 parser.add_argument("--title", help="Plot Title")
 parser.add_argument("--sort-by", choices=["median"], help="Sort method")
@@ -30,26 +33,30 @@ parser.add_argument(
 parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
 args = parser.parse_args()
+metric_label, metric_scale = METRICS[args.metric]
 
 with open(args.file, encoding="utf-8") as f:
     results = json.load(f)["results"]
+validate_metric(parser, results, args.metric)
 
 if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b.get("name", b["command"]) for b in results]
-times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results]
+values = [
+    [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results
+]
 
 if args.sort_by == "median":
-    medians = [b["summary"]["time_wall_clock"]["median"] for b in results]
+    medians = [b["summary"][args.metric]["median"] for b in results]
     indices = sorted(range(len(labels)), key=lambda k: medians[k])
     labels = [labels[i] for i in indices]
-    times = [times[i] for i in indices]
+    values = [values[i] for i in indices]
 
 plt.figure(figsize=(10, 6), constrained_layout=True)
-boxplot = plt.boxplot(times, vert=True, patch_artist=True)
+boxplot = plt.boxplot(values, vert=True, patch_artist=True)
 cmap = plt.get_cmap("rainbow")
-colors = [cmap(val / len(times)) for val in range(len(times))]
+colors = [cmap(val / len(values)) for val in range(len(values))]
 
 for patch, color in zip(boxplot["boxes"], colors):
     patch.set_facecolor(color)
@@ -57,7 +64,7 @@ for patch, color in zip(boxplot["boxes"], colors):
 if args.title:
     plt.title(args.title)
 plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
-plt.ylabel("Time [s]")
+plt.ylabel(metric_label)
 plt.ylim(0, None)
 plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
 if args.output: