From ef86c830b509fd734b9f13d2f30cbe3e8c834276 Mon Sep 17 00:00:00 2001 From: David Peter Date: Mon, 5 Oct 2026 22:48:06 +0200 Subject: [PATCH] Support all exported metrics in plotting scripts (#981) --- scripts/plot_benchmark_comparison.py | 21 +++++++++------ scripts/plot_histogram.py | 39 +++++++++++++++++++++------- scripts/plot_parametrized.py | 26 +++++++++++++------ scripts/plot_progression.py | 23 +++++++++------- scripts/plot_utils.py | 27 +++++++++++++++++++ scripts/plot_whisker.py | 19 +++++++++----- 6 files changed, 115 insertions(+), 40 deletions(-) create mode 100644 scripts/plot_utils.py diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index 754ee17..44f66d9 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -20,7 +20,10 @@ import pathlib import matplotlib.pyplot as plt import numpy as np +from plot_utils import METRICS, add_metric_argument, validate_metric + parser = argparse.ArgumentParser(description=__doc__) +add_metric_argument(parser) parser.add_argument( "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results" ) @@ -31,27 +34,29 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename") args = parser.parse_args() +metric_label, metric_scale = METRICS[args.metric] commands = None data = [] inputs = [] if args.benchmark_names: - assert len(args.files) == len( - args.benchmark_names - ), "Number of benchmark names must match the number of input files." + assert len(args.files) == len(args.benchmark_names), ( + "Number of benchmark names must match the number of input files." + ) for i, filename in enumerate(args.files): with open(filename) as f: results = json.load(f)["results"] + validate_metric(parser, results, args.metric) benchmark_commands = [b.get("name", b["command"]) for b in results] if commands is None: commands = benchmark_commands else: - assert ( - commands == benchmark_commands - ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" - data.append([b["summary"]["time_wall_clock"]["mean"] for b in results]) + assert commands == benchmark_commands, ( + f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" + ) + data.append([b["summary"][args.metric]["mean"] / metric_scale for b in results]) if args.benchmark_names: inputs.append(args.benchmark_names[i]) else: @@ -74,7 +79,7 @@ ax.grid(visible=True, axis="y") if args.title: plt.title(args.title) plt.xlabel("Benchmark") -plt.ylabel("Time [s]") +plt.ylabel(metric_label) plt.legend(title="Command") if args.output: diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index 13d51c4..5d7be2a 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -16,7 +16,10 @@ import json import matplotlib.pyplot as plt import numpy as np +from plot_utils import METRICS, add_metric_argument, validate_metric + parser = argparse.ArgumentParser(description=__doc__) +add_metric_argument(parser) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot title") parser.add_argument( @@ -47,10 +50,18 @@ parser.add_argument( ) parser.add_argument("-o", "--output", help="Save image to the given filename.") parser.add_argument( - "--t-min", metavar="T", help="Minimum time to be displayed (seconds)" + "--min", + "--t-min", + dest="value_min", + type=float, + help="Minimum value to display (seconds, or MiB for memory)", ) parser.add_argument( - "--t-max", metavar="T", help="Maximum time to be displayed (seconds)" + "--max", + "--t-max", + dest="value_max", + type=float, + help="Maximum value to display (seconds, or MiB for memory)", ) parser.add_argument( "--log-count", @@ -59,31 +70,41 @@ parser.add_argument( ) args = parser.parse_args() +metric_label, metric_scale = METRICS[args.metric] with open(args.file) as f: results = json.load(f)["results"] +validate_metric(parser, results, args.metric) if args.labels: labels = args.labels.split(",") else: labels = [b.get("name", b["command"]) for b in results] -all_times = [ - [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results +all_values = [ + [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results ] -t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times))) -t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times))) +value_min = ( + args.value_min + if args.value_min is not None + else np.min(list(map(np.min, all_values))) +) +value_max = ( + args.value_max + if args.value_max is not None + else np.max(list(map(np.max, all_values))) +) bins = int(args.bins) if args.bins else "auto" histtype = args.type if args.type else "bar" plt.figure(figsize=(10, 5)) plt.hist( - all_times, + all_values, label=labels, bins=bins, histtype=histtype, - range=(t_min, t_max), + range=(value_min, value_max), ) plt.legend( loc=args.legend_location, @@ -92,7 +113,7 @@ plt.legend( prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]}, ) -plt.xlabel("Time [s]") +plt.xlabel(metric_label) if args.title: plt.title(args.title) diff --git a/scripts/plot_parametrized.py b/scripts/plot_parametrized.py index 0db1cb6..4973347 100755 --- a/scripts/plot_parametrized.py +++ b/scripts/plot_parametrized.py @@ -16,7 +16,10 @@ import sys import matplotlib.pyplot as plt +from plot_utils import METRICS, add_metric_argument, validate_metric + parser = argparse.ArgumentParser(description=__doc__) +add_metric_argument(parser) parser.add_argument("file", help="JSON file with benchmark results", nargs="+") parser.add_argument( "--parameter-name", @@ -28,7 +31,11 @@ parser.add_argument( "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true" ) parser.add_argument( - "--log-time", help="Use a logarithmic time axis", action="store_true" + "--log-y", + "--log-time", + dest="log_y", + help="Use a logarithmic metric axis", + action="store_true", ) parser.add_argument( "--titles", help="Comma-separated list of titles for the plot legend" @@ -36,6 +43,7 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() +metric_label, metric_scale = METRICS[args.metric] if args.parameter_name is not None: sys.stderr.write( "warning: --parameter-name is deprecated; names are inferred from " @@ -79,6 +87,7 @@ parameter_name = None for filename in args.file: with open(filename) as f: results = json.load(f)["results"] + validate_metric(parser, results, args.metric) (this_parameter_name, parameter_values) = extract_parameters(results) if parameter_name is not None and this_parameter_name != parameter_name: @@ -87,18 +96,19 @@ for filename in args.file: ) parameter_name = this_parameter_name - summaries = [b["summary"]["time_wall_clock"] for b in results] - times_mean = [s["mean"] for s in summaries] - times_stddev = [ - s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries + summaries = [b["summary"][args.metric] for b in results] + means = [s["mean"] / metric_scale for s in summaries] + stddevs = [ + s["stddev"] / metric_scale if s["stddev"] is not None else float("nan") + for s in summaries ] - plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2) + plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2) plt.xlabel(parameter_name) -plt.ylabel("Time [s]") +plt.ylabel(metric_label) -if args.log_time: +if args.log_y: plt.yscale("log") else: plt.ylim(0, None) diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index 9cd7808..e0edf67 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -19,16 +19,19 @@ import json import matplotlib.pyplot as plt import numpy as np +from plot_utils import METRICS, add_metric_argument, validate_metric -def moving_average(times, num_runs): - times_padded = np.pad( - times, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge" + +def moving_average(values, num_runs): + values_padded = np.pad( + values, (num_runs // 2, num_runs - 1 - num_runs // 2), mode="edge" ) kernel = np.ones(num_runs) / num_runs - return np.convolve(times_padded, kernel, mode="valid") + return np.convolve(values_padded, kernel, mode="valid") parser = argparse.ArgumentParser(description=__doc__) +add_metric_argument(parser) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot Title") parser.add_argument("-o", "--output", help="Save image to the given filename.") @@ -47,17 +50,19 @@ parser.add_argument( args = parser.parse_args() +metric_label, metric_scale = METRICS[args.metric] with open(args.file) as f: results = json.load(f)["results"] +validate_metric(parser, results, args.metric) for result in results: label = result.get("name", result["command"]) - times = [m["time_wall_clock"]["value"] for m in result["measurements"]] - num = len(times) + values = [m[args.metric]["value"] / metric_scale for m in result["measurements"]] + num = len(values) nums = range(num) - plt.scatter(x=nums, y=times, marker=".") + plt.scatter(x=nums, y=values, marker=".") plt.ylim([0, None]) plt.xlim([-1, num]) @@ -68,7 +73,7 @@ for result in results: else args.moving_average_width ) - average = moving_average(times, moving_average_width) + average = moving_average(values, moving_average_width) plt.plot(nums, average, "-") if args.title: @@ -81,7 +86,7 @@ for result in results: legend.append("moving average") plt.legend(legend) -plt.ylabel("Time [s]") +plt.ylabel(metric_label) if args.output: plt.savefig(args.output) diff --git a/scripts/plot_utils.py b/scripts/plot_utils.py new file mode 100644 index 0000000..0ed0b2e --- /dev/null +++ b/scripts/plot_utils.py @@ -0,0 +1,27 @@ +"""Shared metric options for the plotting scripts.""" + +METRICS = { + "time_wall_clock": ("Time [s]", 1), + "time_user": ("User CPU time [s]", 1), + "time_system": ("System CPU time [s]", 1), + "memory_peak_resident": ("Peak resident memory [MiB]", 1024**2), +} + + +def add_metric_argument(parser): + parser.add_argument( + "--metric", + choices=METRICS, + default="time_wall_clock", + help="Metric to plot (default: time_wall_clock). " + "Times are in seconds; memory is in MiB.", + ) + + +def validate_metric(parser, results, metric): + for result in results: + if metric not in result["summary"]: + parser.error( + f"metric {metric!r} is unavailable for " + f"{result.get('name', result['command'])!r}" + ) diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index baaf175..e53d375 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -20,7 +20,10 @@ import json import matplotlib.pyplot as plt +from plot_utils import METRICS, add_metric_argument, validate_metric + parser = argparse.ArgumentParser(description=__doc__) +add_metric_argument(parser) parser.add_argument("file", help="JSON file with benchmark results") parser.add_argument("--title", help="Plot Title") parser.add_argument("--sort-by", choices=["median"], help="Sort method") @@ -30,26 +33,30 @@ parser.add_argument( parser.add_argument("-o", "--output", help="Save image to the given filename.") args = parser.parse_args() +metric_label, metric_scale = METRICS[args.metric] with open(args.file, encoding="utf-8") as f: results = json.load(f)["results"] +validate_metric(parser, results, args.metric) if args.labels: labels = args.labels.split(",") else: labels = [b.get("name", b["command"]) for b in results] -times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results] +values = [ + [m[args.metric]["value"] / metric_scale for m in b["measurements"]] for b in results +] if args.sort_by == "median": - medians = [b["summary"]["time_wall_clock"]["median"] for b in results] + medians = [b["summary"][args.metric]["median"] for b in results] indices = sorted(range(len(labels)), key=lambda k: medians[k]) labels = [labels[i] for i in indices] - times = [times[i] for i in indices] + values = [values[i] for i in indices] plt.figure(figsize=(10, 6), constrained_layout=True) -boxplot = plt.boxplot(times, vert=True, patch_artist=True) +boxplot = plt.boxplot(values, vert=True, patch_artist=True) cmap = plt.get_cmap("rainbow") -colors = [cmap(val / len(times)) for val in range(len(times))] +colors = [cmap(val / len(values)) for val in range(len(values))] for patch, color in zip(boxplot["boxes"], colors): patch.set_facecolor(color) @@ -57,7 +64,7 @@ for patch, color in zip(boxplot["boxes"], colors): if args.title: plt.title(args.title) plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium") -plt.ylabel("Time [s]") +plt.ylabel(metric_label) plt.ylim(0, None) plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45) if args.output: