From ce495297802766c3d916df930462d4e8a1bce86e Mon Sep 17 00:00:00 2001 From: David Peter Date: Wed, 7 Oct 2026 11:24:14 +0200 Subject: [PATCH] Make plotting scripts runnable with uv --- README.md | 11 +- pyproject.toml | 36 +++++ scripts/README.md | 36 +++-- scripts/advanced_statistics.py | 96 +++++++------ scripts/plot_benchmark_comparison.py | 116 ++++++++-------- scripts/plot_histogram.py | 201 ++++++++++++++------------- scripts/plot_parametrized.py | 194 +++++++++++++------------- scripts/plot_progression.py | 119 ++++++++-------- scripts/plot_whisker.py | 87 ++++++------ scripts/welch_ttest.py | 64 +++++---- 10 files changed, 526 insertions(+), 434 deletions(-) create mode 100644 pyproject.toml diff --git a/README.md b/README.md index 8e24014..5c1c2a5 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,16 @@ The JSON output is useful if you want to analyze the benchmark results in more d [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot of helpful Python programs to further analyze benchmark results and create helpful visualizations, like a histogram of runtimes or a whisker plot to compare -multiple benchmarks: +multiple benchmarks. + +For example, these two commands create the following plots: + +```bash +uvx --from 'git+https://github.com/sharkdp/hyperfine' \ + plot_histogram results.json +uvx --from 'git+https://github.com/sharkdp/hyperfine' \ + plot_whisker results.json +``` | ![](doc/histogram.png) | ![](doc/whisker.png) | |---:|---:| diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..663d170 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "hyperfine-scripts" +version = "0.1.0" +description = "Plot and analyze hyperfine JSON benchmark results" +requires-python = ">=3.10" +license-files = ["LICENSE-APACHE", "LICENSE-MIT"] +dependencies = [ + "matplotlib", + "numpy", + "pyqt6", + "scipy", +] + +[project.scripts] +advanced_statistics = "advanced_statistics:main" +plot_benchmark_comparison = "plot_benchmark_comparison:main" +plot_histogram = "plot_histogram:main" +plot_parametrized = "plot_parametrized:main" +plot_progression = "plot_progression:main" +plot_whisker = "plot_whisker:main" +welch_ttest = "welch_ttest:main" + +[tool.uv.build-backend] +module-root = "scripts" +module-name = "" +namespace = true +source-exclude = [ + "/scripts/README.md", + "/scripts/ruff.toml", + "/scripts/.*", + "*.egg-info", +] + +[build-system] +requires = ["uv_build>=0.12.23,<0.13"] +build-backend = "uv_build" diff --git a/scripts/README.md b/scripts/README.md index fcf9c7d..9875a5e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,23 +1,33 @@ This folder contains scripts that can be used in combination with hyperfine's `--export-json` option. -### Example: +### Run directly from GitHub + +Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), then +run one of the scripts: ```bash hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json -uv run plot_whisker.py sleep.json +uvx --from 'git+https://github.com/sharkdp/hyperfine' \ + plot_whisker sleep.json ``` -All plotting and analysis scripts use the JSON export's `primary_metric` by default. +Replace `plot_whisker` with any of the following commands. + +| Command | Description | +| --- | --- | +| `plot_whisker` | Compare benchmarks with a box and whisker plot | +| `plot_histogram` | Show the distribution of measurements | +| `plot_parametrized` | Plot results from a parameter scan | +| `plot_progression` | Show measurements in run order | +| `plot_benchmark_comparison` | Compare commands across multiple JSON exports | +| `advanced_statistics` | Print additional statistics and percentiles | +| `welch_ttest` | Compare two benchmarks using Welch's t-test | + +By default, the plotting and analysis scripts use the metric that was selected +first in the actual hyperfine run (`--metrics` option). +Use `--metric` to select another metric: ```bash -uv run plot_histogram.py benchmark.json --metric instructions +uvx --from 'git+https://github.com/sharkdp/hyperfine' \ + plot_histogram benchmark.json --metric instructions ``` - -### Prerequisites - -Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the -commands above from this directory. - -The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/) -inline script metadata. `uv run` automatically installs these dependencies in an -isolated environment before running the script. diff --git a/scripts/advanced_statistics.py b/scripts/advanced_statistics.py index 1a8ddda..a5212e0 100755 --- a/scripts/advanced_statistics.py +++ b/scripts/advanced_statistics.py @@ -12,51 +12,55 @@ import numpy as np from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser() -parser.add_argument("file", help="JSON file with benchmark results") -add_metric_argument(parser) -parser.add_argument( - "--time-unit", - help="Display unit for time metrics (default: second).", - choices=["second", "millisecond"], -) -args = parser.parse_args() -metric, [results] = load_results(parser, [args.file], args.metric) -description = METRICS[metric] -unit, scale = description.unit, description.scale -if args.time_unit is not None: - if not metric.startswith("time_"): - parser.error("--time-unit can only be used with a time metric") - if args.time_unit == "millisecond": - unit, scale = "ms", 0.001 - - -def format_value(value): - suffix = f" {unit}" if unit else "" - return f"{value:.3f}{suffix}" - - -print(f"Metric: {description.label}\n") -for result in results: - summary = result["summary"][metric] - values = [m[metric]["value"] / scale for m in result["measurements"]] - - p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95]) - - iqr = p75 - p25 - - print(f"Command '{result.get('name', result['command'])}'") - print(f" runs: {summary['count']:8d}") - for statistic in ["mean", "stddev", "median", "min", "max"]: - value = summary[statistic] - formatted = "N/A" if value is None else format_value(value / scale) - print(f" {statistic + ':':<8}{formatted:>8}") - print() - print(" percentiles:") - print(f" P_05 .. P_95: {format_value(p05)} .. {format_value(p95)}") - print( - f" P_25 .. P_75: {format_value(p25)} .. {format_value(p75)}" - f" (IQR = {format_value(iqr)})" +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("file", help="JSON file with benchmark results") + add_metric_argument(parser) + parser.add_argument( + "--time-unit", + help="Display unit for time metrics (default: second).", + choices=["second", "millisecond"], ) - print() + args = parser.parse_args() + + metric, [results] = load_results(parser, [args.file], args.metric) + description = METRICS[metric] + unit, scale = description.unit, description.scale + if args.time_unit is not None: + if not metric.startswith("time_"): + parser.error("--time-unit can only be used with a time metric") + if args.time_unit == "millisecond": + unit, scale = "ms", 0.001 + + def format_value(value): + suffix = f" {unit}" if unit else "" + return f"{value:.3f}{suffix}" + + print(f"Metric: {description.label}\n") + for result in results: + summary = result["summary"][metric] + values = [m[metric]["value"] / scale for m in result["measurements"]] + + p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95]) + + iqr = p75 - p25 + + print(f"Command '{result.get('name', result['command'])}'") + print(f" runs: {summary['count']:8d}") + for statistic in ["mean", "stddev", "median", "min", "max"]: + value = summary[statistic] + formatted = "N/A" if value is None else format_value(value / scale) + print(f" {statistic + ':':<8}{formatted:>8}") + print() + print(" percentiles:") + print(f" P_05 .. P_95: {format_value(p05)} .. {format_value(p95)}") + print( + f" P_25 .. P_75: {format_value(p25)} .. {format_value(p75)}" + f" (IQR = {format_value(iqr)})" + ) + print() + + +if __name__ == "__main__": + main() diff --git a/scripts/plot_benchmark_comparison.py b/scripts/plot_benchmark_comparison.py index a0e7459..6d191c9 100755 --- a/scripts/plot_benchmark_comparison.py +++ b/scripts/plot_benchmark_comparison.py @@ -21,65 +21,71 @@ import numpy as np from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser(description=__doc__) -add_metric_argument(parser) -parser.add_argument( - "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results" -) -parser.add_argument("--title", help="Plot Title") -parser.add_argument( - "--benchmark-names", nargs="+", help="Names of the benchmark groups" -) -parser.add_argument("-o", "--output", help="Save image to the given filename") -args = parser.parse_args() -metric, datasets = load_results(parser, args.files, args.metric) -metric_info = METRICS[metric] - -commands = None -data = [] -inputs = [] - -if args.benchmark_names: - assert len(args.files) == len(args.benchmark_names), ( - "Number of benchmark names must match the number of input files." +def main(): + parser = argparse.ArgumentParser(description=__doc__) + add_metric_argument(parser) + parser.add_argument( + "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results" ) + parser.add_argument("--title", help="Plot Title") + parser.add_argument( + "--benchmark-names", nargs="+", help="Names of the benchmark groups" + ) + parser.add_argument("-o", "--output", help="Save image to the given filename") + + args = parser.parse_args() + metric, datasets = load_results(parser, args.files, args.metric) + metric_info = METRICS[metric] + + commands = None + data = [] + inputs = [] -for i, (filename, results) in enumerate(zip(args.files, datasets)): - benchmark_commands = [b.get("name", b["command"]) for b in results] - if commands is None: - commands = benchmark_commands - else: - assert commands == benchmark_commands, ( - f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" - ) - data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results]) if args.benchmark_names: - inputs.append(args.benchmark_names[i]) + assert len(args.files) == len(args.benchmark_names), ( + "Number of benchmark names must match the number of input files." + ) + + for i, (filename, results) in enumerate(zip(args.files, datasets)): + benchmark_commands = [b.get("name", b["command"]) for b in results] + if commands is None: + commands = benchmark_commands + else: + assert commands == benchmark_commands, ( + f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}" + ) + data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results]) + if args.benchmark_names: + inputs.append(args.benchmark_names[i]) + else: + inputs.append(filename.stem) + + data = np.transpose(data) + x = np.arange(len(inputs)) # the label locations + width = 0.25 # the width of the bars + + fig, ax = plt.subplots(layout="constrained") + fig.set_figheight(5) + fig.set_figwidth(10) + for i, command in enumerate(commands): + offset = width * (i + 1) + ax.bar(x + offset, data[i], width, label=command) + + ax.set_xticks(x + 0.5, inputs) + ax.grid(visible=True, axis="y") + + if args.title: + plt.title(args.title) + plt.xlabel("Benchmark") + plt.ylabel(metric_info.axis_label) + plt.legend(title="Command") + + if args.output: + plt.savefig(args.output) else: - inputs.append(filename.stem) + plt.show() -data = np.transpose(data) -x = np.arange(len(inputs)) # the label locations -width = 0.25 # the width of the bars -fig, ax = plt.subplots(layout="constrained") -fig.set_figheight(5) -fig.set_figwidth(10) -for i, command in enumerate(commands): - offset = width * (i + 1) - rects = ax.bar(x + offset, data[i], width, label=command) - -ax.set_xticks(x + 0.5, inputs) -ax.grid(visible=True, axis="y") - -if args.title: - plt.title(args.title) -plt.xlabel("Benchmark") -plt.ylabel(metric_info.axis_label) -plt.legend(title="Command") - -if args.output: - plt.savefig(args.output) -else: - plt.show() +if __name__ == "__main__": + main() diff --git a/scripts/plot_histogram.py b/scripts/plot_histogram.py index 46b15ce..62a2954 100755 --- a/scripts/plot_histogram.py +++ b/scripts/plot_histogram.py @@ -17,108 +17,115 @@ import numpy as np from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser(description=__doc__) -add_metric_argument(parser) -parser.add_argument("file", help="JSON file with benchmark results") -parser.add_argument("--title", help="Plot title") -parser.add_argument( - "--labels", help="Comma-separated list of entries for the plot legend" -) -parser.add_argument("--bins", help="Number of bins (default: auto)") -parser.add_argument( - "--legend-location", - help="Location of the legend on plot (default: upper center)", - choices=[ - "upper center", - "lower center", - "right", - "left", - "best", - "upper left", - "upper right", - "lower left", - "lower right", - "center left", - "center right", - "center", - ], - default="upper center", -) -parser.add_argument( - "--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)" -) -parser.add_argument("-o", "--output", help="Save image to the given filename.") -parser.add_argument( - "--min", - "--t-min", - dest="value_min", - type=float, - help="Minimum metric value to display", -) -parser.add_argument( - "--max", - "--t-max", - dest="value_max", - type=float, - help="Maximum metric value to display", -) -parser.add_argument( - "--log-count", - help="Use a logarithmic y-axis for the event count", - action="store_true", -) -args = parser.parse_args() -metric, [results] = load_results(parser, [args.file], args.metric) -metric_info = METRICS[metric] +def main(): + parser = argparse.ArgumentParser(description=__doc__) + add_metric_argument(parser) + parser.add_argument("file", help="JSON file with benchmark results") + parser.add_argument("--title", help="Plot title") + parser.add_argument( + "--labels", help="Comma-separated list of entries for the plot legend" + ) + parser.add_argument("--bins", help="Number of bins (default: auto)") + parser.add_argument( + "--legend-location", + help="Location of the legend on plot (default: upper center)", + choices=[ + "upper center", + "lower center", + "right", + "left", + "best", + "upper left", + "upper right", + "lower left", + "lower right", + "center left", + "center right", + "center", + ], + default="upper center", + ) + parser.add_argument( + "--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)" + ) + parser.add_argument("-o", "--output", help="Save image to the given filename.") + parser.add_argument( + "--min", + "--t-min", + dest="value_min", + type=float, + help="Minimum metric value to display", + ) + parser.add_argument( + "--max", + "--t-max", + dest="value_max", + type=float, + help="Maximum metric value to display", + ) + parser.add_argument( + "--log-count", + help="Use a logarithmic y-axis for the event count", + action="store_true", + ) -if args.labels: - labels = args.labels.split(",") -else: - labels = [b.get("name", b["command"]) for b in results] -all_values = [ - [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results -] + args = parser.parse_args() + metric, [results] = load_results(parser, [args.file], args.metric) + metric_info = METRICS[metric] -value_min = ( - args.value_min - if args.value_min is not None - else np.min(list(map(np.min, all_values))) -) -value_max = ( - args.value_max - if args.value_max is not None - else np.max(list(map(np.max, all_values))) -) + if args.labels: + labels = args.labels.split(",") + else: + labels = [b.get("name", b["command"]) for b in results] + all_values = [ + [m[metric]["value"] / metric_info.scale for m in b["measurements"]] + for b in results + ] -bins = int(args.bins) if args.bins else "auto" -histtype = args.type if args.type else "bar" + value_min = ( + args.value_min + if args.value_min is not None + else np.min(list(map(np.min, all_values))) + ) + value_max = ( + args.value_max + if args.value_max is not None + else np.max(list(map(np.max, all_values))) + ) -plt.figure(figsize=(10, 5)) -plt.hist( - all_values, - label=labels, - bins=bins, - histtype=histtype, - range=(value_min, value_max), -) -plt.legend( - loc=args.legend_location, - fancybox=True, - shadow=True, - prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]}, -) + bins = int(args.bins) if args.bins else "auto" + histtype = args.type if args.type else "bar" -plt.xlabel(metric_info.axis_label) -if args.title: - plt.title(args.title) + plt.figure(figsize=(10, 5)) + plt.hist( + all_values, + label=labels, + bins=bins, + histtype=histtype, + range=(value_min, value_max), + ) + plt.legend( + loc=args.legend_location, + fancybox=True, + shadow=True, + prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]}, + ) -if args.log_count: - plt.yscale("log") -else: - plt.ylim(0, None) + plt.xlabel(metric_info.axis_label) + if args.title: + plt.title(args.title) -if args.output: - plt.savefig(args.output, dpi=600) -else: - plt.show() + if args.log_count: + plt.yscale("log") + else: + plt.ylim(0, None) + + if args.output: + plt.savefig(args.output, dpi=600) + else: + plt.show() + + +if __name__ == "__main__": + main() diff --git a/scripts/plot_parametrized.py b/scripts/plot_parametrized.py index 724f0b2..47eb814 100755 --- a/scripts/plot_parametrized.py +++ b/scripts/plot_parametrized.py @@ -17,105 +17,107 @@ import matplotlib.pyplot as plt from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser(description=__doc__) -add_metric_argument(parser) -parser.add_argument("file", help="JSON file with benchmark results", nargs="+") -parser.add_argument( - "--parameter-name", - metavar="name", - type=str, - help="Deprecated; parameter names are now inferred from benchmark files", -) -parser.add_argument( - "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true" -) -parser.add_argument( - "--log-y", - "--log-time", - dest="log_y", - help="Use a logarithmic metric axis", - action="store_true", -) -parser.add_argument( - "--titles", help="Comma-separated list of titles for the plot legend" -) -parser.add_argument("-o", "--output", help="Save image to the given filename.") -args = parser.parse_args() -metric, datasets = load_results(parser, args.file, args.metric) -metric_info = METRICS[metric] -if args.parameter_name is not None: - sys.stderr.write( - "warning: --parameter-name is deprecated; names are inferred from " - "benchmark results\n" +def main(): + parser = argparse.ArgumentParser(description=__doc__) + add_metric_argument(parser) + parser.add_argument("file", help="JSON file with benchmark results", nargs="+") + parser.add_argument( + "--parameter-name", + metavar="name", + type=str, + help="Deprecated; parameter names are now inferred from benchmark files", ) + parser.add_argument( + "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true" + ) + parser.add_argument( + "--log-y", + "--log-time", + dest="log_y", + help="Use a logarithmic metric axis", + action="store_true", + ) + parser.add_argument( + "--titles", help="Comma-separated list of titles for the plot legend" + ) + parser.add_argument("-o", "--output", help="Save image to the given filename.") - -def die(msg): - sys.stderr.write(f"fatal: {msg}\n") - sys.exit(1) - - -def extract_parameters(results): - """Return `(parameter_name: str, parameter_values: List[float])`.""" - if not results: - die("no benchmark data to plot") - (names, values) = zip(*(unique_parameter(b) for b in results)) - names = frozenset(names) - if len(names) != 1: - die( - f"benchmarks must all have the same parameter name, but found: {sorted(names)}" + args = parser.parse_args() + metric, datasets = load_results(parser, args.file, args.metric) + metric_info = METRICS[metric] + if args.parameter_name is not None: + sys.stderr.write( + "warning: --parameter-name is deprecated; names are inferred from " + "benchmark results\n" ) - return (next(iter(names)), list(values)) + + def die(msg): + sys.stderr.write(f"fatal: {msg}\n") + sys.exit(1) + + def extract_parameters(results): + """Return `(parameter_name: str, parameter_values: List[float])`.""" + if not results: + die("no benchmark data to plot") + (names, values) = zip(*(unique_parameter(b) for b in results)) + names = frozenset(names) + if len(names) != 1: + die( + f"benchmarks must all have the same parameter name, but found: {sorted(names)}" + ) + return (next(iter(names)), list(values)) + + def unique_parameter(benchmark): + """Return the unique parameter `(name: str, value: float)`, or die.""" + params_dict = benchmark.get("parameters", {}) + if not params_dict: + die("benchmarks must have exactly one parameter, but found none") + if len(params_dict) > 1: + die( + f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}" + ) + [(name, value)] = params_dict.items() + return (name, float(value["value"])) + + parameter_name = None + + for results in datasets: + (this_parameter_name, parameter_values) = extract_parameters(results) + if parameter_name is not None and this_parameter_name != parameter_name: + die( + f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}" + ) + parameter_name = this_parameter_name + + summaries = [b["summary"][metric] for b in results] + means = [s["mean"] / metric_info.scale for s in summaries] + stddevs = [ + s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan") + for s in summaries + ] + + plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2) + + plt.xlabel(parameter_name) + plt.ylabel(metric_info.axis_label) + + if args.log_y: + plt.yscale("log") + else: + plt.ylim(0, None) + + if args.log_x: + plt.xscale("log") + + if args.titles: + plt.legend(args.titles.split(",")) + + if args.output: + plt.savefig(args.output) + else: + plt.show() -def unique_parameter(benchmark): - """Return the unique parameter `(name: str, value: float)`, or die.""" - params_dict = benchmark.get("parameters", {}) - if not params_dict: - die("benchmarks must have exactly one parameter, but found none") - if len(params_dict) > 1: - die( - f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}" - ) - [(name, value)] = params_dict.items() - return (name, float(value["value"])) - - -parameter_name = None - -for results in datasets: - (this_parameter_name, parameter_values) = extract_parameters(results) - if parameter_name is not None and this_parameter_name != parameter_name: - die( - f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}" - ) - parameter_name = this_parameter_name - - summaries = [b["summary"][metric] for b in results] - means = [s["mean"] / metric_info.scale for s in summaries] - stddevs = [ - s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan") - for s in summaries - ] - - plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2) - -plt.xlabel(parameter_name) -plt.ylabel(metric_info.axis_label) - -if args.log_y: - plt.yscale("log") -else: - plt.ylim(0, None) - -if args.log_x: - plt.xscale("log") - -if args.titles: - plt.legend(args.titles.split(",")) - -if args.output: - plt.savefig(args.output) -else: - plt.show() +if __name__ == "__main__": + main() diff --git a/scripts/plot_progression.py b/scripts/plot_progression.py index c84c482..c1de074 100755 --- a/scripts/plot_progression.py +++ b/scripts/plot_progression.py @@ -29,62 +29,67 @@ def moving_average(values, num_runs): return np.convolve(values_padded, kernel, mode="valid") -parser = argparse.ArgumentParser(description=__doc__) -add_metric_argument(parser) -parser.add_argument("file", help="JSON file with benchmark results") -parser.add_argument("--title", help="Plot Title") -parser.add_argument("-o", "--output", help="Save image to the given filename.") -parser.add_argument( - "-w", - "--moving-average-width", - type=int, - metavar="num_runs", - help="Width of the moving-average window (default: N/5)", -) -parser.add_argument( - "--no-moving-average", - action="store_true", - help="Do not show moving average curve", -) +def main(): + parser = argparse.ArgumentParser(description=__doc__) + add_metric_argument(parser) + parser.add_argument("file", help="JSON file with benchmark results") + parser.add_argument("--title", help="Plot Title") + parser.add_argument("-o", "--output", help="Save image to the given filename.") + parser.add_argument( + "-w", + "--moving-average-width", + type=int, + metavar="num_runs", + help="Width of the moving-average window (default: N/5)", + ) + parser.add_argument( + "--no-moving-average", + action="store_true", + help="Do not show moving average curve", + ) + + args = parser.parse_args() + metric, [results] = load_results(parser, [args.file], args.metric) + metric_info = METRICS[metric] + + for result in results: + values = [ + m[metric]["value"] / metric_info.scale for m in result["measurements"] + ] + num = len(values) + nums = range(num) + + plt.scatter(x=nums, y=values, marker=".") + plt.ylim([0, None]) + plt.xlim([-1, num]) + + if not args.no_moving_average: + moving_average_width = ( + max(1, num // 5) + if args.moving_average_width is None + else args.moving_average_width + ) + + average = moving_average(values, moving_average_width) + plt.plot(nums, average, "-") + + if args.title: + plt.title(args.title) + + legend = [] + for result in results: + legend.append(result.get("name", result["command"])) + if not args.no_moving_average: + legend.append("moving average") + plt.legend(legend) + + plt.ylabel(metric_info.axis_label) + + if args.output: + plt.savefig(args.output) + else: + plt.show() -args = parser.parse_args() -metric, [results] = load_results(parser, [args.file], args.metric) -metric_info = METRICS[metric] - -for result in results: - label = result.get("name", result["command"]) - values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]] - num = len(values) - nums = range(num) - - plt.scatter(x=nums, y=values, marker=".") - plt.ylim([0, None]) - plt.xlim([-1, num]) - - if not args.no_moving_average: - moving_average_width = ( - max(1, num // 5) - if args.moving_average_width is None - else args.moving_average_width - ) - - average = moving_average(values, moving_average_width) - plt.plot(nums, average, "-") - -if args.title: - plt.title(args.title) - -legend = [] -for result in results: - legend.append(result.get("name", result["command"])) - if not args.no_moving_average: - legend.append("moving average") -plt.legend(legend) - -plt.ylabel(metric_info.axis_label) - -if args.output: - plt.savefig(args.output) -else: - plt.show() +if __name__ == "__main__": + main() diff --git a/scripts/plot_whisker.py b/scripts/plot_whisker.py index d3c33b8..fc27c93 100755 --- a/scripts/plot_whisker.py +++ b/scripts/plot_whisker.py @@ -21,49 +21,56 @@ import matplotlib.pyplot as plt from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser(description=__doc__) -add_metric_argument(parser) -parser.add_argument("file", help="JSON file with benchmark results") -parser.add_argument("--title", help="Plot Title") -parser.add_argument("--sort-by", choices=["median"], help="Sort method") -parser.add_argument( - "--labels", help="Comma-separated list of entries for the plot legend" -) -parser.add_argument("-o", "--output", help="Save image to the given filename.") -args = parser.parse_args() -metric, [results] = load_results(parser, [args.file], args.metric) -metric_info = METRICS[metric] +def main(): + parser = argparse.ArgumentParser(description=__doc__) + add_metric_argument(parser) + parser.add_argument("file", help="JSON file with benchmark results") + parser.add_argument("--title", help="Plot Title") + parser.add_argument("--sort-by", choices=["median"], help="Sort method") + parser.add_argument( + "--labels", help="Comma-separated list of entries for the plot legend" + ) + parser.add_argument("-o", "--output", help="Save image to the given filename.") -if args.labels: - labels = args.labels.split(",") -else: - labels = [b.get("name", b["command"]) for b in results] -values = [ - [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results -] + args = parser.parse_args() + metric, [results] = load_results(parser, [args.file], args.metric) + metric_info = METRICS[metric] -if args.sort_by == "median": - medians = [b["summary"][metric]["median"] for b in results] - indices = sorted(range(len(labels)), key=lambda k: medians[k]) - labels = [labels[i] for i in indices] - values = [values[i] for i in indices] + if args.labels: + labels = args.labels.split(",") + else: + labels = [b.get("name", b["command"]) for b in results] + values = [ + [m[metric]["value"] / metric_info.scale for m in b["measurements"]] + for b in results + ] -plt.figure(figsize=(10, 6), constrained_layout=True) -boxplot = plt.boxplot(values, vert=True, patch_artist=True) -cmap = plt.get_cmap("rainbow") -colors = [cmap(val / len(values)) for val in range(len(values))] + if args.sort_by == "median": + medians = [b["summary"][metric]["median"] for b in results] + indices = sorted(range(len(labels)), key=lambda k: medians[k]) + labels = [labels[i] for i in indices] + values = [values[i] for i in indices] -for patch, color in zip(boxplot["boxes"], colors): - patch.set_facecolor(color) + plt.figure(figsize=(10, 6), constrained_layout=True) + boxplot = plt.boxplot(values, vert=True, patch_artist=True) + cmap = plt.get_cmap("rainbow") + colors = [cmap(val / len(values)) for val in range(len(values))] -if args.title: - plt.title(args.title) -plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium") -plt.ylabel(metric_info.axis_label) -plt.ylim(0, None) -plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45) -if args.output: - plt.savefig(args.output) -else: - plt.show() + for patch, color in zip(boxplot["boxes"], colors): + patch.set_facecolor(color) + + if args.title: + plt.title(args.title) + plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium") + plt.ylabel(metric_info.axis_label) + plt.ylim(0, None) + plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45) + if args.output: + plt.savefig(args.output) + else: + plt.show() + + +if __name__ == "__main__": + main() diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index 1f76b37..35a2605 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -16,38 +16,44 @@ from scipy import stats from plot_utils import METRICS, add_metric_argument, load_results -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("file", help="JSON file with two benchmark results") -add_metric_argument(parser) -args = parser.parse_args() -metric, [results] = load_results(parser, [args.file], args.metric) +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("file", help="JSON file with two benchmark results") + add_metric_argument(parser) + args = parser.parse_args() -if len(results) != 2: - parser.error("the input file must contain exactly two benchmarks") + metric, [results] = load_results(parser, [args.file], args.metric) -a, b = (x.get("name", x["command"]) for x in results[:2]) -X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2]) -if min(len(X), len(Y)) < 2: - parser.error("Welch's t-test requires at least two measurements per benchmark") + if len(results) != 2: + parser.error("the input file must contain exactly two benchmarks") -print(f"Metric: {METRICS[metric].label}") -print(f"Command 1: {a}") -print(f"Command 2: {b}\n") + a, b = (x.get("name", x["command"]) for x in results[:2]) + X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2]) + if min(len(X), len(Y)) < 2: + parser.error("Welch's t-test requires at least two measurements per benchmark") -t, p = stats.ttest_ind(X, Y, equal_var=False) -if not math.isfinite(p): - parser.error("Welch's t-test is undefined for these measurements") -th = 0.05 -dispose = p < th -print(f"t = {t:.3}, p = {p:.3}") -print() + print(f"Metric: {METRICS[metric].label}") + print(f"Command 1: {a}") + print(f"Command 2: {b}\n") -if dispose: - print( - f"A statistically significant difference in mean value was detected (p < {th})." - ) -else: - print( - f"No statistically significant difference in mean value was detected (p >= {th})." - ) + t, p = stats.ttest_ind(X, Y, equal_var=False) + if not math.isfinite(p): + parser.error("Welch's t-test is undefined for these measurements") + th = 0.05 + dispose = p < th + print(f"t = {t:.3}, p = {p:.3}") + print() + + if dispose: + print( + f"A statistically significant difference in mean value was detected (p < {th})." + ) + else: + print( + f"No statistically significant difference in mean value was detected (p >= {th})." + ) + + +if __name__ == "__main__": + main()