Make plotting scripts runnable with uv

README.md

@@ -257,7 +257,16 @@ The JSON output is useful if you want to analyze the benchmark results in more d
 [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot
 of helpful Python programs to further analyze benchmark results and create helpful
 visualizations, like a histogram of runtimes or a whisker plot to compare
-multiple benchmarks:
+multiple benchmarks.
+
+For example, these two commands create the following plots:
+
+```bash
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_histogram results.json
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_whisker results.json
+```
 
 | ![](doc/histogram.png) | ![](doc/whisker.png) |
 |---:|---:|

pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "hyperfine-scripts"
+version = "0.1.0"
+description = "Plot and analyze hyperfine JSON benchmark results"
+requires-python = ">=3.10"
+license-files = ["LICENSE-APACHE", "LICENSE-MIT"]
+dependencies = [
+    "matplotlib",
+    "numpy",
+    "pyqt6",
+    "scipy",
+]
+
+[project.scripts]
+advanced_statistics = "advanced_statistics:main"
+plot_benchmark_comparison = "plot_benchmark_comparison:main"
+plot_histogram = "plot_histogram:main"
+plot_parametrized = "plot_parametrized:main"
+plot_progression = "plot_progression:main"
+plot_whisker = "plot_whisker:main"
+welch_ttest = "welch_ttest:main"
+
+[tool.uv.build-backend]
+module-root = "scripts"
+module-name = ""
+namespace = true
+source-exclude = [
+    "/scripts/README.md",
+    "/scripts/ruff.toml",
+    "/scripts/.*",
+    "*.egg-info",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.23,<0.13"]
+build-backend = "uv_build"

scripts/README.md

@@ -1,23 +1,33 @@
 This folder contains scripts that can be used in combination with hyperfine's `--export-json` option.
 
-### Example:
+### Run directly from GitHub
+
+Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), then
+run one of the scripts:
 
 ```bash
 hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
-uv run plot_whisker.py sleep.json
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_whisker sleep.json
 ```
 
-All plotting and analysis scripts use the JSON export's `primary_metric` by default.
+Replace `plot_whisker` with any of the following commands.
+
+| Command | Description |
+| --- | --- |
+| `plot_whisker` | Compare benchmarks with a box and whisker plot |
+| `plot_histogram` | Show the distribution of measurements |
+| `plot_parametrized` | Plot results from a parameter scan |
+| `plot_progression` | Show measurements in run order |
+| `plot_benchmark_comparison` | Compare commands across multiple JSON exports |
+| `advanced_statistics` | Print additional statistics and percentiles |
+| `welch_ttest` | Compare two benchmarks using Welch's t-test |
+
+By default, the plotting and analysis scripts use the metric that was selected
+first in the actual hyperfine run (`--metrics` option).
+Use `--metric` to select another metric:
 
 ```bash
-uv run plot_histogram.py benchmark.json --metric instructions
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_histogram benchmark.json --metric instructions
 ```
-
-### Prerequisites
-
-Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
-commands above from this directory.
-
-The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
-inline script metadata. `uv run` automatically installs these dependencies in an
-isolated environment before running the script.

scripts/advanced_statistics.py

@@ -12,51 +12,55 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser()
-parser.add_argument("file", help="JSON file with benchmark results")
-add_metric_argument(parser)
-parser.add_argument(
-    "--time-unit",
-    help="Display unit for time metrics (default: second).",
-    choices=["second", "millisecond"],
-)
-args = parser.parse_args()
 
-metric, [results] = load_results(parser, [args.file], args.metric)
+def main():
-description = METRICS[metric]
+    parser = argparse.ArgumentParser()
-unit, scale = description.unit, description.scale
+    parser.add_argument("file", help="JSON file with benchmark results")
-if args.time_unit is not None:
+    add_metric_argument(parser)
-    if not metric.startswith("time_"):
+    parser.add_argument(
-        parser.error("--time-unit can only be used with a time metric")
+        "--time-unit",
-    if args.time_unit == "millisecond":
+        help="Display unit for time metrics (default: second).",
-        unit, scale = "ms", 0.001
+        choices=["second", "millisecond"],
-
-
-def format_value(value):
-    suffix = f" {unit}" if unit else ""
-    return f"{value:.3f}{suffix}"
-
-
-print(f"Metric: {description.label}\n")
-for result in results:
-    summary = result["summary"][metric]
-    values = [m[metric]["value"] / scale for m in result["measurements"]]
-
-    p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95])
-
-    iqr = p75 - p25
-
-    print(f"Command '{result.get('name', result['command'])}'")
-    print(f"  runs:   {summary['count']:8d}")
-    for statistic in ["mean", "stddev", "median", "min", "max"]:
-        value = summary[statistic]
-        formatted = "N/A" if value is None else format_value(value / scale)
-        print(f"  {statistic + ':':<8}{formatted:>8}")
-    print()
-    print("  percentiles:")
-    print(f"     P_05 .. P_95:    {format_value(p05)} .. {format_value(p95)}")
-    print(
-        f"     P_25 .. P_75:    {format_value(p25)} .. {format_value(p75)}"
-        f"  (IQR = {format_value(iqr)})"
     )
-    print()
+    args = parser.parse_args()
+
+    metric, [results] = load_results(parser, [args.file], args.metric)
+    description = METRICS[metric]
+    unit, scale = description.unit, description.scale
+    if args.time_unit is not None:
+        if not metric.startswith("time_"):
+            parser.error("--time-unit can only be used with a time metric")
+        if args.time_unit == "millisecond":
+            unit, scale = "ms", 0.001
+
+    def format_value(value):
+        suffix = f" {unit}" if unit else ""
+        return f"{value:.3f}{suffix}"
+
+    print(f"Metric: {description.label}\n")
+    for result in results:
+        summary = result["summary"][metric]
+        values = [m[metric]["value"] / scale for m in result["measurements"]]
+
+        p05, p25, p75, p95 = np.percentile(values, [5, 25, 75, 95])
+
+        iqr = p75 - p25
+
+        print(f"Command '{result.get('name', result['command'])}'")
+        print(f"  runs:   {summary['count']:8d}")
+        for statistic in ["mean", "stddev", "median", "min", "max"]:
+            value = summary[statistic]
+            formatted = "N/A" if value is None else format_value(value / scale)
+            print(f"  {statistic + ':':<8}{formatted:>8}")
+        print()
+        print("  percentiles:")
+        print(f"     P_05 .. P_95:    {format_value(p05)} .. {format_value(p95)}")
+        print(
+            f"     P_25 .. P_75:    {format_value(p25)} .. {format_value(p75)}"
+            f"  (IQR = {format_value(iqr)})"
+        )
+        print()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_benchmark_comparison.py

@@ -21,65 +21,71 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser(description=__doc__)
-add_metric_argument(parser)
-parser.add_argument(
-    "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
-)
-parser.add_argument("--title", help="Plot Title")
-parser.add_argument(
-    "--benchmark-names", nargs="+", help="Names of the benchmark groups"
-)
-parser.add_argument("-o", "--output", help="Save image to the given filename")
 
-args = parser.parse_args()
+def main():
-metric, datasets = load_results(parser, args.files, args.metric)
+    parser = argparse.ArgumentParser(description=__doc__)
-metric_info = METRICS[metric]
+    add_metric_argument(parser)
-
+    parser.add_argument(
-commands = None
+        "files", nargs="+", type=pathlib.Path, help="JSON files with benchmark results"
-data = []
-inputs = []
-
-if args.benchmark_names:
-    assert len(args.files) == len(args.benchmark_names), (
-        "Number of benchmark names must match the number of input files."
     )
+    parser.add_argument("--title", help="Plot Title")
+    parser.add_argument(
+        "--benchmark-names", nargs="+", help="Names of the benchmark groups"
+    )
+    parser.add_argument("-o", "--output", help="Save image to the given filename")
+
+    args = parser.parse_args()
+    metric, datasets = load_results(parser, args.files, args.metric)
+    metric_info = METRICS[metric]
+
+    commands = None
+    data = []
+    inputs = []
 
-for i, (filename, results) in enumerate(zip(args.files, datasets)):
-    benchmark_commands = [b.get("name", b["command"]) for b in results]
-    if commands is None:
-        commands = benchmark_commands
-    else:
-        assert commands == benchmark_commands, (
-            f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
-        )
-    data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
     if args.benchmark_names:
-        inputs.append(args.benchmark_names[i])
+        assert len(args.files) == len(args.benchmark_names), (
+            "Number of benchmark names must match the number of input files."
+        )
+
+    for i, (filename, results) in enumerate(zip(args.files, datasets)):
+        benchmark_commands = [b.get("name", b["command"]) for b in results]
+        if commands is None:
+            commands = benchmark_commands
+        else:
+            assert commands == benchmark_commands, (
+                f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
+            )
+        data.append([b["summary"][metric]["mean"] / metric_info.scale for b in results])
+        if args.benchmark_names:
+            inputs.append(args.benchmark_names[i])
+        else:
+            inputs.append(filename.stem)
+
+    data = np.transpose(data)
+    x = np.arange(len(inputs))  # the label locations
+    width = 0.25  # the width of the bars
+
+    fig, ax = plt.subplots(layout="constrained")
+    fig.set_figheight(5)
+    fig.set_figwidth(10)
+    for i, command in enumerate(commands):
+        offset = width * (i + 1)
+        ax.bar(x + offset, data[i], width, label=command)
+
+    ax.set_xticks(x + 0.5, inputs)
+    ax.grid(visible=True, axis="y")
+
+    if args.title:
+        plt.title(args.title)
+    plt.xlabel("Benchmark")
+    plt.ylabel(metric_info.axis_label)
+    plt.legend(title="Command")
+
+    if args.output:
+        plt.savefig(args.output)
     else:
-        inputs.append(filename.stem)
+        plt.show()
 
-data = np.transpose(data)
-x = np.arange(len(inputs))  # the label locations
-width = 0.25  # the width of the bars
 
-fig, ax = plt.subplots(layout="constrained")
+if __name__ == "__main__":
-fig.set_figheight(5)
+    main()
-fig.set_figwidth(10)
-for i, command in enumerate(commands):
-    offset = width * (i + 1)
-    rects = ax.bar(x + offset, data[i], width, label=command)
-
-ax.set_xticks(x + 0.5, inputs)
-ax.grid(visible=True, axis="y")
-
-if args.title:
-    plt.title(args.title)
-plt.xlabel("Benchmark")
-plt.ylabel(metric_info.axis_label)
-plt.legend(title="Command")
-
-if args.output:
-    plt.savefig(args.output)
-else:
-    plt.show()

scripts/plot_histogram.py

@@ -17,108 +17,115 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser(description=__doc__)
-add_metric_argument(parser)
-parser.add_argument("file", help="JSON file with benchmark results")
-parser.add_argument("--title", help="Plot title")
-parser.add_argument(
-    "--labels", help="Comma-separated list of entries for the plot legend"
-)
-parser.add_argument("--bins", help="Number of bins (default: auto)")
-parser.add_argument(
-    "--legend-location",
-    help="Location of the legend on plot (default: upper center)",
-    choices=[
-        "upper center",
-        "lower center",
-        "right",
-        "left",
-        "best",
-        "upper left",
-        "upper right",
-        "lower left",
-        "lower right",
-        "center left",
-        "center right",
-        "center",
-    ],
-    default="upper center",
-)
-parser.add_argument(
-    "--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)"
-)
-parser.add_argument("-o", "--output", help="Save image to the given filename.")
-parser.add_argument(
-    "--min",
-    "--t-min",
-    dest="value_min",
-    type=float,
-    help="Minimum metric value to display",
-)
-parser.add_argument(
-    "--max",
-    "--t-max",
-    dest="value_max",
-    type=float,
-    help="Maximum metric value to display",
-)
-parser.add_argument(
-    "--log-count",
-    help="Use a logarithmic y-axis for the event count",
-    action="store_true",
-)
 
-args = parser.parse_args()
+def main():
-metric, [results] = load_results(parser, [args.file], args.metric)
+    parser = argparse.ArgumentParser(description=__doc__)
-metric_info = METRICS[metric]
+    add_metric_argument(parser)
+    parser.add_argument("file", help="JSON file with benchmark results")
+    parser.add_argument("--title", help="Plot title")
+    parser.add_argument(
+        "--labels", help="Comma-separated list of entries for the plot legend"
+    )
+    parser.add_argument("--bins", help="Number of bins (default: auto)")
+    parser.add_argument(
+        "--legend-location",
+        help="Location of the legend on plot (default: upper center)",
+        choices=[
+            "upper center",
+            "lower center",
+            "right",
+            "left",
+            "best",
+            "upper left",
+            "upper right",
+            "lower left",
+            "lower right",
+            "center left",
+            "center right",
+            "center",
+        ],
+        default="upper center",
+    )
+    parser.add_argument(
+        "--type", help="Type of histogram (*bar*, barstacked, step, stepfilled)"
+    )
+    parser.add_argument("-o", "--output", help="Save image to the given filename.")
+    parser.add_argument(
+        "--min",
+        "--t-min",
+        dest="value_min",
+        type=float,
+        help="Minimum metric value to display",
+    )
+    parser.add_argument(
+        "--max",
+        "--t-max",
+        dest="value_max",
+        type=float,
+        help="Maximum metric value to display",
+    )
+    parser.add_argument(
+        "--log-count",
+        help="Use a logarithmic y-axis for the event count",
+        action="store_true",
+    )
 
-if args.labels:
+    args = parser.parse_args()
-    labels = args.labels.split(",")
+    metric, [results] = load_results(parser, [args.file], args.metric)
-else:
+    metric_info = METRICS[metric]
-    labels = [b.get("name", b["command"]) for b in results]
-all_values = [
-    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
-]
 
-value_min = (
+    if args.labels:
-    args.value_min
+        labels = args.labels.split(",")
-    if args.value_min is not None
+    else:
-    else np.min(list(map(np.min, all_values)))
+        labels = [b.get("name", b["command"]) for b in results]
-)
+    all_values = [
-value_max = (
+        [m[metric]["value"] / metric_info.scale for m in b["measurements"]]
-    args.value_max
+        for b in results
-    if args.value_max is not None
+    ]
-    else np.max(list(map(np.max, all_values)))
-)
 
-bins = int(args.bins) if args.bins else "auto"
+    value_min = (
-histtype = args.type if args.type else "bar"
+        args.value_min
+        if args.value_min is not None
+        else np.min(list(map(np.min, all_values)))
+    )
+    value_max = (
+        args.value_max
+        if args.value_max is not None
+        else np.max(list(map(np.max, all_values)))
+    )
 
-plt.figure(figsize=(10, 5))
+    bins = int(args.bins) if args.bins else "auto"
-plt.hist(
+    histtype = args.type if args.type else "bar"
-    all_values,
-    label=labels,
-    bins=bins,
-    histtype=histtype,
-    range=(value_min, value_max),
-)
-plt.legend(
-    loc=args.legend_location,
-    fancybox=True,
-    shadow=True,
-    prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
-)
 
-plt.xlabel(metric_info.axis_label)
+    plt.figure(figsize=(10, 5))
-if args.title:
+    plt.hist(
-    plt.title(args.title)
+        all_values,
+        label=labels,
+        bins=bins,
+        histtype=histtype,
+        range=(value_min, value_max),
+    )
+    plt.legend(
+        loc=args.legend_location,
+        fancybox=True,
+        shadow=True,
+        prop={"size": 10, "family": ["Source Code Pro", "Fira Mono", "Courier New"]},
+    )
 
-if args.log_count:
+    plt.xlabel(metric_info.axis_label)
-    plt.yscale("log")
+    if args.title:
-else:
+        plt.title(args.title)
-    plt.ylim(0, None)
 
-if args.output:
+    if args.log_count:
-    plt.savefig(args.output, dpi=600)
+        plt.yscale("log")
-else:
+    else:
-    plt.show()
+        plt.ylim(0, None)
+
+    if args.output:
+        plt.savefig(args.output, dpi=600)
+    else:
+        plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_parametrized.py

@@ -17,105 +17,107 @@ import matplotlib.pyplot as plt
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser(description=__doc__)
-add_metric_argument(parser)
-parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
-parser.add_argument(
-    "--parameter-name",
-    metavar="name",
-    type=str,
-    help="Deprecated; parameter names are now inferred from benchmark files",
-)
-parser.add_argument(
-    "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
-)
-parser.add_argument(
-    "--log-y",
-    "--log-time",
-    dest="log_y",
-    help="Use a logarithmic metric axis",
-    action="store_true",
-)
-parser.add_argument(
-    "--titles", help="Comma-separated list of titles for the plot legend"
-)
-parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
-args = parser.parse_args()
+def main():
-metric, datasets = load_results(parser, args.file, args.metric)
+    parser = argparse.ArgumentParser(description=__doc__)
-metric_info = METRICS[metric]
+    add_metric_argument(parser)
-if args.parameter_name is not None:
+    parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
-    sys.stderr.write(
+    parser.add_argument(
-        "warning: --parameter-name is deprecated; names are inferred from "
+        "--parameter-name",
-        "benchmark results\n"
+        metavar="name",
+        type=str,
+        help="Deprecated; parameter names are now inferred from benchmark files",
     )
+    parser.add_argument(
+        "--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
+    )
+    parser.add_argument(
+        "--log-y",
+        "--log-time",
+        dest="log_y",
+        help="Use a logarithmic metric axis",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--titles", help="Comma-separated list of titles for the plot legend"
+    )
+    parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
-
+    args = parser.parse_args()
-def die(msg):
+    metric, datasets = load_results(parser, args.file, args.metric)
-    sys.stderr.write(f"fatal: {msg}\n")
+    metric_info = METRICS[metric]
-    sys.exit(1)
+    if args.parameter_name is not None:
-
+        sys.stderr.write(
-
+            "warning: --parameter-name is deprecated; names are inferred from "
-def extract_parameters(results):
+            "benchmark results\n"
-    """Return `(parameter_name: str, parameter_values: List[float])`."""
-    if not results:
-        die("no benchmark data to plot")
-    (names, values) = zip(*(unique_parameter(b) for b in results))
-    names = frozenset(names)
-    if len(names) != 1:
-        die(
-            f"benchmarks must all have the same parameter name, but found: {sorted(names)}"
         )
-    return (next(iter(names)), list(values))
+
+    def die(msg):
+        sys.stderr.write(f"fatal: {msg}\n")
+        sys.exit(1)
+
+    def extract_parameters(results):
+        """Return `(parameter_name: str, parameter_values: List[float])`."""
+        if not results:
+            die("no benchmark data to plot")
+        (names, values) = zip(*(unique_parameter(b) for b in results))
+        names = frozenset(names)
+        if len(names) != 1:
+            die(
+                f"benchmarks must all have the same parameter name, but found: {sorted(names)}"
+            )
+        return (next(iter(names)), list(values))
+
+    def unique_parameter(benchmark):
+        """Return the unique parameter `(name: str, value: float)`, or die."""
+        params_dict = benchmark.get("parameters", {})
+        if not params_dict:
+            die("benchmarks must have exactly one parameter, but found none")
+        if len(params_dict) > 1:
+            die(
+                f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
+            )
+        [(name, value)] = params_dict.items()
+        return (name, float(value["value"]))
+
+    parameter_name = None
+
+    for results in datasets:
+        (this_parameter_name, parameter_values) = extract_parameters(results)
+        if parameter_name is not None and this_parameter_name != parameter_name:
+            die(
+                f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}"
+            )
+        parameter_name = this_parameter_name
+
+        summaries = [b["summary"][metric] for b in results]
+        means = [s["mean"] / metric_info.scale for s in summaries]
+        stddevs = [
+            s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan")
+            for s in summaries
+        ]
+
+        plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
+
+    plt.xlabel(parameter_name)
+    plt.ylabel(metric_info.axis_label)
+
+    if args.log_y:
+        plt.yscale("log")
+    else:
+        plt.ylim(0, None)
+
+    if args.log_x:
+        plt.xscale("log")
+
+    if args.titles:
+        plt.legend(args.titles.split(","))
+
+    if args.output:
+        plt.savefig(args.output)
+    else:
+        plt.show()
 
 
-def unique_parameter(benchmark):
+if __name__ == "__main__":
-    """Return the unique parameter `(name: str, value: float)`, or die."""
+    main()
-    params_dict = benchmark.get("parameters", {})
-    if not params_dict:
-        die("benchmarks must have exactly one parameter, but found none")
-    if len(params_dict) > 1:
-        die(
-            f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
-        )
-    [(name, value)] = params_dict.items()
-    return (name, float(value["value"]))
-
-
-parameter_name = None
-
-for results in datasets:
-    (this_parameter_name, parameter_values) = extract_parameters(results)
-    if parameter_name is not None and this_parameter_name != parameter_name:
-        die(
-            f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}"
-        )
-    parameter_name = this_parameter_name
-
-    summaries = [b["summary"][metric] for b in results]
-    means = [s["mean"] / metric_info.scale for s in summaries]
-    stddevs = [
-        s["stddev"] / metric_info.scale if s["stddev"] is not None else float("nan")
-        for s in summaries
-    ]
-
-    plt.errorbar(x=parameter_values, y=means, yerr=stddevs, capsize=2)
-
-plt.xlabel(parameter_name)
-plt.ylabel(metric_info.axis_label)
-
-if args.log_y:
-    plt.yscale("log")
-else:
-    plt.ylim(0, None)
-
-if args.log_x:
-    plt.xscale("log")
-
-if args.titles:
-    plt.legend(args.titles.split(","))
-
-if args.output:
-    plt.savefig(args.output)
-else:
-    plt.show()

scripts/plot_progression.py

@@ -29,62 +29,67 @@ def moving_average(values, num_runs):
     return np.convolve(values_padded, kernel, mode="valid")
 
 
-parser = argparse.ArgumentParser(description=__doc__)
+def main():
-add_metric_argument(parser)
+    parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument("file", help="JSON file with benchmark results")
+    add_metric_argument(parser)
-parser.add_argument("--title", help="Plot Title")
+    parser.add_argument("file", help="JSON file with benchmark results")
-parser.add_argument("-o", "--output", help="Save image to the given filename.")
+    parser.add_argument("--title", help="Plot Title")
-parser.add_argument(
+    parser.add_argument("-o", "--output", help="Save image to the given filename.")
-    "-w",
+    parser.add_argument(
-    "--moving-average-width",
+        "-w",
-    type=int,
+        "--moving-average-width",
-    metavar="num_runs",
+        type=int,
-    help="Width of the moving-average window (default: N/5)",
+        metavar="num_runs",
-)
+        help="Width of the moving-average window (default: N/5)",
-parser.add_argument(
+    )
-    "--no-moving-average",
+    parser.add_argument(
-    action="store_true",
+        "--no-moving-average",
-    help="Do not show moving average curve",
+        action="store_true",
-)
+        help="Do not show moving average curve",
+    )
+
+    args = parser.parse_args()
+    metric, [results] = load_results(parser, [args.file], args.metric)
+    metric_info = METRICS[metric]
+
+    for result in results:
+        values = [
+            m[metric]["value"] / metric_info.scale for m in result["measurements"]
+        ]
+        num = len(values)
+        nums = range(num)
+
+        plt.scatter(x=nums, y=values, marker=".")
+        plt.ylim([0, None])
+        plt.xlim([-1, num])
+
+        if not args.no_moving_average:
+            moving_average_width = (
+                max(1, num // 5)
+                if args.moving_average_width is None
+                else args.moving_average_width
+            )
+
+            average = moving_average(values, moving_average_width)
+            plt.plot(nums, average, "-")
+
+    if args.title:
+        plt.title(args.title)
+
+    legend = []
+    for result in results:
+        legend.append(result.get("name", result["command"]))
+        if not args.no_moving_average:
+            legend.append("moving average")
+    plt.legend(legend)
+
+    plt.ylabel(metric_info.axis_label)
+
+    if args.output:
+        plt.savefig(args.output)
+    else:
+        plt.show()
 
 
-args = parser.parse_args()
+if __name__ == "__main__":
-metric, [results] = load_results(parser, [args.file], args.metric)
+    main()
-metric_info = METRICS[metric]
-
-for result in results:
-    label = result.get("name", result["command"])
-    values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]]
-    num = len(values)
-    nums = range(num)
-
-    plt.scatter(x=nums, y=values, marker=".")
-    plt.ylim([0, None])
-    plt.xlim([-1, num])
-
-    if not args.no_moving_average:
-        moving_average_width = (
-            max(1, num // 5)
-            if args.moving_average_width is None
-            else args.moving_average_width
-        )
-
-        average = moving_average(values, moving_average_width)
-        plt.plot(nums, average, "-")
-
-if args.title:
-    plt.title(args.title)
-
-legend = []
-for result in results:
-    legend.append(result.get("name", result["command"]))
-    if not args.no_moving_average:
-        legend.append("moving average")
-plt.legend(legend)
-
-plt.ylabel(metric_info.axis_label)
-
-if args.output:
-    plt.savefig(args.output)
-else:
-    plt.show()

scripts/plot_whisker.py

@@ -21,49 +21,56 @@ import matplotlib.pyplot as plt
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser(description=__doc__)
-add_metric_argument(parser)
-parser.add_argument("file", help="JSON file with benchmark results")
-parser.add_argument("--title", help="Plot Title")
-parser.add_argument("--sort-by", choices=["median"], help="Sort method")
-parser.add_argument(
-    "--labels", help="Comma-separated list of entries for the plot legend"
-)
-parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
-args = parser.parse_args()
+def main():
-metric, [results] = load_results(parser, [args.file], args.metric)
+    parser = argparse.ArgumentParser(description=__doc__)
-metric_info = METRICS[metric]
+    add_metric_argument(parser)
+    parser.add_argument("file", help="JSON file with benchmark results")
+    parser.add_argument("--title", help="Plot Title")
+    parser.add_argument("--sort-by", choices=["median"], help="Sort method")
+    parser.add_argument(
+        "--labels", help="Comma-separated list of entries for the plot legend"
+    )
+    parser.add_argument("-o", "--output", help="Save image to the given filename.")
 
-if args.labels:
+    args = parser.parse_args()
-    labels = args.labels.split(",")
+    metric, [results] = load_results(parser, [args.file], args.metric)
-else:
+    metric_info = METRICS[metric]
-    labels = [b.get("name", b["command"]) for b in results]
-values = [
-    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
-]
 
-if args.sort_by == "median":
+    if args.labels:
-    medians = [b["summary"][metric]["median"] for b in results]
+        labels = args.labels.split(",")
-    indices = sorted(range(len(labels)), key=lambda k: medians[k])
+    else:
-    labels = [labels[i] for i in indices]
+        labels = [b.get("name", b["command"]) for b in results]
-    values = [values[i] for i in indices]
+    values = [
+        [m[metric]["value"] / metric_info.scale for m in b["measurements"]]
+        for b in results
+    ]
 
-plt.figure(figsize=(10, 6), constrained_layout=True)
+    if args.sort_by == "median":
-boxplot = plt.boxplot(values, vert=True, patch_artist=True)
+        medians = [b["summary"][metric]["median"] for b in results]
-cmap = plt.get_cmap("rainbow")
+        indices = sorted(range(len(labels)), key=lambda k: medians[k])
-colors = [cmap(val / len(values)) for val in range(len(values))]
+        labels = [labels[i] for i in indices]
+        values = [values[i] for i in indices]
 
-for patch, color in zip(boxplot["boxes"], colors):
+    plt.figure(figsize=(10, 6), constrained_layout=True)
-    patch.set_facecolor(color)
+    boxplot = plt.boxplot(values, vert=True, patch_artist=True)
+    cmap = plt.get_cmap("rainbow")
+    colors = [cmap(val / len(values)) for val in range(len(values))]
 
-if args.title:
+    for patch, color in zip(boxplot["boxes"], colors):
-    plt.title(args.title)
+        patch.set_facecolor(color)
-plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
+
-plt.ylabel(metric_info.axis_label)
+    if args.title:
-plt.ylim(0, None)
+        plt.title(args.title)
-plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
+    plt.legend(handles=boxplot["boxes"], labels=labels, loc="best", fontsize="medium")
-if args.output:
+    plt.ylabel(metric_info.axis_label)
-    plt.savefig(args.output)
+    plt.ylim(0, None)
-else:
+    plt.xticks(list(range(1, len(labels) + 1)), labels, rotation=45)
-    plt.show()
+    if args.output:
+        plt.savefig(args.output)
+    else:
+        plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/welch_ttest.py

@@ -16,38 +16,44 @@ from scipy import stats
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
-parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument("file", help="JSON file with two benchmark results")
-add_metric_argument(parser)
-args = parser.parse_args()
 
-metric, [results] = load_results(parser, [args.file], args.metric)
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("file", help="JSON file with two benchmark results")
+    add_metric_argument(parser)
+    args = parser.parse_args()
 
-if len(results) != 2:
+    metric, [results] = load_results(parser, [args.file], args.metric)
-    parser.error("the input file must contain exactly two benchmarks")
 
-a, b = (x.get("name", x["command"]) for x in results[:2])
+    if len(results) != 2:
-X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2])
+        parser.error("the input file must contain exactly two benchmarks")
-if min(len(X), len(Y)) < 2:
-    parser.error("Welch's t-test requires at least two measurements per benchmark")
 
-print(f"Metric: {METRICS[metric].label}")
+    a, b = (x.get("name", x["command"]) for x in results[:2])
-print(f"Command 1: {a}")
+    X, Y = ([m[metric]["value"] for m in x["measurements"]] for x in results[:2])
-print(f"Command 2: {b}\n")
+    if min(len(X), len(Y)) < 2:
+        parser.error("Welch's t-test requires at least two measurements per benchmark")
 
-t, p = stats.ttest_ind(X, Y, equal_var=False)
+    print(f"Metric: {METRICS[metric].label}")
-if not math.isfinite(p):
+    print(f"Command 1: {a}")
-    parser.error("Welch's t-test is undefined for these measurements")
+    print(f"Command 2: {b}\n")
-th = 0.05
-dispose = p < th
-print(f"t = {t:.3}, p = {p:.3}")
-print()
 
-if dispose:
+    t, p = stats.ttest_ind(X, Y, equal_var=False)
-    print(
+    if not math.isfinite(p):
-        f"A statistically significant difference in mean value was detected (p < {th})."
+        parser.error("Welch's t-test is undefined for these measurements")
-    )
+    th = 0.05
-else:
+    dispose = p < th
-    print(
+    print(f"t = {t:.3}, p = {p:.3}")
-        f"No statistically significant difference in mean value was detected (p >= {th})."
+    print()
-    )
+
+    if dispose:
+        print(
+            f"A statistically significant difference in mean value was detected (p < {th})."
+        )
+    else:
+        print(
+            f"No statistically significant difference in mean value was detected (p >= {th})."
+        )
+
+
+if __name__ == "__main__":
+    main()