Make plotting scripts runnable with uv

README.md

@@ -257,7 +257,16 @@ The JSON output is useful if you want to analyze the benchmark results in more d
 [`scripts/`](https://github.com/sharkdp/hyperfine/tree/master/scripts) folder includes a lot
 of helpful Python programs to further analyze benchmark results and create helpful
 visualizations, like a histogram of runtimes or a whisker plot to compare
-multiple benchmarks:
+multiple benchmarks.
+
+For example, these two commands create the following plots:
+
+```bash
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_histogram results.json
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_whisker results.json
+```
 
 | ![](doc/histogram.png) | ![](doc/whisker.png) |
 |---:|---:|

pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "hyperfine-scripts"
+version = "0.1.0"
+description = "Plot and analyze hyperfine JSON benchmark results"
+requires-python = ">=3.10"
+license-files = ["LICENSE-APACHE", "LICENSE-MIT"]
+dependencies = [
+    "matplotlib",
+    "numpy",
+    "pyqt6",
+    "scipy",
+]
+
+[project.scripts]
+advanced_statistics = "advanced_statistics:main"
+plot_benchmark_comparison = "plot_benchmark_comparison:main"
+plot_histogram = "plot_histogram:main"
+plot_parametrized = "plot_parametrized:main"
+plot_progression = "plot_progression:main"
+plot_whisker = "plot_whisker:main"
+welch_ttest = "welch_ttest:main"
+
+[tool.uv.build-backend]
+module-root = "scripts"
+module-name = ""
+namespace = true
+source-exclude = [
+    "/scripts/README.md",
+    "/scripts/ruff.toml",
+    "/scripts/.*",
+    "*.egg-info",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.23,<0.13"]
+build-backend = "uv_build"

scripts/README.md

@@ -1,23 +1,33 @@
 This folder contains scripts that can be used in combination with hyperfine's `--export-json` option.
 
-### Example:
+### Run directly from GitHub
+
+Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), then
+run one of the scripts:
 
 ```bash
 hyperfine 'sleep 0.020' 'sleep 0.021' 'sleep 0.022' --export-json sleep.json
-uv run plot_whisker.py sleep.json
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_whisker sleep.json
 ```
 
-All plotting and analysis scripts use the JSON export's `primary_metric` by default.
+Replace `plot_whisker` with any of the following commands.
+
+| Command | Description |
+| --- | --- |
+| `plot_whisker` | Compare benchmarks with a box and whisker plot |
+| `plot_histogram` | Show the distribution of measurements |
+| `plot_parametrized` | Plot results from a parameter scan |
+| `plot_progression` | Show measurements in run order |
+| `plot_benchmark_comparison` | Compare commands across multiple JSON exports |
+| `advanced_statistics` | Print additional statistics and percentiles |
+| `welch_ttest` | Compare two benchmarks using Welch's t-test |
+
+By default, the plotting and analysis scripts use the metric that was selected
+first in the actual hyperfine run (`--metrics` option).
+Use `--metric` to select another metric:
 
 ```bash
-uv run plot_histogram.py benchmark.json --metric instructions
+uvx --from 'git+https://github.com/sharkdp/hyperfine' \
+  plot_histogram benchmark.json --metric instructions
 ```
-
-### Prerequisites
-
-Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/) and run the
-commands above from this directory.
-
-The scripts declare their dependencies using [PEP 723](https://peps.python.org/pep-0723/)
-inline script metadata. `uv run` automatically installs these dependencies in an
-isolated environment before running the script.

scripts/advanced_statistics.py

@@ -12,6 +12,8 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("file", help="JSON file with benchmark results")
     add_metric_argument(parser)
@@ -31,12 +33,10 @@ if args.time_unit is not None:
         if args.time_unit == "millisecond":
             unit, scale = "ms", 0.001
 
-
     def format_value(value):
         suffix = f" {unit}" if unit else ""
         return f"{value:.3f}{suffix}"
 
-
     print(f"Metric: {description.label}\n")
     for result in results:
         summary = result["summary"][metric]
@@ -60,3 +60,7 @@ for result in results:
             f"  (IQR = {format_value(iqr)})"
         )
         print()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_benchmark_comparison.py

@@ -21,6 +21,8 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     add_metric_argument(parser)
     parser.add_argument(
@@ -68,7 +70,7 @@ fig.set_figheight(5)
     fig.set_figwidth(10)
     for i, command in enumerate(commands):
         offset = width * (i + 1)
-    rects = ax.bar(x + offset, data[i], width, label=command)
+        ax.bar(x + offset, data[i], width, label=command)
 
     ax.set_xticks(x + 0.5, inputs)
     ax.grid(visible=True, axis="y")
@@ -83,3 +85,7 @@ if args.output:
         plt.savefig(args.output)
     else:
         plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_histogram.py

@@ -17,6 +17,8 @@ import numpy as np
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     add_metric_argument(parser)
     parser.add_argument("file", help="JSON file with benchmark results")
@@ -77,7 +79,8 @@ if args.labels:
     else:
         labels = [b.get("name", b["command"]) for b in results]
     all_values = [
-    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
+        [m[metric]["value"] / metric_info.scale for m in b["measurements"]]
+        for b in results
     ]
 
     value_min = (
@@ -122,3 +125,7 @@ if args.output:
         plt.savefig(args.output, dpi=600)
     else:
         plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_parametrized.py

@@ -17,6 +17,8 @@ import matplotlib.pyplot as plt
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     add_metric_argument(parser)
     parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
@@ -50,12 +52,10 @@ if args.parameter_name is not None:
             "benchmark results\n"
         )
 
-
     def die(msg):
         sys.stderr.write(f"fatal: {msg}\n")
         sys.exit(1)
 
-
     def extract_parameters(results):
         """Return `(parameter_name: str, parameter_values: List[float])`."""
         if not results:
@@ -68,7 +68,6 @@ def extract_parameters(results):
             )
         return (next(iter(names)), list(values))
 
-
     def unique_parameter(benchmark):
         """Return the unique parameter `(name: str, value: float)`, or die."""
         params_dict = benchmark.get("parameters", {})
@@ -81,7 +80,6 @@ def unique_parameter(benchmark):
         [(name, value)] = params_dict.items()
         return (name, float(value["value"]))
 
-
     parameter_name = None
 
     for results in datasets:
@@ -119,3 +117,7 @@ if args.output:
         plt.savefig(args.output)
     else:
         plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_progression.py

@@ -29,6 +29,7 @@ def moving_average(values, num_runs):
     return np.convolve(values_padded, kernel, mode="valid")
 
 
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     add_metric_argument(parser)
     parser.add_argument("file", help="JSON file with benchmark results")
@@ -47,14 +48,14 @@ parser.add_argument(
         help="Do not show moving average curve",
     )
 
-
     args = parser.parse_args()
     metric, [results] = load_results(parser, [args.file], args.metric)
     metric_info = METRICS[metric]
 
     for result in results:
-    label = result.get("name", result["command"])
-    values = [m[metric]["value"] / metric_info.scale for m in result["measurements"]]
+        values = [
+            m[metric]["value"] / metric_info.scale for m in result["measurements"]
+        ]
         num = len(values)
         nums = range(num)
 
@@ -88,3 +89,7 @@ if args.output:
         plt.savefig(args.output)
     else:
         plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/plot_whisker.py

@@ -21,6 +21,8 @@ import matplotlib.pyplot as plt
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     add_metric_argument(parser)
     parser.add_argument("file", help="JSON file with benchmark results")
@@ -40,7 +42,8 @@ if args.labels:
     else:
         labels = [b.get("name", b["command"]) for b in results]
     values = [
-    [m[metric]["value"] / metric_info.scale for m in b["measurements"]] for b in results
+        [m[metric]["value"] / metric_info.scale for m in b["measurements"]]
+        for b in results
     ]
 
     if args.sort_by == "median":
@@ -67,3 +70,7 @@ if args.output:
         plt.savefig(args.output)
     else:
         plt.show()
+
+
+if __name__ == "__main__":
+    main()

scripts/welch_ttest.py

@@ -16,6 +16,8 @@ from scipy import stats
 
 from plot_utils import METRICS, add_metric_argument, load_results
 
+
+def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("file", help="JSON file with two benchmark results")
     add_metric_argument(parser)
@@ -51,3 +53,7 @@ else:
         print(
             f"No statistically significant difference in mean value was detected (p >= {th})."
         )
+
+
+if __name__ == "__main__":
+    main()