Export metric summaries and adapt analysis scripts

scripts/advanced_statistics.py

@@ -50,11 +50,9 @@ unit_str = str(unit)
 with open(args.file) as f:
     results = json.load(f)["results"]
 
-commands = [b["command"] for b in results]
-times = [b["times"] for b in results]
-
-for command, ts in zip(commands, times):
-    ts = [t * unit.factor() for t in ts]
+for result in results:
+    summary = result["summary"]["time_wall_clock"]
+    ts = [m["time_wall_clock"]["value"] * unit.factor() for m in result["measurements"]]
 
     p05 = np.percentile(ts, 5)
     p25 = np.percentile(ts, 25)
@@ -63,13 +61,16 @@ for command, ts in zip(commands, times):
 
     iqr = p75 - p25
 
-    print(f"Command '{command}'")
-    print(f"  runs:   {len(ts):8d}")
-    print(f"  mean:   {np.mean(ts):8.3f} {unit_str}")
-    print(f"  stddev: {np.std(ts, ddof=1):8.3f} {unit_str}")
-    print(f"  median: {np.median(ts):8.3f} {unit_str}")
-    print(f"  min:    {np.min(ts):8.3f} {unit_str}")
-    print(f"  max:    {np.max(ts):8.3f} {unit_str}")
+    print(f"Command '{result['command']}'")
+    print(f"  runs:   {summary['count']:8d}")
+    print(f"  mean:   {summary['mean'] * unit.factor():8.3f} {unit_str}")
+    if summary["stddev"] is None:
+        print(f"  stddev: {'N/A':>8}")
+    else:
+        print(f"  stddev: {summary['stddev'] * unit.factor():8.3f} {unit_str}")
+    print(f"  median: {summary['median'] * unit.factor():8.3f} {unit_str}")
+    print(f"  min:    {summary['min'] * unit.factor():8.3f} {unit_str}")
+    print(f"  max:    {summary['max'] * unit.factor():8.3f} {unit_str}")
     print()
     print("  percentiles:")
     print(f"     P_05 .. P_95:    {p05:.3f} {unit_str} .. {p95:.3f} {unit_str}")

scripts/plot_benchmark_comparison.py

@@ -51,7 +51,7 @@ for i, filename in enumerate(args.files):
         assert (
             commands == benchmark_commands
         ), f"Unexpected commands in {filename}: {benchmark_commands}, expected: {commands}"
-    data.append([round(b["mean"], 2) for b in results])
+    data.append([b["summary"]["time_wall_clock"]["mean"] for b in results])
     if args.benchmark_names:
         inputs.append(args.benchmark_names[i])
     else:

scripts/plot_histogram.py

@@ -67,7 +67,9 @@ if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b["command"] for b in results]
-all_times = [b["times"] for b in results]
+all_times = [
+    [m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results
+]
 
 t_min = float(args.t_min) if args.t_min else np.min(list(map(np.min, all_times)))
 t_max = float(args.t_max) if args.t_max else np.max(list(map(np.max, all_times)))

scripts/plot_parametrized.py

@@ -71,7 +71,7 @@ def unique_parameter(benchmark):
             f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
         )
     [(name, value)] = params_dict.items()
-    return (name, float(value))
+    return (name, float(value["value"]))
 
 
 parameter_name = None
@@ -87,8 +87,11 @@ for filename in args.file:
         )
     parameter_name = this_parameter_name
 
-    times_mean = [b["mean"] for b in results]
-    times_stddev = [b["stddev"] for b in results]
+    summaries = [b["summary"]["time_wall_clock"] for b in results]
+    times_mean = [s["mean"] for s in summaries]
+    times_stddev = [
+        s["stddev"] if s["stddev"] is not None else float("nan") for s in summaries
+    ]
 
     plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)
 

scripts/plot_progression.py

@@ -53,7 +53,7 @@ with open(args.file) as f:
 
 for result in results:
     label = result["command"]
-    times = result["times"]
+    times = [m["time_wall_clock"]["value"] for m in result["measurements"]]
     num = len(times)
     nums = range(num)
 
@@ -63,7 +63,9 @@ for result in results:
 
     if not args.no_moving_average:
         moving_average_width = (
-            num // 5 if args.moving_average_width is None else args.moving_average_width
+            max(1, num // 5)
+            if args.moving_average_width is None
+            else args.moving_average_width
         )
 
         average = moving_average(times, moving_average_width)

scripts/plot_whisker.py

@@ -38,10 +38,10 @@ if args.labels:
     labels = args.labels.split(",")
 else:
     labels = [b["command"] for b in results]
-times = [b["times"] for b in results]
+times = [[m["time_wall_clock"]["value"] for m in b["measurements"]] for b in results]
 
 if args.sort_by == "median":
-    medians = [b["median"] for b in results]
+    medians = [b["summary"]["time_wall_clock"]["median"] for b in results]
     indices = sorted(range(len(labels)), key=lambda k: medians[k])
     labels = [labels[i] for i in indices]
     times = [times[i] for i in indices]

scripts/welch_ttest.py

@@ -27,7 +27,7 @@ if len(results) != 2:
     sys.exit(1)
 
 a, b = (x["command"] for x in results[:2])
-X, Y = (x["times"] for x in results[:2])
+X, Y = ([m["time_wall_clock"]["value"] for m in x["measurements"]] for x in results[:2])
 
 print(f"Command 1: {a}")
 print(f"Command 2: {b}\n")