From 49ebab2eb45c8530f509e82ca0364de9e62f45c5 Mon Sep 17 00:00:00 2001 From: David Peter Date: Fri, 2 Oct 2026 11:29:15 +0200 Subject: [PATCH] Correct the interpretation of Welch's t-test results --- scripts/welch_ttest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py index 78d6303..6183848 100755 --- a/scripts/welch_ttest.py +++ b/scripts/welch_ttest.py @@ -7,8 +7,7 @@ # /// """This script performs Welch's t-test on a JSON export file with two -benchmark results to test whether or not the two distributions are -the same.""" +benchmark results to test for a difference in their population mean runtimes.""" import argparse import json @@ -40,6 +39,10 @@ print(f"t = {t:.3}, p = {p:.3}") print() if dispose: - print(f"There is a difference between the two benchmarks (p < {th}).") + print( + f"A statistically significant difference in mean runtime was detected (p < {th})." + ) else: - print(f"The two benchmarks are almost the same (p >= {th}).") + print( + f"No statistically significant difference in mean runtime was detected (p >= {th})." + )