From 5a3fc3c92afb3c005e099a97153e248e3f6b1945 Mon Sep 17 00:00:00 2001 From: Yasuaki Uechi Date: Mon, 7 Oct 2019 14:57:42 +0900 Subject: [PATCH] scripts: add welch's t test --- scripts/welch_ttest.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 scripts/welch_ttest.py diff --git a/scripts/welch_ttest.py b/scripts/welch_ttest.py new file mode 100755 index 0000000..ace25a2 --- /dev/null +++ b/scripts/welch_ttest.py @@ -0,0 +1,28 @@ +#!/usr/bin/python + +import argparse +import json +from scipy import stats + +parser = argparse.ArgumentParser() +parser.add_argument("file", help="JSON file with benchmark results") +args = parser.parse_args() + +with open(args.file) as f: + results = json.load(f)["results"] + +a, b = [x["command"] for x in results[:2]] +X, Y = [x["times"] for x in results[:2]] + +print("Command 1: {}".format(a)) +print("Command 2: {}\n".format(b)) + +t, p = stats.ttest_ind(X, Y, equal_var=False) +th = 0.05 +dispose = p < th +print("t = {}, p = {}".format(t, p)) +print("p < {} = {}\n".format(th, dispose)) +if dispose: + print("There is a difference between the two benchmarks.") +else: + print("The two benchmarks are almost the same.") \ No newline at end of file