blob: f77971c205700edfe7db27b80b6ca76577b8194c [file] [log] [blame]
#!/usr/bin/env python -u
# Copyright (C) 2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. Neither the name of Apple Inc. ("Apple") nor the names of
# its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import collections
import sys
import argparse
import json
import itertools
from webkitpy.benchmark_runner.benchmark_results import BenchmarkResults
from webkitpy.benchmark_runner.benchmark_json_merge import mergeJSONs
try:
from scipy import stats
except:
print("ERROR: scipy package is not installed. Run `pip install scipy`")
sys.exit(1)
try:
import numpy
except:
print("ERROR: numpy package is not installed. Run `pip install numpy`")
sys.exit(1)
def readJSONFile(path):
with open(path, 'r') as contents:
result = json.loads(contents.read())
if 'debugOutput' in result:
del result['debugOutput']
return result
Speedometer2 = "Speedometer2"
JetStream2 = "JetStream2"
PLT5 = "PLT5"
CompetitivePLT = "CompetitivePLT"
PLUM3 = "PLUM3"
MotionMark = "MotionMark"
MotionMark1_1 = "MotionMark-1.1"
MotionMark1_1_1 = "MotionMark-1.1.1"
unitMarker = "__unit__"
def speedometer2Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "ms"
for test in breakdown._results["Speedometer-2"]["tests"].keys():
result[test] = breakdown._results["Speedometer-2"]["tests"][test]["metrics"]["Time"]["Total"]["current"]
return result
def speedometer2BreakdownSyncAsync(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "ms"
for test in breakdown._results["Speedometer-2"]["tests"].keys():
syncTime = None
asyncTime = None
for value in breakdown._results["Speedometer-2"]["tests"][test]["tests"].values():
syncArray = value["tests"]["Sync"]["metrics"]["Time"][None]["current"]
asyncArray = value["tests"]["Async"]["metrics"]["Time"][None]["current"]
if not syncTime:
syncTime = syncArray
asyncTime = asyncArray
else:
syncTime = [x + y for x, y in zip(syncTime, syncArray)]
asyncTime = [x + y for x, y in zip(asyncTime, asyncArray)]
result[test + "-sync"] = syncTime
result[test + "-async"] = asyncTime
return result
def jetStream2Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "pts"
for test in breakdown._results["JetStream2.0"]["tests"].keys():
result[test] = breakdown._results["JetStream2.0"]["tests"][test]["metrics"]["Score"][None]["current"]
return result
def motionMarkBreakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "pts"
if detectMotionMark(jsonObject):
name = "MotionMark"
elif detectMotionMark1_1(jsonObject):
name = "MotionMark-1.1"
else:
name = "MotionMark-1.1.1"
for test in breakdown._results[name]["tests"].keys():
result[test] = breakdown._results[name]["tests"][test]["metrics"]["Score"][None]["current"]
return result
def plt5Breakdown(jsonObject):
nameMapping = {}
for mappings in jsonObject["urls"]:
for key in mappings.keys():
nameMapping[key] = mappings[key]
result = {}
result[unitMarker] = "ms"
for test in jsonObject["iterations"][0]["warm"].keys():
if test == "Geometric":
continue
result["warm--" + nameMapping[test]] = []
result["cold--" + nameMapping[test]] = []
for payload in jsonObject["iterations"]:
warmTests = payload["warm"]
coldTests = payload["cold"]
for test in warmTests.keys():
if test == "Geometric":
continue
result["warm--" + nameMapping[test]].append(warmTests[test]["Geometric"])
result["cold--" + nameMapping[test]].append(coldTests[test]["Geometric"])
return result
def competitivePLTBreakdown(jsonObject):
result = collections.defaultdict(list)
result[unitMarker] = "sec"
safari_results = jsonObject.get('Safari', {})
cold_results = safari_results.get('cold', {})
warm_results = safari_results.get('warm', {})
cold_link_results = cold_results.get('add-and-click-link', {})
warm_link_results = warm_results.get('add-and-click-link', {})
for site_to_times in cold_link_results.values():
for site, times in site_to_times.items():
result["cold--fmp--" + site].append(times['first_meaningful_paint'])
result["cold--load-end--" + site].append(times['load_end'])
for site_to_times in warm_link_results.values():
for site, times in site_to_times.items():
result["warm--fmp--" + site].append(times['first_meaningful_paint'])
result["warm--load-end--" + site].append(times['load_end'])
return result
def plum3Breakdown(jsonObject):
breakdown = BenchmarkResults(jsonObject)
result = {}
result[unitMarker] = "B"
for test in breakdown._results["PLUM3-PhysFootprint"]["tests"].keys():
result[test] = breakdown._results["PLUM3-PhysFootprint"]["tests"][test]["metrics"]["Allocations"]["Geometric"]["current"]
return result
def displayStr(value):
return "{:.6f}".format(float(value))
def computeMultipleHypothesesSignificance(a, b):
# This is using the Benjamini-Hochberg procedure based on False Discovery Rate
# for computing signifcance in multiple hypothesis testing
# Read more here:
# - https://en.wikipedia.org/wiki/False_discovery_rate
# - https://www.stat.berkeley.edu/~mgoldman/Section0402.pdf
# This is best used for independent variables. We know subtests aren't
# fully independent, this it's a reasonable approximation.
# We use this instead of Bonferroni because we control for almost the same
# false positive error rate (marking as signficant when it's not), but with a much
# lower false negative error rate (not marking something as signficant when it is).
sortedPValues = []
reversePValueMap = {}
for key in a.keys():
if key == unitMarker:
continue
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
sortedPValues.append(pValue)
if pValue not in reversePValueMap:
reversePValueMap[pValue] = []
reversePValueMap[pValue].append(key)
sortedPValues.sort()
assert sortedPValues[0] <= sortedPValues[-1]
isSignificant = False
result = {}
rank = float(len(sortedPValues))
for pValue in reversed(sortedPValues):
assert rank >= 1.0
threshold = (rank * .05) / float(len(sortedPValues))
if pValue <= threshold:
isSignificant = True
assert len(reversePValueMap[pValue]) > 0
for test in reversePValueMap[pValue]:
result[test] = isSignificant
rank = rank - 1.0
return result
def dumpBreakdowns(a, b):
nameLength = len("subtest")
aLength = len(a[unitMarker])
bLength = len(a[unitMarker])
ratioLength = len("b / a")
pValueHeader = "pValue (significance using False Discovery Rate)"
pLength = len(pValueHeader)
isSignificant = computeMultipleHypothesesSignificance(a, b)
for key in a.keys():
if key == unitMarker:
continue
nameLength = max(nameLength, len(key))
aLength = max(aLength, len(displayStr(numpy.mean(a[key]))))
bLength = max(bLength, len(displayStr(numpy.mean(b[key]))))
ratioLength = max(ratioLength, len(displayStr(numpy.mean(b[key]) / numpy.mean(a[key]))))
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = ""
if isSignificant[key]:
significantStr = " (significant)"
pLength = max(pLength, len(displayStr(pValue)) + len(significantStr))
aLength += 2
bLength += 2
nameLength += 2
ratioLength += 2
pLength += 2
strings = []
strings.append("|{key:^{nameLength}}|{aScore:^{aLength}} |{bScore:^{bLength}} |{compare:^{ratioLength}}|{pMarker:^{pLength}}|".format(key="subtest", aScore=a[unitMarker], bScore=b[unitMarker], nameLength=nameLength, aLength=aLength, bLength=bLength , compare="b / a", ratioLength=ratioLength, pMarker=pValueHeader, pLength=pLength))
for key in sorted(a.keys()):
if key == unitMarker:
continue
aScore = numpy.mean(a[key])
bScore = numpy.mean(b[key])
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = ""
if isSignificant[key]:
significantStr = " (significant)"
strings.append("| {key:{nameLength}}|{aScore:{aLength}} |{bScore:{bLength}} |{compare:{ratioLength}}| {pValue:<{pLength}}|".format(key=key, aScore=displayStr(aScore), bScore=displayStr(bScore), nameLength=nameLength - 1, aLength=aLength, bLength=bLength, ratioLength=ratioLength, compare=displayStr(bScore / aScore), pValue = displayStr(pValue) + significantStr, pLength=pLength - 1))
maxLen = 0
for s in strings:
maxLen = max(maxLen, len(s))
verticalSeparator = "-" * maxLen
strings.insert(0, verticalSeparator)
strings.insert(2, verticalSeparator)
strings.append(verticalSeparator)
print("\n")
for s in strings:
print(s)
print("\n")
def writeCSV(a, b, fileName):
strings = []
result = ""
result += "test_name, {}, {}, b_divided_by_a, pValue, is_significant_using_False_Discovery_Rate\n".format("a_in_" + a[unitMarker], "b_in_" + b[unitMarker])
isSignificant = computeMultipleHypothesesSignificance(a, b)
for key in a.keys():
if key == unitMarker:
continue
aScore = numpy.mean(a[key])
bScore = numpy.mean(b[key])
(tStatistic, pValue) = stats.ttest_ind(a[key], b[key], equal_var=False)
significantStr = "No"
if isSignificant[key]:
significantStr = "Yes"
result += "{},{},{},{},{},{}\n".format(key, displayStr(aScore), displayStr(bScore), displayStr(bScore / aScore), displayStr(pValue), significantStr)
f = open(fileName, "w")
f.write(result)
f.close()
def detectJetStream2(payload):
return "JetStream2.0" in payload
def JetStream2Results(payload):
assert detectJetStream2(payload)
js = payload["JetStream2.0"]
iterations = len(js["tests"]["gaussian-blur"]["metrics"]["Score"]["current"])
results = []
for i in range(iterations):
scores = []
for test in js["tests"].keys():
scores.append(js["tests"][test]["metrics"]["Score"]["current"][i])
geomean = stats.gmean(scores)
results.append(geomean)
return results
def detectSpeedometer2(payload):
return "Speedometer-2" in payload
def Speedometer2Results(payload):
assert detectSpeedometer2(payload)
results = []
for arr in payload["Speedometer-2"]["metrics"]["Score"]["current"]:
results.append(numpy.mean(arr))
return results
def detectPLT5(payload):
if "iterations" not in payload:
return False
iterations = payload["iterations"]
if not isinstance(iterations, list):
return False
if not len(iterations):
return False
if "cold" not in iterations[0]:
return False
if "warm" not in iterations[0]:
return False
if "Geometric" not in iterations[0]:
return False
return True
def PLT5Results(payload):
assert detectPLT5(payload)
results = []
for obj in payload["iterations"]:
results.append(obj["Geometric"])
return results
def detectCompetitivePLT(payload):
return 'add-and-click-link' in payload.get('Safari', {}).get('cold', {})
def CompetitivePLTResults(payload):
def calculate_time_for_run(run):
# We geomean all FMP and load_end times together to produce a result for the run.
fmp_vals = [obj['first_meaningful_paint'] for obj in run.values()]
load_end_vals = [obj['load_end'] for obj in run.values()]
return stats.gmean(fmp_vals + load_end_vals)
safari_results = payload.get('Safari', {})
cold_results = safari_results.get('cold', {})
warm_results = safari_results.get('warm', {})
cold_link_results = cold_results.get('add-and-click-link', {})
warm_link_results = warm_results.get('add-and-click-link', {})
cold_times = [calculate_time_for_run(run) for run in cold_link_results.values()]
warm_times = [calculate_time_for_run(run) for run in warm_link_results.values()]
return [stats.gmean((cold_time, warm_time)) for cold_time, warm_time in zip(cold_times, warm_times)]
def detectPLUM3(payload):
return "PLUM3-PhysFootprint" in payload
def PLUM3Results(payload):
assert detectPLUM3(payload)
breakdown = BenchmarkResults(payload)
return breakdown._results["PLUM3-PhysFootprint"]["metrics"]["Allocations"]["Arithmetic"]["current"]
def detectMotionMark(payload):
return "MotionMark" in payload
def detectMotionMark1_1(payload):
return "MotionMark-1.1" in payload
def detectMotionMark1_1_1(payload):
return "MotionMark-1.1.1" in payload
def motionMarkResults(payload):
assert detectMotionMark(payload) or detectMotionMark1_1(payload) or detectMotionMark1_1_1(payload)
if detectMotionMark(payload):
payload = payload["MotionMark"]
elif detectMotionMark1_1(payload):
payload = payload["MotionMark-1.1"]
else:
payload = payload["MotionMark-1.1.1"]
testNames = list(payload["tests"].keys())
numTests = len(payload["tests"][testNames[0]]["metrics"]["Score"]["current"])
results = []
for i in range(numTests):
scores = []
for test in testNames:
scores.append(payload["tests"][test]["metrics"]["Score"]["current"][i])
results.append(stats.gmean(scores))
return results
def detectBenchmark(payload):
if detectJetStream2(payload):
return JetStream2
if detectSpeedometer2(payload):
return Speedometer2
if detectPLT5(payload):
return PLT5
if detectCompetitivePLT(payload):
return CompetitivePLT
if detectPLUM3(payload):
return PLUM3
if detectMotionMark(payload):
return MotionMark
if detectMotionMark1_1(payload):
return MotionMark1_1
if detectMotionMark1_1_1(payload):
return MotionMark1_1
return None
def biggerIsBetter(benchmarkType):
if benchmarkType == JetStream2:
return True
if benchmarkType == Speedometer2:
return True
if benchmarkType == MotionMark:
return True
if benchmarkType == MotionMark1_1:
return True
if benchmarkType == PLT5:
return False
if benchmarkType == CompetitivePLT:
return False
if benchmarkType == PLUM3:
return False
print("Should not be reached.")
assert False
def ttest(benchmarkType, a, b):
# We use two-tailed Welch's
(tStatistic, pValue) = stats.ttest_ind(a, b, equal_var=False)
aMean = numpy.mean(a)
bMean = numpy.mean(b)
print("a mean = {:.5f}".format(aMean))
print("b mean = {:.5f}".format(bMean))
print("pValue = {:.10f}".format(pValue))
if biggerIsBetter(benchmarkType):
print("(Bigger means are better.)")
if aMean > bMean:
print("{:.3f} times worse".format((aMean / bMean)))
else:
print("{:.3f} times better".format((bMean / aMean)))
else:
print("(Smaller means are better.)")
if aMean > bMean:
print("{:.3f} times better".format((aMean / bMean)))
else:
print("{:.3f} times worse".format((bMean / aMean)))
if pValue <= 0.05:
print("Results ARE significant")
else:
print("Results ARE NOT significant")
def getOptions():
parser = argparse.ArgumentParser(description="Compare two WebKit benchmark results. Pass in at least two JSON result files to compare them. This script prints the pValue along with the magnitude of the change. If more than one JSON is passed as a/b they will be merged when computing the breakdown.")
parser.add_argument("-a",
type=str,
required=True,
nargs='+',
action="append",
help="a JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.")
parser.add_argument("-b",
type=str,
required=True,
nargs='+',
action="append",
help="b JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.")
parser.add_argument("--csv",
type=str,
required=False,
help="Path to write a csv file containing subtest breakdown.")
parser.add_argument("--breakdown", action="store_true",
default=False, help="Print a per subtest breakdown.")
parser.add_argument("--sync-vs-async", action="store_true",
default=False, help="Print a per subtest breakdown in Speedometer2 by sync and async time.")
return parser.parse_known_args()[0]
def main():
args = getOptions()
# Flatten the list of lists of JSON files.
a = itertools.chain.from_iterable(args.a)
b = itertools.chain.from_iterable(args.b)
a = mergeJSONs(list(map(readJSONFile, a)))
b = mergeJSONs(list(map(readJSONFile, b)))
typeA = detectBenchmark(a)
typeB = detectBenchmark(b)
if typeA != typeB:
print("-a and -b are not the same benchmark. a={} b={}".format(typeA, typeB))
sys.exit(1)
if not (typeA and typeB):
print("Unknown benchmark type. a={} b={}".format(typeA, typeB))
sys.exit(1)
if typeA == JetStream2:
if args.breakdown:
dumpBreakdowns(jetStream2Breakdown(a), jetStream2Breakdown(b))
ttest(typeA, JetStream2Results(a), JetStream2Results(b))
if args.csv:
writeCSV(jetStream2Breakdown(a), jetStream2Breakdown(b), args.csv)
elif typeA == Speedometer2:
if args.breakdown:
if args.sync_vs_async:
dumpBreakdowns(speedometer2BreakdownSyncAsync(a), speedometer2BreakdownSyncAsync(b))
else:
dumpBreakdowns(speedometer2Breakdown(a), speedometer2Breakdown(b))
ttest(typeA, Speedometer2Results(a), Speedometer2Results(b))
if args.csv:
writeCSV(speedometer2Breakdown(a), speedometer2Breakdown(b), args.csv)
elif typeA == MotionMark or typeA == MotionMark1_1 or typeA == MotionMark1_1_1:
if args.breakdown:
dumpBreakdowns(motionMarkBreakdown(a), motionMarkBreakdown(b))
ttest(typeA, motionMarkResults(a), motionMarkResults(b))
if args.csv:
writeCSV(motionMarkBreakdown(a), motionMarkBreakdown(b), args.csv)
elif typeA == PLT5:
if args.breakdown:
dumpBreakdowns(plt5Breakdown(a), plt5Breakdown(b))
ttest(typeA, PLT5Results(a), PLT5Results(b))
if args.csv:
writeCSV(plt5Breakdown(a), plt5Breakdown(b), args.csv)
elif typeA == CompetitivePLT:
if args.breakdown:
dumpBreakdowns(competitivePLTBreakdown(a), competitivePLTBreakdown(b))
ttest(typeA, CompetitivePLTResults(a), CompetitivePLTResults(b))
if args.csv:
writeCSV(competitivePLTBreakdown(a), competitivePLTBreakdown(b), args.csv)
elif typeA == PLUM3:
if args.breakdown:
dumpBreakdowns(plum3Breakdown(a), plum3Breakdown(b))
ttest(typeA, PLUM3Results(a), PLUM3Results(b))
if args.csv:
writeCSV(plum3Breakdown(a), plum3Breakdown(b), args.csv)
else:
print("Unknown benchmark type")
sys.exit(1)
if __name__ == "__main__":
main()