| /* |
| * Copyright (C) 2018 Apple Inc. All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * 1. Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * 2. Redistributions in binary form must reproduce the above copyright |
| * notice, this list of conditions and the following disclaimer in the |
| * documentation and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' |
| * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
| * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS |
| * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF |
| * THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| ResultsDashboard = Utilities.createClass( |
| function(version, options, testData) |
| { |
| this._iterationsSamplers = []; |
| this._options = options; |
| this._results = null; |
| this._version = version; |
| if (testData) { |
| this._iterationsSamplers = testData; |
| this._processData(); |
| } |
| }, { |
| |
| push: function(suitesSamplers) |
| { |
| this._iterationsSamplers.push(suitesSamplers); |
| }, |
| |
| _processData: function() |
| { |
| this._results = {}; |
| this._results[Strings.json.results.iterations] = []; |
| |
| var iterationsScores = []; |
| this._iterationsSamplers.forEach(function(iteration, index) { |
| var testsScores = []; |
| var testsLowerBoundScores = []; |
| var testsUpperBoundScores = []; |
| |
| var result = {}; |
| this._results[Strings.json.results.iterations][index] = result; |
| |
| var suitesResult = {}; |
| result[Strings.json.results.tests] = suitesResult; |
| |
| for (var suiteName in iteration) { |
| var suiteData = iteration[suiteName]; |
| |
| var suiteResult = {}; |
| suitesResult[suiteName] = suiteResult; |
| |
| for (var testName in suiteData) { |
| if (!suiteData[testName][Strings.json.result]) |
| this.calculateScore(suiteData[testName]); |
| |
| suiteResult[testName] = suiteData[testName][Strings.json.result]; |
| delete suiteData[testName][Strings.json.result]; |
| |
| testsScores.push(suiteResult[testName][Strings.json.score]); |
| testsLowerBoundScores.push(suiteResult[testName][Strings.json.scoreLowerBound]); |
| testsUpperBoundScores.push(suiteResult[testName][Strings.json.scoreUpperBound]); |
| } |
| } |
| |
| result[Strings.json.score] = Statistics.geometricMean(testsScores); |
| result[Strings.json.scoreLowerBound] = Statistics.geometricMean(testsLowerBoundScores); |
| result[Strings.json.scoreUpperBound] = Statistics.geometricMean(testsUpperBoundScores); |
| iterationsScores.push(result[Strings.json.score]); |
| }, this); |
| |
| this._results[Strings.json.version] = this._version; |
| this._results[Strings.json.score] = Statistics.sampleMean(iterationsScores.length, iterationsScores.reduce(function(a, b) { return a + b; })); |
| this._results[Strings.json.scoreLowerBound] = this._results[Strings.json.results.iterations][0][Strings.json.scoreLowerBound]; |
| this._results[Strings.json.scoreUpperBound] = this._results[Strings.json.results.iterations][0][Strings.json.scoreUpperBound]; |
| }, |
| |
| calculateScore: function(data) |
| { |
| var result = {}; |
| data[Strings.json.result] = result; |
| var samples = data[Strings.json.samples]; |
| |
| function findRegression(series, profile) { |
| var minIndex = Math.round(.025 * series.length); |
| var maxIndex = Math.round(.975 * (series.length - 1)); |
| var minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity); |
| var maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity); |
| |
| if (Math.abs(maxComplexity - minComplexity) < 20 && maxIndex - minIndex < 20) { |
| minIndex = 0; |
| maxIndex = series.length - 1; |
| minComplexity = series.getFieldInDatum(minIndex, Strings.json.complexity); |
| maxComplexity = series.getFieldInDatum(maxIndex, Strings.json.complexity); |
| } |
| |
| var complexityIndex = series.fieldMap[Strings.json.complexity]; |
| var frameLengthIndex = series.fieldMap[Strings.json.frameLength]; |
| var regressionOptions = { desiredFrameLength: 1000/60 }; |
| if (profile) |
| regressionOptions.preferredProfile = profile; |
| return { |
| minComplexity: minComplexity, |
| maxComplexity: maxComplexity, |
| samples: series.slice(minIndex, maxIndex + 1), |
| regression: new Regression( |
| series.data, |
| function (data, i) { return data[i][complexityIndex]; }, |
| function (data, i) { return data[i][frameLengthIndex]; }, |
| minIndex, maxIndex, regressionOptions) |
| }; |
| } |
| |
| // Convert these samples into SampleData objects if needed |
| [Strings.json.complexity, Strings.json.controller].forEach(function(seriesName) { |
| var series = samples[seriesName]; |
| if (series && !(series instanceof SampleData)) |
| samples[seriesName] = new SampleData(series.fieldMap, series.data); |
| }); |
| |
| var isRampController = this._options["controller"] == "ramp"; |
| var predominantProfile = ""; |
| if (isRampController) { |
| var profiles = {}; |
| data[Strings.json.controller].forEach(function(regression) { |
| if (regression[Strings.json.regressions.profile]) { |
| var profile = regression[Strings.json.regressions.profile]; |
| profiles[profile] = (profiles[profile] || 0) + 1; |
| } |
| }); |
| |
| var maxProfileCount = 0; |
| for (var profile in profiles) { |
| if (profiles[profile] > maxProfileCount) { |
| predominantProfile = profile; |
| maxProfileCount = profiles[profile]; |
| } |
| } |
| } |
| |
| var regressionResult = findRegression(samples[Strings.json.complexity], predominantProfile); |
| var calculation = regressionResult.regression; |
| result[Strings.json.complexity] = {}; |
| result[Strings.json.complexity][Strings.json.regressions.segment1] = [ |
| [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity], |
| [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity] |
| ]; |
| result[Strings.json.complexity][Strings.json.regressions.segment2] = [ |
| [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity], |
| [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity] |
| ]; |
| result[Strings.json.complexity][Strings.json.complexity] = calculation.complexity; |
| result[Strings.json.complexity][Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[Strings.json.complexity].length); |
| |
| if (isRampController) { |
| var timeComplexity = new Experiment; |
| data[Strings.json.controller].forEach(function(regression) { |
| timeComplexity.sample(regression[Strings.json.complexity]); |
| }); |
| |
| var experimentResult = {}; |
| result[Strings.json.controller] = experimentResult; |
| experimentResult[Strings.json.score] = timeComplexity.mean(); |
| experimentResult[Strings.json.measurements.average] = timeComplexity.mean(); |
| experimentResult[Strings.json.measurements.stdev] = timeComplexity.standardDeviation(); |
| experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage(); |
| |
| const bootstrapIterations = 2500; |
| var bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) { |
| var complexityIndex = regressionResult.samples.fieldMap[Strings.json.complexity]; |
| resampleData.sort(function(a, b) { |
| return a[complexityIndex] - b[complexityIndex]; |
| }); |
| |
| var resample = new SampleData(regressionResult.samples.fieldMap, resampleData); |
| var bootstrapRegressionResult = findRegression(resample, predominantProfile); |
| return bootstrapRegressionResult.regression.complexity; |
| }, .8); |
| |
| result[Strings.json.complexity][Strings.json.bootstrap] = bootstrapResult; |
| result[Strings.json.score] = bootstrapResult.median; |
| result[Strings.json.scoreLowerBound] = bootstrapResult.confidenceLow; |
| result[Strings.json.scoreUpperBound] = bootstrapResult.confidenceHigh; |
| } else { |
| var marks = data[Strings.json.marks]; |
| var samplingStartIndex = 0, samplingEndIndex = -1; |
| if (Strings.json.samplingStartTimeOffset in marks) |
| samplingStartIndex = marks[Strings.json.samplingStartTimeOffset].index; |
| if (Strings.json.samplingEndTimeOffset in marks) |
| samplingEndIndex = marks[Strings.json.samplingEndTimeOffset].index; |
| |
| var averageComplexity = new Experiment; |
| var averageFrameLength = new Experiment; |
| var controllerSamples = samples[Strings.json.controller]; |
| controllerSamples.forEach(function (sample, i) { |
| if (i >= samplingStartIndex && (samplingEndIndex == -1 || i < samplingEndIndex)) { |
| averageComplexity.sample(controllerSamples.getFieldInDatum(sample, Strings.json.complexity)); |
| var smoothedFrameLength = controllerSamples.getFieldInDatum(sample, Strings.json.smoothedFrameLength); |
| if (smoothedFrameLength && smoothedFrameLength != -1) |
| averageFrameLength.sample(smoothedFrameLength); |
| } |
| }); |
| |
| var experimentResult = {}; |
| result[Strings.json.controller] = experimentResult; |
| experimentResult[Strings.json.measurements.average] = averageComplexity.mean(); |
| experimentResult[Strings.json.measurements.concern] = averageComplexity.concern(Experiment.defaults.CONCERN); |
| experimentResult[Strings.json.measurements.stdev] = averageComplexity.standardDeviation(); |
| experimentResult[Strings.json.measurements.percent] = averageComplexity.percentage(); |
| |
| experimentResult = {}; |
| result[Strings.json.frameLength] = experimentResult; |
| experimentResult[Strings.json.measurements.average] = 1000 / averageFrameLength.mean(); |
| experimentResult[Strings.json.measurements.concern] = averageFrameLength.concern(Experiment.defaults.CONCERN); |
| experimentResult[Strings.json.measurements.stdev] = averageFrameLength.standardDeviation(); |
| experimentResult[Strings.json.measurements.percent] = averageFrameLength.percentage(); |
| |
| result[Strings.json.score] = averageComplexity.score(Experiment.defaults.CONCERN); |
| result[Strings.json.scoreLowerBound] = result[Strings.json.score] - averageFrameLength.standardDeviation(); |
| result[Strings.json.scoreUpperBound] = result[Strings.json.score] + averageFrameLength.standardDeviation(); |
| } |
| }, |
| |
| get data() |
| { |
| return this._iterationsSamplers; |
| }, |
| |
| get results() |
| { |
| if (this._results) |
| return this._results[Strings.json.results.iterations]; |
| this._processData(); |
| return this._results[Strings.json.results.iterations]; |
| }, |
| |
| get options() |
| { |
| return this._options; |
| }, |
| |
| get version() |
| { |
| return this._version; |
| }, |
| |
| _getResultsProperty: function(property) |
| { |
| if (this._results) |
| return this._results[property]; |
| this._processData(); |
| return this._results[property]; |
| }, |
| |
| get score() |
| { |
| return this._getResultsProperty(Strings.json.score); |
| }, |
| |
| get scoreLowerBound() |
| { |
| return this._getResultsProperty(Strings.json.scoreLowerBound); |
| }, |
| |
| get scoreUpperBound() |
| { |
| return this._getResultsProperty(Strings.json.scoreUpperBound); |
| } |
| }); |
| |
| ResultsTable = Utilities.createClass( |
| function(element, headers) |
| { |
| this.element = element; |
| this._headers = headers; |
| |
| this._flattenedHeaders = []; |
| this._headers.forEach(function(header) { |
| if (header.disabled) |
| return; |
| |
| if (header.children) |
| this._flattenedHeaders = this._flattenedHeaders.concat(header.children); |
| else |
| this._flattenedHeaders.push(header); |
| }, this); |
| |
| this._flattenedHeaders = this._flattenedHeaders.filter(function (header) { |
| return !header.disabled; |
| }); |
| |
| this.clear(); |
| }, { |
| |
| clear: function() |
| { |
| this.element.textContent = ""; |
| }, |
| |
| _addHeader: function() |
| { |
| var thead = Utilities.createElement("thead", {}, this.element); |
| var row = Utilities.createElement("tr", {}, thead); |
| |
| this._headers.forEach(function (header) { |
| if (header.disabled) |
| return; |
| |
| var th = Utilities.createElement("th", {}, row); |
| if (header.title != Strings.text.graph) |
| th.innerHTML = header.title; |
| if (header.children) |
| th.colSpan = header.children.length; |
| }); |
| }, |
| |
| _addBody: function() |
| { |
| this.tbody = Utilities.createElement("tbody", {}, this.element); |
| }, |
| |
| _addEmptyRow: function() |
| { |
| var row = Utilities.createElement("tr", {}, this.tbody); |
| this._flattenedHeaders.forEach(function (header) { |
| return Utilities.createElement("td", { class: "suites-separator" }, row); |
| }); |
| }, |
| |
| _addTest: function(testName, testResult, options) |
| { |
| var row = Utilities.createElement("tr", {}, this.tbody); |
| |
| this._flattenedHeaders.forEach(function (header) { |
| var td = Utilities.createElement("td", {}, row); |
| if (header.text == Strings.text.testName) { |
| td.textContent = testName; |
| } else if (typeof header.text == "string") { |
| var data = testResult[header.text]; |
| if (typeof data == "number") |
| data = data.toFixed(2); |
| td.innerHTML = data; |
| } else |
| td.innerHTML = header.text(testResult); |
| }, this); |
| }, |
| |
| _addIteration: function(iterationResult, iterationData, options) |
| { |
| var testsResults = iterationResult[Strings.json.results.tests]; |
| for (var suiteName in testsResults) { |
| this._addEmptyRow(); |
| var suiteResult = testsResults[suiteName]; |
| var suiteData = iterationData[suiteName]; |
| for (var testName in suiteResult) |
| this._addTest(testName, suiteResult[testName], options, suiteData[testName]); |
| } |
| }, |
| |
| showIterations: function(dashboard) |
| { |
| this.clear(); |
| this._addHeader(); |
| this._addBody(); |
| |
| var iterationsResults = dashboard.results; |
| iterationsResults.forEach(function(iterationResult, index) { |
| this._addIteration(iterationResult, dashboard.data[index], dashboard.options); |
| }, this); |
| } |
| }); |
| |
| window.benchmarkRunnerClient = { |
| iterationCount: 1, |
| options: null, |
| results: null, |
| |
| initialize: function(suites, options) |
| { |
| this.options = options; |
| }, |
| |
| willStartFirstIteration: function() |
| { |
| this.results = new ResultsDashboard(Strings.version, this.options); |
| }, |
| |
| didRunSuites: function(suitesSamplers) |
| { |
| this.results.push(suitesSamplers); |
| }, |
| |
| didRunTest: function(testData) |
| { |
| this.results.calculateScore(testData); |
| }, |
| |
| didFinishLastIteration: function() |
| { |
| benchmarkController.showResults(); |
| } |
| }; |
| |
| window.sectionsManager = |
| { |
| showSection: function(sectionIdentifier, pushState) |
| { |
| var sections = document.querySelectorAll("main > section"); |
| for (var i = 0; i < sections.length; ++i) { |
| document.body.classList.remove("showing-" + sections[i].id); |
| } |
| document.body.classList.add("showing-" + sectionIdentifier); |
| |
| var currentSectionElement = document.querySelector("section.selected"); |
| console.assert(currentSectionElement); |
| |
| var newSectionElement = document.getElementById(sectionIdentifier); |
| console.assert(newSectionElement); |
| |
| currentSectionElement.classList.remove("selected"); |
| newSectionElement.classList.add("selected"); |
| |
| if (pushState) |
| history.pushState({section: sectionIdentifier}, document.title); |
| }, |
| |
| setSectionVersion: function(sectionIdentifier, version) |
| { |
| document.querySelector("#" + sectionIdentifier + " .version").textContent = version; |
| }, |
| |
| setSectionScore: function(sectionIdentifier, score, confidence) |
| { |
| document.querySelector("#" + sectionIdentifier + " .score").textContent = score; |
| if (confidence) |
| document.querySelector("#" + sectionIdentifier + " .confidence").textContent = confidence; |
| }, |
| |
| populateTable: function(tableIdentifier, headers, dashboard) |
| { |
| var table = new ResultsTable(document.getElementById(tableIdentifier), headers); |
| table.showIterations(dashboard); |
| } |
| }; |
| |
| window.benchmarkController = { |
| initialize: function() |
| { |
| document.title = Strings.text.title.replace("%s", Strings.version); |
| document.querySelectorAll(".version").forEach(function(e) { |
| e.textContent = Strings.version; |
| }); |
| benchmarkController.addOrientationListenerIfNecessary(); |
| }, |
| |
| determineCanvasSize: function() { |
| var match = window.matchMedia("(max-device-width: 760px)"); |
| if (match.matches) { |
| document.body.classList.add("small"); |
| return; |
| } |
| |
| match = window.matchMedia("(max-device-width: 1600px)"); |
| if (match.matches) { |
| document.body.classList.add("medium"); |
| return; |
| } |
| |
| match = window.matchMedia("(max-width: 1600px)"); |
| if (match.matches) { |
| document.body.classList.add("medium"); |
| return; |
| } |
| |
| document.body.classList.add("large"); |
| }, |
| |
| addOrientationListenerIfNecessary: function() { |
| if (!("orientation" in window)) |
| return; |
| |
| this.orientationQuery = window.matchMedia("(orientation: landscape)"); |
| this._orientationChanged(this.orientationQuery); |
| this.orientationQuery.addListener(this._orientationChanged); |
| }, |
| |
| _orientationChanged: function(match) |
| { |
| benchmarkController.isInLandscapeOrientation = match.matches; |
| if (match.matches) |
| document.querySelector(".start-benchmark p").classList.add("hidden"); |
| else |
| document.querySelector(".start-benchmark p").classList.remove("hidden"); |
| benchmarkController.updateStartButtonState(); |
| }, |
| |
| updateStartButtonState: function() |
| { |
| document.getElementById("run-benchmark").disabled = !this.isInLandscapeOrientation; |
| }, |
| |
| _startBenchmark: function(suites, options, frameContainerID) |
| { |
| benchmarkController.determineCanvasSize(); |
| |
| var configuration = document.body.className.match(/small|medium|large/); |
| if (configuration) |
| options[Strings.json.configuration] = configuration[0]; |
| |
| benchmarkRunnerClient.initialize(suites, options); |
| var frameContainer = document.getElementById(frameContainerID); |
| var runner = new BenchmarkRunner(suites, frameContainer, benchmarkRunnerClient); |
| runner.runMultipleIterations(); |
| |
| sectionsManager.showSection("test-container"); |
| }, |
| |
| startBenchmark: function() |
| { |
| var options = { |
| "test-interval": 30, |
| "display": "minimal", |
| "tiles": "big", |
| "controller": "ramp", |
| "kalman-process-error": 1, |
| "kalman-measurement-error": 4, |
| "time-measurement": "performance" |
| }; |
| this._startBenchmark(Suites, options, "test-container"); |
| }, |
| |
| showResults: function() |
| { |
| if (!this.addedKeyEvent) { |
| document.addEventListener("keypress", this.handleKeyPress, false); |
| this.addedKeyEvent = true; |
| } |
| |
| var dashboard = benchmarkRunnerClient.results; |
| var score = dashboard.score; |
| var confidence = "±" + (Statistics.largestDeviationPercentage(dashboard.scoreLowerBound, score, dashboard.scoreUpperBound) * 100).toFixed(2) + "%"; |
| sectionsManager.setSectionVersion("results", dashboard.version); |
| sectionsManager.setSectionScore("results", score.toFixed(2), confidence); |
| sectionsManager.populateTable("results-header", Headers.testName, dashboard); |
| sectionsManager.populateTable("results-score", Headers.score, dashboard); |
| sectionsManager.populateTable("results-data", Headers.details, dashboard); |
| sectionsManager.showSection("results", true); |
| }, |
| |
| handleKeyPress: function(event) |
| { |
| switch (event.charCode) |
| { |
| case 27: // esc |
| benchmarkController.hideDebugInfo(); |
| break; |
| case 106: // j |
| benchmarkController.showDebugInfo(); |
| break; |
| case 115: // s |
| benchmarkController.selectResults(event.target); |
| break; |
| } |
| }, |
| |
| hideDebugInfo: function() |
| { |
| var overlay = document.getElementById("overlay"); |
| if (!overlay) |
| return; |
| document.body.removeChild(overlay); |
| }, |
| |
| showDebugInfo: function() |
| { |
| if (document.getElementById("overlay")) |
| return; |
| |
| var overlay = Utilities.createElement("div", { |
| id: "overlay" |
| }, document.body); |
| var container = Utilities.createElement("div", {}, overlay); |
| |
| var header = Utilities.createElement("h3", {}, container); |
| header.textContent = "Debug Output"; |
| |
| var data = Utilities.createElement("div", {}, container); |
| data.textContent = "Please wait..."; |
| setTimeout(function() { |
| var output = { |
| version: benchmarkRunnerClient.results.version, |
| options: benchmarkRunnerClient.results.options, |
| data: benchmarkRunnerClient.results.data |
| }; |
| data.textContent = JSON.stringify(output, function(key, value) { |
| if (typeof value === 'number') |
| return Utilities.toFixedNumber(value, 3); |
| return value; |
| }, 1); |
| }, 0); |
| data.onclick = function() { |
| var selection = window.getSelection(); |
| selection.removeAllRanges(); |
| var range = document.createRange(); |
| range.selectNode(data); |
| selection.addRange(range); |
| }; |
| |
| var button = Utilities.createElement("button", {}, container); |
| button.textContent = "Done"; |
| button.onclick = function() { |
| benchmarkController.hideDebugInfo(); |
| }; |
| }, |
| |
| selectResults: function(target) |
| { |
| target.selectRange = ((target.selectRange || 0) + 1) % 3; |
| |
| var selection = window.getSelection(); |
| selection.removeAllRanges(); |
| var range = document.createRange(); |
| switch (target.selectRange) { |
| case 0: { |
| range.selectNode(document.getElementById("results-score")); |
| break; |
| } |
| case 1: { |
| range.setStart(document.querySelector("#results .score"), 0); |
| range.setEndAfter(document.querySelector("#results-score"), 0); |
| break; |
| } |
| case 2: { |
| range.selectNodeContents(document.querySelector("#results .score")); |
| break; |
| } |
| } |
| selection.addRange(range); |
| } |
| }; |
| |
| window.addEventListener("load", function() { benchmarkController.initialize(); }); |
