Index: tools/telemetry/support/html_output/results-template.html |
diff --git a/tools/telemetry/support/html_output/results-template.html b/tools/telemetry/support/html_output/results-template.html |
new file mode 100644 |
index 0000000000000000000000000000000000000000..b0b84c7f7f134c9313c512c72533c5a8370bd94f |
--- /dev/null |
+++ b/tools/telemetry/support/html_output/results-template.html |
@@ -0,0 +1,596 @@ |
+<!DOCTYPE html> |
+<html> |
+<head> |
+<title>Telemetry Performance Test Results</title> |
+<style type="text/css"> |
+ |
+section { |
+ background: white; |
+ padding: 10px; |
+ position: relative; |
+} |
+ |
+.time-plots { |
+ padding-left: 25px; |
+} |
+ |
+.time-plots > div { |
+ display: inline-block; |
+ width: 90px; |
+ height: 40px; |
+ margin-right: 10px; |
+} |
+ |
+section h1 { |
+ text-align: center; |
+ font-size: 1em; |
+} |
+ |
+section .tooltip { |
+ position: absolute; |
+ text-align: center; |
+ background: #ffcc66; |
+ border-radius: 5px; |
+ padding: 0px 5px; |
+} |
+ |
+body { |
+ padding: 0px; |
+ margin: 0px; |
+ font-family: sans-serif; |
+} |
+ |
+table { |
+ background: white; |
+ width: 100%; |
+} |
+ |
+table, td, th { |
+ border-collapse: collapse; |
+ padding: 5px; |
+ white-space: nowrap; |
+} |
+ |
+tr.even { |
+ background: #f6f6f6; |
+} |
+ |
+table td { |
+ position: relative; |
+ font-family: monospace; |
+} |
+ |
+th, td { |
+ cursor: pointer; |
+ cursor: hand; |
+} |
+ |
+th { |
+ background: #e6eeee; |
+ background: -webkit-gradient(linear, left top, left bottom, from(rgb(244, 244, 244)), to(rgb(217, 217, 217))); |
+ border: 1px solid #ccc; |
+} |
+ |
+th:after { |
+ content: ' \25B8'; |
+} |
+ |
+th.headerSortUp:after { |
+ content: ' \25BE'; |
+} |
+ |
+th.headerSortDown:after { |
+ content: ' \25B4'; |
+} |
+ |
+td.comparison, td.result { |
+ text-align: right; |
+} |
+ |
+td.better { |
+ color: #6c6; |
+} |
+ |
+td.worse { |
+ color: #c66; |
+} |
+ |
+td.missing { |
+ text-align: center; |
+} |
+ |
+.checkbox { |
+ display: inline-block; |
+ background: #eee; |
+ background: -webkit-gradient(linear, left bottom, left top, from(rgb(220, 220, 220)), to(rgb(200, 200, 200))); |
+ border: inset 1px #ddd; |
+ border-radius: 5px; |
+ margin: 10px; |
+ font-size: small; |
+ cursor: pointer; |
+ cursor: hand; |
+ -webkit-user-select: none; |
+ font-weight: bold; |
+} |
+ |
+.checkbox span { |
+ display: inline-block; |
+ line-height: 100%; |
+ padding: 5px 8px; |
+ border: outset 1px transparent; |
+} |
+ |
+.checkbox .checked { |
+ background: #e6eeee; |
+ background: -webkit-gradient(linear, left top, left bottom, from(rgb(255, 255, 255)), to(rgb(235, 235, 235))); |
+ border: outset 1px #eee; |
+ border-radius: 5px; |
+} |
+ |
+</style> |
+</head> |
+<body onload="init()"> |
+<div style="padding: 0 10px; white-space: nowrap;"> |
+Result <span id="time-memory" class="checkbox"><span class="checked">Time</span><span>Memory</span></span> |
+Reference <span id="reference" class="checkbox"></span> |
+Run Telemetry with --reset-html-results to clear all runs |
+</div> |
+<table id="container"></table> |
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script> |
+<script> |
+%plugins% |
+</script> |
+<script> |
+function TestResult(metric, values, associatedRun) { |
+ if (values[0] instanceof Array) { |
+ var flattenedValues = []; |
+ for (var i = 0; i < values.length; i++) |
+ flattenedValues = flattenedValues.concat(values[i]); |
+ values = flattenedValues; |
+ } |
+ |
+ this.test = function () { return metric; } |
+ this.values = function () { return values.map(function (value) { return metric.scalingFactor() * value; }); } |
+ this.unscaledMean = function () { return Statistics.sum(values) / values.length; } |
+ this.mean = function () { return metric.scalingFactor() * this.unscaledMean(); } |
+ this.min = function () { return metric.scalingFactor() * Statistics.min(values); } |
+ this.max = function () { return metric.scalingFactor() * Statistics.max(values); } |
+ this.confidenceIntervalDelta = function () { |
+ return metric.scalingFactor() * Statistics.confidenceIntervalDelta(0.95, values.length, |
+ Statistics.sum(values), Statistics.squareSum(values)); |
+ } |
+ this.confidenceIntervalDeltaRatio = function () { return this.confidenceIntervalDelta() / this.mean(); } |
+ this.percentDifference = function(other) { return (other.unscaledMean() - this.unscaledMean()) / this.unscaledMean(); } |
+ this.isStatisticallySignificant = function (other) { |
+ var diff = Math.abs(other.mean() - this.mean()); |
+ return diff > this.confidenceIntervalDelta() && diff > other.confidenceIntervalDelta(); |
+ } |
+ this.run = function () { return associatedRun; } |
+} |
+ |
+function TestRun(entry) { |
+ this.description = function () { return entry['description']; } |
+ this.revision = function () { return entry['revision']; } |
+ this.label = function () { |
+ var label = 'r' + this.revision(); |
+ if (this.description()) |
+ label += ' ‐ ' + this.description(); |
+ return label; |
+ } |
+} |
+ |
+function PerfTestMetric(name, metric, unit, isImportant) { |
+ var testResults = []; |
+ var cachedUnit = null; |
+ var cachedScalingFactor = null; |
+ |
+ // We can't do this in TestResult because all results for each test need to share the same unit and the same scaling factor. |
+ function computeScalingFactorIfNeeded() { |
+ // FIXME: We shouldn't be adjusting units on every test result. |
+ // We can only do this on the first test. |
+ if (!testResults.length || cachedUnit) |
+ return; |
+ |
+ var mean = testResults[0].unscaledMean(); // FIXME: We should look at all values. |
+ var kilo = unit == 'bytes' ? 1024 : 1000; |
+ if (mean > 10 * kilo * kilo && unit != 'ms') { |
+ cachedScalingFactor = 1 / kilo / kilo; |
+ cachedUnit = 'M ' + unit; |
+ } else if (mean > 10 * kilo) { |
+ cachedScalingFactor = 1 / kilo; |
+ cachedUnit = unit == 'ms' ? 's' : ('K ' + unit); |
+ } else { |
+ cachedScalingFactor = 1; |
+ cachedUnit = unit; |
+ } |
+ } |
+ |
+ this.name = function () { return name + ':' + metric; } |
+ this.isImportant = isImportant; |
+ this.isMemoryTest = function () { |
+ return (unit == 'kb' || |
+ unit == 'KB' || |
+ unit == 'MB' || |
+ unit == 'bytes'); |
+ } |
+ this.addResult = function (newResult) { |
+ testResults.push(newResult); |
+ cachedUnit = null; |
+ cachedScalingFactor = null; |
+ } |
+ this.results = function () { return testResults; } |
+ this.scalingFactor = function() { |
+ computeScalingFactorIfNeeded(); |
+ return cachedScalingFactor; |
+ } |
+ this.unit = function () { |
+ computeScalingFactorIfNeeded(); |
+ return cachedUnit; |
+ } |
+ this.biggerIsBetter = function () { |
+ if (window.unitToBiggerIsBetter == undefined) { |
+ window.unitToBiggerIsBetter = {}; |
+ var units = JSON.parse(document.getElementById('units-json').textContent); |
+ for (var unit in units) { |
+ if (units[unit].improvement_direction == 'up') { |
+ window.unitToBiggerIsBetter[unit] = true; |
+ } |
+ } |
+ } |
+ return window.unitToBiggerIsBetter[unit]; |
+ } |
+} |
+ |
+var plotColor = 'rgb(230,50,50)'; |
+var subpointsPlotOptions = { |
+ lines: {show:true, lineWidth: 0}, |
+ color: plotColor, |
+ points: {show: true, radius: 1}, |
+ bars: {show: false}}; |
+ |
+var mainPlotOptions = { |
+ xaxis: { |
+ min: -0.5, |
+ tickSize: 1, |
+ }, |
+ crosshair: { mode: 'y' }, |
+ series: { shadowSize: 0 }, |
+ bars: {show: true, align: 'center', barWidth: 0.5}, |
+ lines: { show: false }, |
+ points: { show: true }, |
+ grid: { |
+ borderWidth: 1, |
+ borderColor: '#ccc', |
+ backgroundColor: '#fff', |
+ hoverable: true, |
+ autoHighlight: false, |
+ } |
+}; |
+ |
+var timePlotOptions = { |
+ yaxis: { show: false }, |
+ xaxis: { show: false }, |
+ lines: { show: true }, |
+ grid: { borderWidth: 1, borderColor: '#ccc' }, |
+ colors: [ plotColor ] |
+}; |
+ |
+function createPlot(container, test) { |
+ var section = $('<section><div class="plot"></div><div class="time-plots"></div>' |
+ + '<span class="tooltip"></span></section>'); |
+ section.children('.plot').css({'width': (100 * test.results().length + 25) + 'px', 'height': '300px'}); |
+ $(container).append(section); |
+ |
+ var plotContainer = section.children('.plot'); |
+ var minIsZero = true; |
+ attachPlot(test, plotContainer, minIsZero); |
+ |
+ attachTimePlots(test, section.children('.time-plots')); |
+ |
+ var tooltip = section.children('.tooltip'); |
+ plotContainer.bind('plothover', function (event, position, item) { |
+ if (item) { |
+ var postfix = item.series.id ? ' (' + item.series.id + ')' : ''; |
+ tooltip.html(item.datapoint[1].toPrecision(4) + postfix); |
+ var sectionOffset = $(section).offset(); |
+ tooltip.css({left: item.pageX - sectionOffset.left - tooltip.outerWidth() / 2, top: item.pageY - sectionOffset.top + 10}); |
+ tooltip.fadeIn(200); |
+ } else |
+ tooltip.hide(); |
+ }); |
+ plotContainer.mouseout(function () { |
+ tooltip.hide(); |
+ }); |
+ plotContainer.click(function (event) { |
+ event.preventDefault(); |
+ minIsZero = !minIsZero; |
+ attachPlot(test, plotContainer, minIsZero); |
+ }); |
+ |
+ return section; |
+} |
+ |
+function attachTimePlots(test, container) { |
+ var results = test.results(); |
+ var attachedPlot = false; |
+ for (var i = 0; i < results.length; i++) { |
+ container.append('<div></div>'); |
+ var values = results[i].values(); |
+ if (!values) |
+ continue; |
+ attachedPlot = true; |
+ |
+ $.plot(container.children().last(), [values.map(function (value, index) { return [index, value]; })], |
+ $.extend(true, {}, timePlotOptions, {yaxis: {min: Math.min.apply(Math, values) * 0.9, max: Math.max.apply(Math, values) * 1.1}, |
+ xaxis: {min: -0.5, max: values.length - 0.5}})); |
+ } |
+ if (!attachedPlot) |
+ container.children().remove(); |
+} |
+ |
+function attachPlot(test, plotContainer, minIsZero) { |
+ var results = test.results(); |
+ |
+ var values = results.reduce(function (values, result, index) { |
+ var newValues = result.values(); |
+ return newValues ? values.concat(newValues.map(function (value) { return [index, value]; })) : values; |
+ }, []); |
+ |
+ var plotData = [$.extend(true, {}, subpointsPlotOptions, {data: values})]; |
+ plotData.push({id: 'μ', data: results.map(function (result, index) { return [index, result.mean()]; }), color: plotColor}); |
+ |
+ var overallMax = Statistics.max(results.map(function (result, index) { return result.max(); })); |
+ var overallMin = Statistics.min(results.map(function (result, index) { return result.min(); })); |
+ var margin = (overallMax - overallMin) * 0.1; |
+ var currentPlotOptions = $.extend(true, {}, mainPlotOptions, {yaxis: { |
+ min: minIsZero ? 0 : overallMin - margin, |
+ max: minIsZero ? overallMax * 1.1 : overallMax + margin}}); |
+ |
+ currentPlotOptions.xaxis.max = results.length - 0.5; |
+ currentPlotOptions.xaxis.ticks = results.map(function (result, index) { return [index, result.run().label()]; }); |
+ |
+ $.plot(plotContainer, plotData, currentPlotOptions); |
+} |
+ |
+function toFixedWidthPrecision(value) { |
+ var decimal = value.toFixed(2); |
+ return decimal; |
+} |
+ |
+function formatPercentage(fraction) { |
+ var percentage = fraction * 100; |
+ return (fraction * 100).toFixed(2) + '%'; |
+} |
+ |
+function createTable(tests, runs, shouldIgnoreMemory, referenceIndex) { |
+ $('#container').html('<thead><tr><th>Test</th><th>Unit</th>' + runs.map(function (run, index) { |
+ return '<th colspan="' + (index == referenceIndex ? 2 : 3) + '" class="{sorter: \'comparison\'}">' + run.label() + '</th>'; |
+ }).reduce(function (markup, cell) { return markup + cell; }, '') + '</tr></head><tbody></tbody>'); |
+ |
+ var testNames = []; |
+ for (testName in tests) |
+ testNames.push(testName); |
+ |
+ testNames.sort().map(function (testName) { |
+ var test = tests[testName]; |
+ if (test.isMemoryTest() != shouldIgnoreMemory) |
+ createTableRow(runs, test, referenceIndex); |
+ }); |
+ |
+ $('#container').tablesorter({widgets: ['zebra']}); |
+} |
+ |
+function linearRegression(points) { |
+ // Implement http://www.easycalculation.com/statistics/learn-correlation.php. |
+ // x = magnitude |
+ // y = iterations |
+ var sumX = 0; |
+ var sumY = 0; |
+ var sumXSquared = 0; |
+ var sumYSquared = 0; |
+ var sumXTimesY = 0; |
+ |
+ for (var i = 0; i < points.length; i++) { |
+ var x = i; |
+ var y = points[i]; |
+ sumX += x; |
+ sumY += y; |
+ sumXSquared += x * x; |
+ sumYSquared += y * y; |
+ sumXTimesY += x * y; |
+ } |
+ |
+ var r = (points.length * sumXTimesY - sumX * sumY) / |
+ Math.sqrt((points.length * sumXSquared - sumX * sumX) * |
+ (points.length * sumYSquared - sumY * sumY)); |
+ |
+ if (isNaN(r) || r == Math.Infinity) |
+ r = 0; |
+ |
+ var slope = (points.length * sumXTimesY - sumX * sumY) / (points.length * sumXSquared - sumX * sumX); |
+ var intercept = sumY / points.length - slope * sumX / points.length; |
+ return {slope: slope, intercept: intercept, rSquared: r * r}; |
+} |
+ |
+var warningSign = '<svg viewBox="0 0 100 100" style="width: 18px; height: 18px; vertical-align: bottom;" version="1.1">' |
+ + '<polygon fill="red" points="50,10 90,80 10,80 50,10" stroke="red" stroke-width="10" stroke-linejoin="round" />' |
+ + '<polygon fill="white" points="47,30 48,29, 50, 28.7, 52,29 53,30 50,60" stroke="white" stroke-width="10" stroke-linejoin="round" />' |
+ + '<circle cx="50" cy="73" r="6" fill="white" />' |
+ + '</svg>'; |
+ |
+function createTableRow(runs, test, referenceIndex) { |
+ var tableRow = $('<tr><td class="test"' + (test.isImportant ? ' style="font-weight:bold"' : '') + '>' + test.name() + '</td><td class="unit">' + test.unit() + '</td></tr>'); |
+ |
+ function markupForRun(result, referenceResult) { |
+ var comparisonCell = ''; |
+ var hiddenValue = ''; |
+ var shouldCompare = result !== referenceResult; |
+ if (shouldCompare && referenceResult) { |
+ var percentDifference = referenceResult.percentDifference(result); |
+ var better = test.biggerIsBetter() ? percentDifference > 0 : percentDifference < 0; |
+ var comparison = ''; |
+ var className = 'comparison'; |
+ if (referenceResult.isStatisticallySignificant(result)) { |
+ comparison = formatPercentage(Math.abs(percentDifference)) + (better ? ' Better' : ' Worse '); |
+ className += better ? ' better' : ' worse'; |
+ } |
+ hiddenValue = '<span style="display: none">|' + comparison + '</span>'; |
+ comparisonCell = '<td class="' + className + '">' + comparison + '</td>'; |
+ } else if (shouldCompare) |
+ comparisonCell = '<td class="comparison"></td>'; |
+ |
+ var values = result.values(); |
+ var warning = ''; |
+ var regressionAnalysis = ''; |
+ if (values && values.length > 3) { |
+ regressionResult = linearRegression(values); |
+ regressionAnalysis = 'slope=' + toFixedWidthPrecision(regressionResult.slope) |
+ + ', R^2=' + toFixedWidthPrecision(regressionResult.rSquared); |
+ if (regressionResult.rSquared > 0.6 && Math.abs(regressionResult.slope) > 0.01) { |
+ warning = ' <span class="regression-warning" title="Detected a time dependency with ' + regressionAnalysis + '">' + warningSign + ' </span>'; |
+ } |
+ } |
+ |
+ var statistics = 'σ=' + toFixedWidthPrecision(result.confidenceIntervalDelta()) + ', min=' + toFixedWidthPrecision(result.min()) |
+ + ', max=' + toFixedWidthPrecision(result.max()) + '\n' + regressionAnalysis; |
+ |
+ // Tablesorter doesn't know about the second cell so put the comparison in the invisible element. |
+ return '<td class="result" title="' + statistics + '">' + toFixedWidthPrecision(result.mean()) + hiddenValue |
+ + '</td><td class="confidenceIntervalDelta" title="' + statistics + '">± ' |
+ + formatPercentage(result.confidenceIntervalDeltaRatio()) + warning + '</td>' + comparisonCell; |
+ } |
+ |
+ function markupForMissingRun(isReference) { |
+ return '<td colspan="' + (isReference ? 2 : 3) + '" class="missing">Missing</td>'; |
+ } |
+ |
+ var runIndex = 0; |
+ var results = test.results(); |
+ var referenceResult = undefined; |
+ var resultIndexMap = {}; |
+ for (var i = 0; i < results.length; i++) { |
+ while (runs[runIndex] !== results[i].run()) |
+ runIndex++; |
+ if (runIndex == referenceIndex) |
+ referenceResult = results[i]; |
+ resultIndexMap[runIndex] = i; |
+ } |
+ for (var i = 0; i < runs.length; i++) { |
+ var resultIndex = resultIndexMap[i]; |
+ if (resultIndex == undefined) |
+ tableRow.append(markupForMissingRun(i == referenceIndex)); |
+ else |
+ tableRow.append(markupForRun(results[resultIndex], referenceResult)); |
+ } |
+ |
+ $('#container').children('tbody').last().append(tableRow); |
+ |
+ function toggle() { |
+ var firstCell = tableRow.children('td').first(); |
+ if (firstCell.children('section').length) { |
+ firstCell.children('section').remove(); |
+ tableRow.children('td').css({'padding-bottom': ''}); |
+ } else { |
+ var plot = createPlot(firstCell, test); |
+ plot.css({'position': 'absolute', 'z-index': 2}); |
+ var offset = tableRow.offset(); |
+ offset.left += 1; |
+ offset.top += tableRow.outerHeight(); |
+ plot.offset(offset); |
+ tableRow.children('td').css({'padding-bottom': plot.outerHeight() + 5}); |
+ } |
+ |
+ return false; |
+ }; |
+ |
+ tableRow.click(function(event) { |
+ if (event.target != tableRow[0] && event.target.parentNode != tableRow[0]) |
+ return; |
+ |
+ event.preventDefault(); |
+ |
+ toggle(); |
+ }); |
+ |
+ if (test.isImportant) { |
+ toggle(); |
+ } |
+} |
+ |
+function init() { |
+ $.tablesorter.addParser({ |
+ id: 'comparison', |
+ is: function(s) { |
+ return s.indexOf('|') >= 0; |
+ }, |
+ format: function(s) { |
+ var parsed = parseFloat(s.substring(s.indexOf('|') + 1)); |
+ return isNaN(parsed) ? 0 : parsed; |
+ }, |
+ type: 'numeric', |
+ }); |
+ |
+ var runs = []; |
+ var metrics = {}; |
+ $.each(JSON.parse(document.getElementById('results-json').textContent), function (index, entry) { |
+ var run = new TestRun(entry); |
+ runs.push(run); |
+ |
+ function addTests(tests, parentFullName) { |
+ for (var testName in tests) { |
+ var fullTestName = parentFullName + '/' + testName; |
+ var rawMetrics = tests[testName].metrics; |
+ |
+ for (var metricName in rawMetrics) { |
+ var fullMetricName = fullTestName + ':' + metricName; |
+ var metric = metrics[fullMetricName]; |
+ if (!metric) { |
+ metric = new PerfTestMetric(fullTestName, metricName, rawMetrics[metricName].units, rawMetrics[metricName].important); |
+ metrics[fullMetricName] = metric; |
+ } |
+ metric.addResult(new TestResult(metric, rawMetrics[metricName].current, run)); |
+ } |
+ |
+ if (tests[testName].tests) |
+ addTests(tests[testName].tests, fullTestName); |
+ } |
+ } |
+ |
+ addTests(entry.tests, ''); |
+ }); |
+ |
+ var shouldIgnoreMemory= true; |
+ var referenceIndex = 0; |
+ |
+ createTable(metrics, runs, shouldIgnoreMemory, referenceIndex); |
+ |
+ $('#time-memory').bind('change', function (event, checkedElement) { |
+ shouldIgnoreMemory = checkedElement.textContent == 'Time'; |
+ createTable(metrics, runs, shouldIgnoreMemory, referenceIndex); |
+ }); |
+ |
+ runs.map(function (run, index) { |
+ $('#reference').append('<span value="' + index + '"' + (index == referenceIndex ? ' class="checked"' : '') + '>' + run.label() + '</span>'); |
+ }) |
+ |
+ $('#reference').bind('change', function (event, checkedElement) { |
+ referenceIndex = parseInt(checkedElement.getAttribute('value')); |
+ createTable(metrics, runs, shouldIgnoreMemory, referenceIndex); |
+ }); |
+ |
+ $('.checkbox').each(function (index, checkbox) { |
+ $(checkbox).children('span').click(function (event) { |
+ if ($(this).hasClass('checked')) |
+ return; |
+ $(checkbox).children('span').removeClass('checked'); |
+ $(this).addClass('checked'); |
+ $(checkbox).trigger('change', $(this)); |
+ }); |
+ }); |
+} |
+ |
+</script> |
+<script id="results-json" type="application/json">%json_results%</script> |
+<script id="units-json" type="application/json">%json_units%</script> |
+</body> |
+</html> |