| OLD | NEW |
|---|
| 1 <!DOCTYPE html> | 1 <!DOCTYPE html> |
| 2 <html> | 2 <html> |
| 3 <head> | 3 <head> |
| 4 <title>Telemetry Performance Test Results</title> | 4 <title>Telemetry Performance Test Results</title> |
| 5 <style type="text/css"> | 5 <style type="text/css"> |
| 6 | 6 |
| 7 section { | 7 section { |
| 8 background: white; | 8 background: white; |
| 9 padding: 10px; | 9 padding: 10px; |
| 10 position: relative; | 10 position: relative; |
| (...skipping 205 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 216 } | 216 } |
| 217 this.confidenceIntervalDeltaRatio = function () { return this.confidenceInte
rvalDelta() / this.mean(); } | 217 this.confidenceIntervalDeltaRatio = function () { return this.confidenceInte
rvalDelta() / this.mean(); } |
| 218 this.percentDifference = function(other) { return (other.unscaledMean() - th
is.unscaledMean()) / this.unscaledMean(); } | 218 this.percentDifference = function(other) { return (other.unscaledMean() - th
is.unscaledMean()) / this.unscaledMean(); } |
| 219 this.isStatisticallySignificant = function (other) { | 219 this.isStatisticallySignificant = function (other) { |
| 220 var diff = Math.abs(other.mean() - this.mean()); | 220 var diff = Math.abs(other.mean() - this.mean()); |
| 221 return diff > this.confidenceIntervalDelta() && diff > other.confidenceI
ntervalDelta(); | 221 return diff > this.confidenceIntervalDelta() && diff > other.confidenceI
ntervalDelta(); |
| 222 } | 222 } |
| 223 this.run = function () { return associatedRun; } | 223 this.run = function () { return associatedRun; } |
| 224 } | 224 } |
| 225 | 225 |
| 226 // A comparison statistic is the fractional change between the reference result |
| 227 // and the comparion result. |
| 228 function TestComparisonResult(metric, referenceResult, comparisonResult, associa
tedRun) { |
| 229 var stddev = function(result) { |
| 230 values = result.values(); |
| 231 return Statistics.sampleStandardDeviation( |
| 232 values.length, Statistics.sum(values), Statistics.squareSum(values))
; |
| 233 } |
| 234 var ReferenceStddev = stddev(referenceResult); |
| 235 var compareStddev = stddev(comparisonResult); |
| 236 |
| 237 var meanCompare = |
| 238 ((comparisonResult.unscaledMean() - referenceResult.unscaledMean()) / |
| 239 referenceResult.unscaledMean()); |
| 240 // Formuls is |(comp - ref)/ref| = |comp/ref - 1| |
| 241 // SD(comp/ref - 1) = SD(comp/ref) |
| 242 // If R = Y/X, SD(R)/R = SD(Y)/Y + SD(X)/X |
| 243 var stddevCompare = (comparisonResult.unscaledMean() / referenceResult.unsca
ledMean()) * |
| 244 (compareStddev / comparisonResult.unscaledMean() + |
| 245 ReferenceStddev / referenceResult.unscaledMean()); |
| 246 |
| 247 this.test = function () { return metric; } |
| 248 this.values = function () { return null; } |
| 249 this.unscaledMean = function () { return meanCompare; } |
| 250 this.mean = function () { return metric.scalingFactor() * this.unscaledMean(
); } |
| 251 this.min = function () { return null; } |
| 252 this.max = function () { return null; } |
| 253 this.confidenceIntervalDelta = function () { |
| 254 return metric.scalingFactor() * 2.0 * stddevCompare; |
| 255 } |
| 256 this.confidenceIntervalDeltaRatio = function () { return this.confidenceInte
rvalDelta() / this.mean(); } |
| 257 this.percentDifference = function(other) { return (other.unscaledMean() - th
is.unscaledMean()) / this.unscaledMean(); } |
| 258 this.isStatisticallySignificant = function (other) { |
| 259 var diff = Math.abs(other.mean() - this.mean()); |
| 260 return diff > this.confidenceIntervalDelta() && diff > other.confidenceI
ntervalDelta(); |
| 261 } |
| 262 this.run = function () { return associatedRun; } |
| 263 } |
| 264 |
| 226 function TestRun(entry) { | 265 function TestRun(entry) { |
| 227 this.id = function() { return entry['buildTime']; } | 266 this.id = function() { return entry['buildTime']; } |
| 228 this.revision = function () { return entry['revision']; } | 267 this.revision = function () { return entry['revision']; } |
| 229 this.label = function () { | 268 this.label = function () { |
| 230 if (labelKey in localStorage) | 269 if (labelKey in localStorage) |
| 231 return localStorage[labelKey]; | 270 return localStorage[labelKey]; |
| 232 if (entry['label']) | 271 if (entry['label']) |
| 233 return entry['label']; | 272 return entry['label']; |
| 234 return 'r' + this.revision(); | 273 return 'r' + this.revision(); |
| 235 } | 274 } |
| (...skipping 27 matching lines...) Expand all Loading... |
| 263 return; | 302 return; |
| 264 | 303 |
| 265 var mean = testResults[0].unscaledMean(); // FIXME: We should look at al
l values. | 304 var mean = testResults[0].unscaledMean(); // FIXME: We should look at al
l values. |
| 266 var kilo = unit == 'bytes' ? 1024 : 1000; | 305 var kilo = unit == 'bytes' ? 1024 : 1000; |
| 267 if (mean > 10 * kilo * kilo && unit != 'ms') { | 306 if (mean > 10 * kilo * kilo && unit != 'ms') { |
| 268 cachedScalingFactor = 1 / kilo / kilo; | 307 cachedScalingFactor = 1 / kilo / kilo; |
| 269 cachedUnit = 'M ' + unit; | 308 cachedUnit = 'M ' + unit; |
| 270 } else if (mean > 10 * kilo) { | 309 } else if (mean > 10 * kilo) { |
| 271 cachedScalingFactor = 1 / kilo; | 310 cachedScalingFactor = 1 / kilo; |
| 272 cachedUnit = unit == 'ms' ? 's' : ('K ' + unit); | 311 cachedUnit = unit == 'ms' ? 's' : ('K ' + unit); |
| 312 } else if (unit == 'fraction') { |
| 313 cachedScalingFactor = 100; |
| 314 cachedUnit = 'percent'; |
| 273 } else { | 315 } else { |
| 274 cachedScalingFactor = 1; | 316 cachedScalingFactor = 1; |
| 275 cachedUnit = unit; | 317 cachedUnit = unit; |
| 276 } | 318 } |
| 277 } | 319 } |
| 278 | 320 |
| 279 this.name = function () { return name + ':' + metric; } | 321 this.name = function () { return name + ':' + metric; } |
| 280 this.isImportant = isImportant; | 322 this.isImportant = isImportant; |
| 281 this.isMemoryTest = function () { | 323 this.isMemoryTest = function () { |
| 282 return (unit == 'kb' || | 324 return (unit == 'kb' || |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 341 } | 383 } |
| 342 var undeleteManager = new UndeleteManager(); | 384 var undeleteManager = new UndeleteManager(); |
| 343 | 385 |
| 344 var plotColor = 'rgb(230,50,50)'; | 386 var plotColor = 'rgb(230,50,50)'; |
| 345 var subpointsPlotOptions = { | 387 var subpointsPlotOptions = { |
| 346 lines: {show:true, lineWidth: 0}, | 388 lines: {show:true, lineWidth: 0}, |
| 347 color: plotColor, | 389 color: plotColor, |
| 348 points: {show: true, radius: 1}, | 390 points: {show: true, radius: 1}, |
| 349 bars: {show: false}}; | 391 bars: {show: false}}; |
| 350 | 392 |
| 393 var comparisonPlotOptions = { |
| 394 color: plotColor, |
| 395 lines: {show:false}, |
| 396 points: { |
| 397 show: true, |
| 398 radius: 1, |
| 399 errorbars: "y", |
| 400 yerr: {show: true, upperCap: "-", lowerCap: "-", radius:5} |
| 401 }, |
| 402 bars: { show: false} |
| 403 }; |
| 404 |
| 351 var mainPlotOptions = { | 405 var mainPlotOptions = { |
| 352 xaxis: { | 406 xaxis: { |
| 353 min: -0.5, | 407 min: -0.5, |
| 354 tickSize: 1, | 408 tickSize: 1, |
| 355 }, | 409 }, |
| 356 crosshair: { mode: 'y' }, | 410 crosshair: { mode: 'y' }, |
| 357 series: { shadowSize: 0 }, | 411 series: { shadowSize: 0 }, |
| 358 bars: {show: true, align: 'center', barWidth: 0.5}, | 412 bars: {show: true, align: 'center', barWidth: 0.5}, |
| 359 lines: { show: false }, | 413 lines: { show: false }, |
| 360 points: { show: true }, | 414 points: { show: true }, |
| (...skipping 111 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 472 var results = test.results(); | 526 var results = test.results(); |
| 473 var attachedPlot = false; | 527 var attachedPlot = false; |
| 474 for (var i = 0; i < results.length; i++) { | 528 for (var i = 0; i < results.length; i++) { |
| 475 container.append('<div>' + results[i].run().label() + '</div>'); | 529 container.append('<div>' + results[i].run().label() + '</div>'); |
| 476 } | 530 } |
| 477 } | 531 } |
| 478 | 532 |
| 479 function attachPlot(test, plotContainer, minIsZero) { | 533 function attachPlot(test, plotContainer, minIsZero) { |
| 480 var results = test.results(); | 534 var results = test.results(); |
| 481 | 535 |
| 536 // Actual values |
| 482 var values = results.reduce(function (values, result, index) { | 537 var values = results.reduce(function (values, result, index) { |
| 483 var newValues = result.values(); | 538 var newValues = result.values(); |
| 484 return newValues ? values.concat(newValues.map(function (value) { return
[index, value]; })) : values; | 539 return newValues ? values.concat(newValues.map(function (value) { return
[index, value]; })) : values; |
| 485 }, []); | 540 }, []); |
| 486 | 541 |
| 487 var plotData = [$.extend(true, {}, subpointsPlotOptions, {data: values})]; | 542 var plotData = [$.extend(true, {}, subpointsPlotOptions, {data: values})]; |
| 488 plotData.push({id: 'μ', data: results.map(function (result, index) { retu
rn [index, result.mean()]; }), color: plotColor}); | |
| 489 | 543 |
| 490 var overallMax = Statistics.max(results.map(function (result, index) { retur
n result.max(); })); | 544 // Means for actual values. |
| 491 var overallMin = Statistics.min(results.map(function (result, index) { retur
n result.min(); })); | 545 plotData.push({ |
| 546 » id: 'μ', |
| 547 » data: results.reduce(function (values, result, index) { |
| 548 » if (result.values()) { |
| 549 » » values.push([index, result.mean()]); |
| 550 » } |
| 551 » return values; |
| 552 » }, []), |
| 553 » color: plotColor}); |
| 554 |
| 555 // Comparison values with bars. |
| 556 var comparison_values = results.reduce(function(pointList, result, index) { |
| 557 » return result.values() ? pointList : pointList.concat( |
| 558 » [[index, result.mean(), result.confidenceIntervalDelta()],]); |
| 559 }, []); |
| 560 plotData.push($.extend(true, {}, comparisonPlotOptions, {data: comparison_va
lues})); |
| 561 |
| 562 var overallMax = Statistics.max(results.map(function (result, index) { |
| 563 » return result.values() ? result.max() : result.mean() + result.confidenc
eIntervalDelta(); |
| 564 })); |
| 565 var overallMin = Statistics.min(results.map(function (result, index) { |
| 566 » return result.values() ? result.min() : result.mean() - result.confidenc
eIntervalDelta(); |
| 567 })); |
| 568 // For minIsZero == true, percents are shown from 0:100. Otherwise we |
| 569 // scale |
| 570 if (minIsZero && test.unit() == 'percent') { |
| 571 » overallMax = 100; |
| 572 } |
| 573 |
| 492 var margin = (overallMax - overallMin) * 0.1; | 574 var margin = (overallMax - overallMin) * 0.1; |
| 493 var currentPlotOptions = $.extend(true, {}, mainPlotOptions, {yaxis: { | 575 var currentPlotOptions = $.extend(true, {}, mainPlotOptions, {yaxis: { |
| 494 min: minIsZero ? 0 : overallMin - margin, | 576 min: minIsZero ? 0 : overallMin - margin, |
| 495 max: minIsZero ? overallMax * 1.1 : overallMax + margin}}); | 577 max: minIsZero ? overallMax * 1.1 : overallMax + margin}}); |
| 496 | 578 |
| 497 currentPlotOptions.xaxis.max = results.length - 0.5; | 579 currentPlotOptions.xaxis.max = results.length - 0.5; |
| 498 currentPlotOptions.xaxis.ticks = results.map(function (result, index) { retu
rn [index, result.run().label()]; }); | 580 currentPlotOptions.xaxis.ticks = results.map(function (result, index) { retu
rn [index, result.run().label()]; }); |
| 499 | 581 |
| 500 $.plot(plotContainer, plotData, currentPlotOptions); | 582 $.plot(plotContainer, plotData, currentPlotOptions); |
| 501 } | 583 } |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 633 var regressionAnalysis = ''; | 715 var regressionAnalysis = ''; |
| 634 if (values && values.length > 3) { | 716 if (values && values.length > 3) { |
| 635 regressionResult = linearRegression(values); | 717 regressionResult = linearRegression(values); |
| 636 regressionAnalysis = 'slope=' + toFixedWidthPrecision(regressionResu
lt.slope) | 718 regressionAnalysis = 'slope=' + toFixedWidthPrecision(regressionResu
lt.slope) |
| 637 + ', R^2=' + toFixedWidthPrecision(regressionResult.rSquared); | 719 + ', R^2=' + toFixedWidthPrecision(regressionResult.rSquared); |
| 638 if (regressionResult.rSquared > 0.6 && Math.abs(regressionResult.slo
pe) > 0.01) { | 720 if (regressionResult.rSquared > 0.6 && Math.abs(regressionResult.slo
pe) > 0.01) { |
| 639 warning = ' <span class="regression-warning" title="Detected a t
ime dependency with ' + regressionAnalysis + '">' + warningSign + ' </span>'; | 721 warning = ' <span class="regression-warning" title="Detected a t
ime dependency with ' + regressionAnalysis + '">' + warningSign + ' </span>'; |
| 640 } | 722 } |
| 641 } | 723 } |
| 642 | 724 |
| 643 var statistics = 'σ=' + toFixedWidthPrecision(result.confidenceInt
ervalDelta()) + ', min=' + toFixedWidthPrecision(result.min()) | 725 var statistics = 'σ=' + toFixedWidthPrecision(result.confidenceInt
ervalDelta()); |
| 644 + ', max=' + toFixedWidthPrecision(result.max()) + '\n' + regression
Analysis; | 726 » if (result.min()) |
| 727 » statistics += ', min=' + toFixedWidthPrecision(result.min()); |
| 728 » if (result.max()) |
| 729 » statistics += ', max=' + toFixedWidthPrecision(result.max()) |
| 730 » statistics += '\n' + regressionAnalysis; |
| 645 | 731 |
| 646 // Tablesorter doesn't know about the second cell so put the comparison
in the invisible element. | 732 // Tablesorter doesn't know about the second cell so put the comparison
in the invisible element. |
| 647 return '<td class="result" title="' + statistics + '">' + toFixedWidthPr
ecision(result.mean()) + hiddenValue | 733 return '<td class="result" title="' + statistics + '">' + toFixedWidthPr
ecision(result.mean()) + hiddenValue |
| 648 + '</td><td class="confidenceIntervalDelta" title="' + statistics +
'">± ' | 734 + '</td><td class="confidenceIntervalDelta" title="' + statistics +
'">± ' |
| 649 + formatPercentage(result.confidenceIntervalDeltaRatio()) + warning
+ '</td>' + comparisonCell; | 735 + formatPercentage(result.confidenceIntervalDeltaRatio()) + warning
+ '</td>' + comparisonCell; |
| 650 } | 736 } |
| 651 | 737 |
| 652 function markupForMissingRun(isReference) { | 738 function markupForMissingRun(isReference) { |
| 653 return '<td colspan="' + (isReference ? 2 : 3) + '" class="missing">Miss
ing</td>'; | 739 return '<td colspan="' + (isReference ? 2 : 3) + '" class="missing">Miss
ing</td>'; |
| 654 } | 740 } |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 733 deletedRunsById[run.id()] = run; | 819 deletedRunsById[run.id()] = run; |
| 734 return; | 820 return; |
| 735 } | 821 } |
| 736 | 822 |
| 737 runs.push(run); | 823 runs.push(run); |
| 738 | 824 |
| 739 function addTests(tests) { | 825 function addTests(tests) { |
| 740 for (var testName in tests) { | 826 for (var testName in tests) { |
| 741 var rawMetrics = tests[testName].metrics; | 827 var rawMetrics = tests[testName].metrics; |
| 742 | 828 |
| 829 var baseResults = {}; |
| 743 for (var metricName in rawMetrics) { | 830 for (var metricName in rawMetrics) { |
| 744 var fullMetricName = testName + ':' + metricName; | 831 var fullMetricName = testName + ':' + metricName; |
| 745 var metric = metrics[fullMetricName]; | 832 var metric = metrics[fullMetricName]; |
| 746 if (!metric) { | 833 if (!metric) { |
| 747 metric = new PerfTestMetric(testName, metricName, rawMet
rics[metricName].units, rawMetrics[metricName].important); | 834 metric = new PerfTestMetric(testName, metricName, rawMet
rics[metricName].units, rawMetrics[metricName].important); |
| 748 metrics[fullMetricName] = metric; | 835 metrics[fullMetricName] = metric; |
| 749 } | 836 } |
| 750 metric.addResult(new TestResult(metric, rawMetrics[metricNam
e].current, run)); | 837 if ('current' in rawMetrics[metricName]) { |
| 838 // Regular metric; add the result now. |
| 839 » » » var result = new TestResult(metric, rawMetrics[metricNam
e].current, run); |
| 840 » » » baseResults[metricName] = result; |
| 841 metric.addResult(result); |
| 842 } |
| 843 } |
| 844 // Handle comparison metrics |
| 845 for (var metricName in rawMetrics) { |
| 846 var fullMetricName = testName + ':' + metricName; |
| 847 |
| 848 if ('current' in rawMetrics[metricName]) |
| 849 continue; |
| 850 |
| 851 referenceResult = baseResults[rawMetrics[metricName].referen
ce_statistic]; |
| 852 comparisonResult = baseResults[rawMetrics[metricName].compar
ison_statistic]; |
| 853 » » if (!referenceResult || !comparisonResult) continue; |
| 854 |
| 855 var metric = metrics[fullMetricName]; |
| 856 metric.addResult(new TestComparisonResult(metric, referenceR
esult, comparisonResult, run)); |
| 751 } | 857 } |
| 752 } | 858 } |
| 753 } | 859 } |
| 754 | 860 |
| 755 addTests(entry.tests); | 861 addTests(entry.tests); |
| 756 }); | 862 }); |
| 757 | 863 |
| 758 var useLargeLinePlots = false; | 864 var useLargeLinePlots = false; |
| 759 var shouldIgnoreMemory= true; | 865 var shouldIgnoreMemory= true; |
| 760 var referenceIndex = 0; | 866 var referenceIndex = 0; |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 801 } else { | 907 } else { |
| 802 $('#undelete').hide(); | 908 $('#undelete').hide(); |
| 803 } | 909 } |
| 804 } | 910 } |
| 805 | 911 |
| 806 </script> | 912 </script> |
| 807 <script id="results-json" type="application/json">%json_results%</script> | 913 <script id="results-json" type="application/json">%json_results%</script> |
| 808 <script id="units-json" type="application/json">%json_units%</script> | 914 <script id="units-json" type="application/json">%json_units%</script> |
| 809 </body> | 915 </body> |
| 810 </html> | 916 </html> |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 23431036:
Create and display "comparison statistics" to compare two statistics
Base URL: svn://svn.chromium.org/chrome/trunk/src