Group flakiness results by blink revision

Tools/TestResultServer/static-dashboards/flakiness_dashboard.js

 // Copyright (C) 2012 Google Inc. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
@@ skipping 519 matching lines @@
             '/' + buildNumber + '/layout-test-results.zip">layout-test-results.zip</a></li>';
     }
 
     if (!g_history.isLayoutTestResults() && opt_testName && isFailure(builder, opt_testName, index))
         html += '<li>' + linkHTMLToOpenWindow(buildBasePath + pathToFailureLog(opt_testName), 'Failure log') + '</li>';
 
     html += '</ul>';
     ui.popup.show(e.target, html);
 }
 
+function showPopupForInterpolatedResult(e, revision)
+{
+    var html = 'This bot did not run the test for r' + revision + '.';
+    ui.popup.show(e.target, html);
+}
+
 function classNameForFailureString(failure)
 {
     return failure.replace(/(\+|\ )/, '');
 }
 
-function htmlForTestResults(test)
+function htmlForTestResults(test, blinkRevisions)
 {
     var html = '';
     var testResults = test.rawResults.concat();
     var times = test.rawTimes.concat();
     var builder = test.builder;
     var master = builders.master(builder);
     var buildNumbers = g_resultsByBuilder[builder].buildNumbers;
 
+    var cells = [];
+    for (var index = 0; index < blinkRevisions.length; index++)
+        cells.push({revision: blinkRevisions[index]});
+
     var indexToReplaceCurrentResult = -1;
     var indexToReplaceCurrentTime = -1;
     for (var i = 0; i < buildNumbers.length; i++) {
-        var currentResultArray, currentTimeArray, innerHTML, resultString;
+        var currentResultArray, currentTimeArray, currentTime, resultString;
 
         if (i > indexToReplaceCurrentResult) {
             currentResultArray = testResults.shift();
             if (currentResultArray) {
                 resultString = g_resultsByBuilder[builder][results.FAILURE_MAP][currentResultArray[results.RLE.VALUE]];
                 indexToReplaceCurrentResult += currentResultArray[results.RLE.LENGTH];
             } else {
                 resultString = results.NO_DATA;
                 indexToReplaceCurrentResult += buildNumbers.length;
             }
         }
 
         if (i > indexToReplaceCurrentTime) {
             currentTimeArray = times.shift();
-            var currentTime = 0;

[ojan, 2014/07/14 23:40:39]

I think deleting this caused the bug we found earlier today. You're never
resetting currentTime, so it always has the last value encountered.

[pdr., 2014/07/17 14:50:28]

Ack. Fixed.

             if (currentResultArray) {
               currentTime = currentTimeArray[results.RLE.VALUE];
               indexToReplaceCurrentTime += currentTimeArray[results.RLE.LENGTH];
             } else
               indexToReplaceCurrentTime += buildNumbers.length;
-
-            innerHTML = currentTime || ' ';
         }
 
-        html += '<td title="' + resultString + '. Click for more info." class="results ' + classNameForFailureString(resultString) +
-          '" onclick=\'showPopupForBuild(event, "' + builder + '",' + i + ',"' + test.test + '")\'>' + innerHTML;
+        var revision = parseInt(
+            g_resultsByBuilder[builder][results.BLINK_REVISIONS][i], 10);
+
+        // Locate the empty cell corresponding to this blink revision.
+        var cell = undefined;

[ojan, 2014/07/14 23:40:38]

nit: use find and get rid of the comment.

var cell = cells.find(function(item) {
    return item.revision == revision && !item.html;
});

[pdr., 2014/07/17 14:50:28]

Unfortunately, it looks like find isn't available in Chrome unless the
experimental javascript flag is enabled. Postponing until ES6

+        for (var index = 0; index < cells.length; index++) {
+            if (cells[index].revision == revision && !cells[index].html) {
+                cell = cells[index];
+                break;
+            }
+        }
+
+        cell.className = classNameForFailureString(resultString);
+        cell.hasResult =
+            resultString !== results.NO_DATA && resultString !== results.NOTRUN;
+        cell.html = '<td'
+            + ' title="' + resultString + '. Click for more info."'
+            + ' class="results ' + cell.className + '"'
+            + ' onclick=\'showPopupForBuild(event,'
+            + ' "' + builder + '",' + i + ',"' + test.test + '")\'>'
+            + (currentTime || (cell.hasResult ? '&nbsp;' : '?')) + '</td>';

[ojan, 2014/07/14 23:40:38]

As discussed in person, lets show a #bbb question mark for the !hasResult case
as well.

[pdr., 2014/07/17 14:50:28]

Done.

     }
+
+    populateEmptyCells(cells);
+
+    for (var index = 0; index < cells.length; index++)
+        html += cells[index].html;
+
     return html;
 }
 
+// Fill in cells where no test data is available with a question mark.
+// If the next and previous runs were equal, an "interpolated" result is shown.
+function populateEmptyCells(cells)
+{
+    for (var index = 1; index < cells.length; index++) {
+        var prevCell = cells[index - 1];
+        cells[index].nextClassName =
+            prevCell.hasResult ? prevCell.className : prevCell.nextClassName;
+    }
+
+    for (var index = cells.length - 2; index >= 0; --index) {
+        var nextCell = cells[index + 1];
+        cells[index].prevClassName =
+            nextCell.hasResult ? nextCell.className : nextCell.prevClassName;
+    }
+
+    cells.forEach(function(cell) {
+        if (!cell.html) {
+            var interpolateResult =
+                cell.nextClassName && cell.nextClassName == cell.prevClassName;
+            cell.className = interpolateResult ? cell.nextClassName : "NODATA";
+            cell.html = '<td'
+                + ' title="Unknown result. Did not run tests."'
+                + ' onclick=\'showPopupForInterpolatedResult(event,'
+                + ' ' + cell.revision + ')\''
+                + ' class="results interpolatedResults ' + cell.className + '"'
+                + ' >?</td>';
+        }
+    });
+}
+
 function shouldShowTest(testResult)
 {
     if (!g_history.isLayoutTestResults())
         return true;
 
     if (testResult.expectations == 'WONTFIX')
         return g_history.dashboardSpecificState.showWontFix;
 
     if (testResult.expectations == results.SKIP)
         return g_history.dashboardSpecificState.showSkip;
@@ skipping 44 matching lines @@
         var bugHtml;
         if (string.startsWith(bug, 'Bug('))
             bugHtml = bug;
         else
             bugHtml = '<a href="http://' + bug + '">' + bug + '</a>';
         html += '<div>' + bugHtml + '</div>'
     });
     return html;
 }
 
-function htmlForSingleTestRow(test, showBuilderNames)
+function htmlForSingleTestRow(test, showBuilderNames, revisions)
 {
     var headers = tableHeaders();
     var html = '';
     for (var i = 0; i < headers.length; i++) {
         var header = headers[i];
         if (string.startsWith(header, 'test') || string.startsWith(header, 'builder')) {
             var testCellClassName = 'test-link' + (showBuilderNames ? ' builder-name' : '');
             var testCellHTML = showBuilderNames ? test.builder : '<span class="link" onclick="g_history.setQueryParameter(\'tests\',\'' + test.test +'\');">' + test.test + '</span>';
             html += '<tr><td class="' + testCellClassName + '">' + testCellHTML;
         } else if (string.startsWith(header, 'bugs'))
             // FIXME: linkify bugs.
             html += '<td class=options-container>' + (linkifyBugs(test.bugs) || createBugHTML(test));
         else if (string.startsWith(header, 'expectations'))
             html += '<td class=options-container>' + test.expectations;
         else if (string.startsWith(header, 'slowest'))
             html += '<td>' + (test.slowestTime ? test.slowestTime + 's' : '');
         else if (string.startsWith(header, 'flakiness'))
-            html += htmlForTestResults(test);
+            html += htmlForTestResults(test, revisions);
     }
     return html;
 }
 
 function sortColumnFromTableHeader(headerText)
 {
     return headerText.split(' ', 1)[0];
 }
 
 function htmlForTableColumnHeader(headerName, opt_fillColSpan)
@@ skipping 95 matching lines @@
         sortFunctionGetter = numericSort;
         resultsProperty = 'slowestTime';
     } else {
         sortFunctionGetter = alphanumericCompare;
         resultsProperty = column;
     }
 
     tests.sort(sortFunctionGetter(resultsProperty, order == BACKWARD));
 }
 
+// Return an array of the unique blink build revisions across all builders.
+// Each entry will correspond to one column of results. Note that the revisions
+// may not be unique: a builder can test the same blink revision but with
+// different chrome revisions and this will result in multiple entries of a
+// single blink revision in the returned array.
+function blinkBuildRevisions(testResults)

[ojan, 2014/07/14 23:40:39]

As discussed in person, we should only do this for the @ToT Blink group. For the
rest of the groups we should sort by chromium revisions.

[pdr., 2014/07/17 14:50:28]

Done.

+{
+    var revisionsCountedSet = {};
+    for (var resultIndex = 0; resultIndex < testResults.length; resultIndex++) {
+        var builder = testResults[resultIndex].builder;
+        var build = g_resultsByBuilder[builder];
+        var buildNumbers = build.buildNumbers;
+        var builderRevisionsCountedSet = {};
+        for (var i = 0; i < buildNumbers.length; i++) {
+            var blinkRevision = build[results.BLINK_REVISIONS][i];
+            builderRevisionsCountedSet[blinkRevision] = (builderRevisionsCountedSet[blinkRevision] || 0) + 1;
+        }
+
+        // Join the builder's revisions with the total revisions for all builders.
+        for (var revision in builderRevisionsCountedSet)
+            revisionsCountedSet[revision] = Math.max(revisionsCountedSet[revision] || 0, builderRevisionsCountedSet[revision]);
+    }
+
+    var revisionsArray = [];
+    for (var revision in revisionsCountedSet) {
+        for (var i = revisionsCountedSet[revision] - 1; i >= 0; --i)
+            revisionsArray.push(revision);
+    }
+
+    revisionsArray.sort(function(a, b) {
+        return (b - a);
+    });
+    return revisionsArray;
+}
+
 function htmlForIndividualTestOnAllBuilders(test)
 {
     processTestRunsForAllBuilders();
 
     var testResults = g_testToResultsMap[test];
     if (!testResults)
         return '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all recorded runs.</div>';
 
     var html = '';
     var shownBuilders = [];
+    var blinkRevisions = blinkBuildRevisions(testResults);
     for (var j = 0; j < testResults.length; j++) {
         shownBuilders.push(testResults[j].builder);
         var showBuilderNames = true;
-        html += htmlForSingleTestRow(testResults[j], showBuilderNames);
+        html += htmlForSingleTestRow(testResults[j], showBuilderNames, blinkRevisions);
     }
 
     var skippedBuilders = []
     for (builder in currentBuilders()) {
         if (shownBuilders.indexOf(builder) == -1)
             skippedBuilders.push(builder);
     }
 
     var skippedBuildersHtml = '';
     if (skippedBuilders.length) {
@@ skipping 444 matching lines @@
 {
     processTestRunsForBuilder(builderName);
 
     var filteredResults = g_perBuilderFailures[builderName].filter(shouldShowTest);
     sortTests(filteredResults, g_history.dashboardSpecificState.sortColumn, g_history.dashboardSpecificState.sortOrder);
 
     var testsHTML = '';
     if (filteredResults.length) {
         var tableRowsHTML = '';
         var showBuilderNames = false;
+        var blinkRevisions = blinkBuildRevisions(filteredResults);
         for (var i = 0; i < filteredResults.length; i++)
-            tableRowsHTML += htmlForSingleTestRow(filteredResults[i], showBuilderNames)
+            tableRowsHTML += htmlForSingleTestRow(filteredResults[i], showBuilderNames, blinkRevisions)
         testsHTML = htmlForTestTable(tableRowsHTML);
     } else {
         if (g_history.isLayoutTestResults())
             testsHTML += '<div>Fill in one of the text inputs or checkboxes above to show failures.</div>';
         else
             testsHTML += '<div>No tests have failed!</div>';
     }
 
     var html = htmlForNavBar();
 
@@ skipping 104 matching lines @@
         // escape key
         hideLegend();
         ui.popup.hide();
     }
 }, false);
 
 window.addEventListener('load', function() {
     resourceLoader = new loader.Loader();
     resourceLoader.load();
 }, false);