| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (C) 2011 Apple Inc. All rights reserved. | |
| 3 * | |
| 4 * Redistribution and use in source and binary forms, with or without | |
| 5 * modification, are permitted provided that the following conditions | |
| 6 * are met: | |
| 7 * 1. Redistributions of source code must retain the above copyright | |
| 8 * notice, this list of conditions and the following disclaimer. | |
| 9 * 2. Redistributions in binary form must reproduce the above copyright | |
| 10 * notice, this list of conditions and the following disclaimer in the | |
| 11 * documentation and/or other materials provided with the distribution. | |
| 12 * | |
| 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | |
| 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | |
| 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
| 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
| 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
| 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | |
| 23 * THE POSSIBILITY OF SUCH DAMAGE. | |
| 24 */ | |
| 25 | |
| 26 function LayoutTestHistoryAnalyzer(builder) { | |
| 27 this._builder = builder; | |
| 28 this._flakinessDetector = new FlakyLayoutTestDetector(); | |
| 29 this._history = {}; | |
| 30 this._loader = new LayoutTestResultsLoader(builder); | |
| 31 this._testRunsSinceLastInterestingChange = 0; | |
| 32 } | |
| 33 | |
| 34 LayoutTestHistoryAnalyzer.prototype = { | |
| 35 /* | |
| 36 * Periodically calls callback until all current failures have been explaine
d. Callback is | |
| 37 * passed an object like the following: | |
| 38 * { | |
| 39 * 'history': { | |
| 40 * 'r12347 (681)': { | |
| 41 * 'tooManyFailures': false, | |
| 42 * 'tests': { | |
| 43 * 'css1/basic/class_as_selector2.html': 'fail', | |
| 44 * }, | |
| 45 * }, | |
| 46 * 'r12346 (680)': { | |
| 47 * 'tooManyFailures': false, | |
| 48 * 'tests': {}, | |
| 49 * }, | |
| 50 * 'r12345 (679)': { | |
| 51 * 'tooManyFailures': false, | |
| 52 * 'tests': { | |
| 53 * 'css1/basic/class_as_selector.html': 'crash', | |
| 54 * }, | |
| 55 * }, | |
| 56 * }, | |
| 57 * 'possiblyFlaky': { | |
| 58 * 'fast/workers/worker-test.html': [ | |
| 59 * { 'build': 'r12344 (678)', 'result': 'fail' }, | |
| 60 * { 'build': 'r12340 (676)', 'result': 'crash' }, | |
| 61 * ], | |
| 62 * }, | |
| 63 * } | |
| 64 * Each build contains just the failures that a) are still occurring on the
bots, and b) were new | |
| 65 * in that build. | |
| 66 */ | |
| 67 start: function(callback) { | |
| 68 var self = this; | |
| 69 self._builder.getBuildNames(function(buildNames) { | |
| 70 self._analyzeBuilds(buildNames, callback, function() { | |
| 71 self._builder.getOldBuildNames(function(oldBuildNames) { | |
| 72 self._analyzeBuilds(oldBuildNames, callback); | |
| 73 }); | |
| 74 }); | |
| 75 }); | |
| 76 }, | |
| 77 | |
| 78 _analyzeBuilds: function(buildNames, callback, analyzedAllBuildsCallback) { | |
| 79 var self = this; | |
| 80 function inner(buildIndex) { | |
| 81 self._incorporateBuildHistory(buildNames, buildIndex, function(callA
gain) { | |
| 82 var data = { | |
| 83 history: self._history, | |
| 84 possiblyFlaky: {}, | |
| 85 }; | |
| 86 self._flakinessDetector.possiblyFlakyTests.forEach(function(test
Name) { | |
| 87 data.possiblyFlaky[testName] = self._flakinessDetector.allFa
ilures(testName); | |
| 88 }); | |
| 89 | |
| 90 var nextIndex = buildIndex + 1; | |
| 91 var analyzedAllBuilds = nextIndex >= buildNames.length; | |
| 92 var haveMoreDataToFetch = !analyzedAllBuilds || analyzedAllBuild
sCallback; | |
| 93 | |
| 94 var callbackRequestedStop = !callback(data, haveMoreDataToFetch)
; | |
| 95 if (callbackRequestedStop) | |
| 96 return; | |
| 97 | |
| 98 if (!callAgain) | |
| 99 return; | |
| 100 | |
| 101 if (analyzedAllBuilds) { | |
| 102 if (analyzedAllBuildsCallback) | |
| 103 analyzedAllBuildsCallback(); | |
| 104 return; | |
| 105 } | |
| 106 | |
| 107 setTimeout(function() { inner(nextIndex) }, 0); | |
| 108 }); | |
| 109 } | |
| 110 inner(0); | |
| 111 }, | |
| 112 | |
| 113 _incorporateBuildHistory: function(buildNames, buildIndex, callback) { | |
| 114 var previousBuildName = Object.keys(this._history).last(); | |
| 115 var nextBuildName = buildNames[buildIndex]; | |
| 116 | |
| 117 var self = this; | |
| 118 self._loader.start(nextBuildName, function(tests, tooManyFailures) { | |
| 119 if (tooManyFailures) { | |
| 120 var firstBuildName = Object.keys(self._history)[0]; | |
| 121 // If the first (i.e., current or most recent) build exited earl
y due to too many | |
| 122 // failures, we want to process other too-many-failures builds n
ormally to try to | |
| 123 // figure out when the too-many-failures started occurring. If t
he first/current | |
| 124 // build did not exit due to too many failures, then too-many-fa
ilures builds will | |
| 125 // only confuse our analysis (since they run a semi-arbitrary su
bset of tests), so | |
| 126 // we should just skip them entirely. | |
| 127 if (firstBuildName && !self._history[firstBuildName].tooManyFail
ures) { | |
| 128 callback(true); | |
| 129 return; | |
| 130 } | |
| 131 } | |
| 132 | |
| 133 ++self._testRunsSinceLastInterestingChange; | |
| 134 | |
| 135 var historyItem = { | |
| 136 tooManyFailures: tooManyFailures, | |
| 137 tests: {}, | |
| 138 }; | |
| 139 self._history[nextBuildName] = historyItem; | |
| 140 | |
| 141 var previousHistoryItem; | |
| 142 if (previousBuildName) | |
| 143 previousHistoryItem = self._history[previousBuildName]; | |
| 144 | |
| 145 var newFlakyTests = self._flakinessDetector.incorporateTestResults(n
extBuildName, tests, tooManyFailures); | |
| 146 if (newFlakyTests.length) { | |
| 147 self._testRunsSinceLastInterestingChange = 0; | |
| 148 // Remove all possibly flaky tests from the failure history, sin
ce when they failed | |
| 149 // is no longer meaningful. | |
| 150 newFlakyTests.forEach(function(testName) { | |
| 151 for (var buildName in self._history) | |
| 152 delete self._history[buildName].tests[testName]; | |
| 153 }); | |
| 154 } | |
| 155 | |
| 156 for (var testName in tests) { | |
| 157 if (previousHistoryItem) { | |
| 158 if (!(testName in previousHistoryItem.tests)) | |
| 159 continue; | |
| 160 delete previousHistoryItem.tests[testName]; | |
| 161 } | |
| 162 historyItem.tests[testName] = tests[testName]; | |
| 163 } | |
| 164 | |
| 165 var previousUnexplainedFailuresCount = previousBuildName ? Object.ke
ys(self._history[previousBuildName].tests).length : 0; | |
| 166 var unexplainedFailuresCount = Object.keys(self._history[nextBuildNa
me].tests).length; | |
| 167 | |
| 168 if (previousUnexplainedFailuresCount && !unexplainedFailuresCount) | |
| 169 self._testRunsSinceLastInterestingChange = 0; | |
| 170 | |
| 171 const minimumRequiredTestRunsWithoutInterestingChanges = 5; | |
| 172 callback(unexplainedFailuresCount || self._testRunsSinceLastInterest
ingChange < minimumRequiredTestRunsWithoutInterestingChanges); | |
| 173 }, | |
| 174 function(tests) { | |
| 175 // Some tests failed, but we couldn't fetch results.html (perhaps be
cause the test | |
| 176 // run aborted early for some reason). Just skip this build entirely
. | |
| 177 callback(true); | |
| 178 }); | |
| 179 }, | |
| 180 }; | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 13712005:
Move GardeningServer out of BuildSlaveSupport (Closed)
Base URL: svn://svn.chromium.org/blink/trunk