| OLD | NEW |
| (Empty) |
| 1 /* | |
| 2 * Copyright (C) 2011 Apple Inc. All rights reserved. | |
| 3 * | |
| 4 * Redistribution and use in source and binary forms, with or without | |
| 5 * modification, are permitted provided that the following conditions | |
| 6 * are met: | |
| 7 * 1. Redistributions of source code must retain the above copyright | |
| 8 * notice, this list of conditions and the following disclaimer. | |
| 9 * 2. Redistributions in binary form must reproduce the above copyright | |
| 10 * notice, this list of conditions and the following disclaimer in the | |
| 11 * documentation and/or other materials provided with the distribution. | |
| 12 * | |
| 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | |
| 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | |
| 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
| 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
| 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
| 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | |
| 23 * THE POSSIBILITY OF SUCH DAMAGE. | |
| 24 */ | |
| 25 | |
| 26 function FlakyLayoutTestDetector() { | |
| 27 this._tests = {}; | |
| 28 this._buildCount = 0; | |
| 29 } | |
| 30 | |
| 31 FlakyLayoutTestDetector.prototype = { | |
| 32 incorporateTestResults: function(buildName, failingTests, tooManyFailures) { | |
| 33 var newFlakyTests = []; | |
| 34 | |
| 35 if (tooManyFailures) { | |
| 36 // Something was going horribly wrong during this test run. We shoul
dn't assume that any | |
| 37 // passes/failures are due to flakiness. | |
| 38 return newFlakyTests; | |
| 39 } | |
| 40 | |
| 41 ++this._buildCount; | |
| 42 | |
| 43 // Record failing tests. | |
| 44 for (var testName in failingTests) { | |
| 45 if (!(testName in this._tests)) { | |
| 46 if (this._buildCount > this._maximumFailOrPassCount) { | |
| 47 // This test hasn't failed in the _maximumFailOrPassCount mos
t recent builds, so | |
| 48 // don't consider it to be flaky. In fact, we don't have to t
rack it at all! | |
| 49 continue; | |
| 50 } | |
| 51 | |
| 52 this._tests[testName] = { | |
| 53 state: this._states.LastSeenFailing, | |
| 54 count: 0, | |
| 55 history: [], | |
| 56 }; | |
| 57 } | |
| 58 | |
| 59 var testData = this._tests[testName]; | |
| 60 testData.history.push({ build: buildName, result: failingTests[testN
ame] }); | |
| 61 | |
| 62 if (testData.state === this._states.LastSeenFailing) { | |
| 63 ++testData.count; | |
| 64 if (testData.count > this._maximumFailOrPassCount) | |
| 65 testData.state = this._states.NotFlaky; | |
| 66 } else if (testData.state === this._states.LastSeenPassing) { | |
| 67 testData.state = this._states.PossiblyFlaky; | |
| 68 newFlakyTests.push(testName); | |
| 69 } | |
| 70 } | |
| 71 | |
| 72 // Record passing tests. | |
| 73 for (var testName in this._tests) { | |
| 74 if (testName in failingTests) | |
| 75 continue; | |
| 76 | |
| 77 var testData = this._tests[testName]; | |
| 78 testData.history.push({ build: buildName, result: { failureType: 'pa
ss' } }); | |
| 79 | |
| 80 if (testData.state === this._states.LastSeenPassing) { | |
| 81 ++testData.count; | |
| 82 if (testData.count > this._maximumFailOrPassCount) | |
| 83 testData.state = this._states.NotFlaky; | |
| 84 } else if (testData.state === this._states.LastSeenFailing) { | |
| 85 testData.state = this._states.LastSeenPassing; | |
| 86 testData.count = 1; | |
| 87 } | |
| 88 } | |
| 89 | |
| 90 return newFlakyTests; | |
| 91 }, | |
| 92 | |
| 93 allFailures: function(testName) { | |
| 94 if (!(testName in this._tests)) | |
| 95 return null; | |
| 96 | |
| 97 return this._tests[testName].history.filter(function(historyItem) { retu
rn historyItem.result.failureType !== 'pass' }); | |
| 98 }, | |
| 99 | |
| 100 get possiblyFlakyTests() { | |
| 101 var self = this; | |
| 102 return Object.keys(self._tests).filter(function(testName) { return self.
_tests[testName].state === self._states.PossiblyFlaky }); | |
| 103 }, | |
| 104 | |
| 105 // If a test has recently failed or passed more than this number of times in
a row we don't | |
| 106 // consider it to be flaky. | |
| 107 _maximumFailOrPassCount: 9, | |
| 108 | |
| 109 _states: { | |
| 110 LastSeenFailing: 0, | |
| 111 LastSeenPassing: 1, | |
| 112 PossiblyFlaky: 2, | |
| 113 NotFlaky: 3, | |
| 114 }, | |
| 115 }; | |
| OLD | NEW |