OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (C) 2011 Apple Inc. All rights reserved. | |
3 * | |
4 * Redistribution and use in source and binary forms, with or without | |
5 * modification, are permitted provided that the following conditions | |
6 * are met: | |
7 * 1. Redistributions of source code must retain the above copyright | |
8 * notice, this list of conditions and the following disclaimer. | |
9 * 2. Redistributions in binary form must reproduce the above copyright | |
10 * notice, this list of conditions and the following disclaimer in the | |
11 * documentation and/or other materials provided with the distribution. | |
12 * | |
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | |
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | |
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | |
23 * THE POSSIBILITY OF SUCH DAMAGE. | |
24 */ | |
25 | |
26 function FlakyLayoutTestDetector() { | |
27 this._tests = {}; | |
28 this._buildCount = 0; | |
29 } | |
30 | |
31 FlakyLayoutTestDetector.prototype = { | |
32 incorporateTestResults: function(buildName, failingTests, tooManyFailures) { | |
33 var newFlakyTests = []; | |
34 | |
35 if (tooManyFailures) { | |
36 // Something was going horribly wrong during this test run. We shoul
dn't assume that any | |
37 // passes/failures are due to flakiness. | |
38 return newFlakyTests; | |
39 } | |
40 | |
41 ++this._buildCount; | |
42 | |
43 // Record failing tests. | |
44 for (var testName in failingTests) { | |
45 if (!(testName in this._tests)) { | |
46 if (this._buildCount > this._maximumFailOrPassCount) { | |
47 // This test hasn't failed in the _maximumFailOrPassCount mos
t recent builds, so | |
48 // don't consider it to be flaky. In fact, we don't have to t
rack it at all! | |
49 continue; | |
50 } | |
51 | |
52 this._tests[testName] = { | |
53 state: this._states.LastSeenFailing, | |
54 count: 0, | |
55 history: [], | |
56 }; | |
57 } | |
58 | |
59 var testData = this._tests[testName]; | |
60 testData.history.push({ build: buildName, result: failingTests[testN
ame] }); | |
61 | |
62 if (testData.state === this._states.LastSeenFailing) { | |
63 ++testData.count; | |
64 if (testData.count > this._maximumFailOrPassCount) | |
65 testData.state = this._states.NotFlaky; | |
66 } else if (testData.state === this._states.LastSeenPassing) { | |
67 testData.state = this._states.PossiblyFlaky; | |
68 newFlakyTests.push(testName); | |
69 } | |
70 } | |
71 | |
72 // Record passing tests. | |
73 for (var testName in this._tests) { | |
74 if (testName in failingTests) | |
75 continue; | |
76 | |
77 var testData = this._tests[testName]; | |
78 testData.history.push({ build: buildName, result: { failureType: 'pa
ss' } }); | |
79 | |
80 if (testData.state === this._states.LastSeenPassing) { | |
81 ++testData.count; | |
82 if (testData.count > this._maximumFailOrPassCount) | |
83 testData.state = this._states.NotFlaky; | |
84 } else if (testData.state === this._states.LastSeenFailing) { | |
85 testData.state = this._states.LastSeenPassing; | |
86 testData.count = 1; | |
87 } | |
88 } | |
89 | |
90 return newFlakyTests; | |
91 }, | |
92 | |
93 allFailures: function(testName) { | |
94 if (!(testName in this._tests)) | |
95 return null; | |
96 | |
97 return this._tests[testName].history.filter(function(historyItem) { retu
rn historyItem.result.failureType !== 'pass' }); | |
98 }, | |
99 | |
100 get possiblyFlakyTests() { | |
101 var self = this; | |
102 return Object.keys(self._tests).filter(function(testName) { return self.
_tests[testName].state === self._states.PossiblyFlaky }); | |
103 }, | |
104 | |
105 // If a test has recently failed or passed more than this number of times in
a row we don't | |
106 // consider it to be flaky. | |
107 _maximumFailOrPassCount: 9, | |
108 | |
109 _states: { | |
110 LastSeenFailing: 0, | |
111 LastSeenPassing: 1, | |
112 PossiblyFlaky: 2, | |
113 NotFlaky: 3, | |
114 }, | |
115 }; | |
OLD | NEW |