OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (C) 2011 Apple Inc. All rights reserved. | |
3 * | |
4 * Redistribution and use in source and binary forms, with or without | |
5 * modification, are permitted provided that the following conditions | |
6 * are met: | |
7 * 1. Redistributions of source code must retain the above copyright | |
8 * notice, this list of conditions and the following disclaimer. | |
9 * 2. Redistributions in binary form must reproduce the above copyright | |
10 * notice, this list of conditions and the following disclaimer in the | |
11 * documentation and/or other materials provided with the distribution. | |
12 * | |
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | |
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | |
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | |
23 * THE POSSIBILITY OF SUCH DAMAGE. | |
24 */ | |
25 | |
26 (function() { | |
27 | |
28 module("Builder"); | |
29 | |
30 function runGetNumberOfFailingTestsTest(jsonData, callback) { | |
31 var mockBuildbot = {}; | |
32 mockBuildbot.baseURL = 'http://example.com/'; | |
33 | |
34 var builder = new Builder('test builder', mockBuildbot); | |
35 | |
36 var realGetResource = window.getResource; | |
37 window.getResource = function(url, successCallback, errorCallback) { | |
38 var mockXHR = {}; | |
39 mockXHR.responseText = JSON.stringify(jsonData); | |
40 | |
41 // FIXME: It would be better for this callback to happen | |
42 // asynchronously, to match what happens for real requests. | |
43 successCallback(mockXHR); | |
44 }; | |
45 | |
46 // FIXME: It's lame to be modifying singletons like this. We should get rid | |
47 // of our singleton usage entirely! | |
48 ok(!("PersistentCache" in window)); | |
49 window.PersistentCache = { | |
50 contains: function() { return false; }, | |
51 set: function() { }, | |
52 get: function() { }, | |
53 }; | |
54 | |
55 builder.getNumberOfFailingTests(1, callback); | |
56 | |
57 window.getResource = realGetResource; | |
58 equal(window.getResource, realGetResource); | |
59 delete window.PersistentCache; | |
60 } | |
61 | |
62 test("getNumberOfFailingTests shouldn't include leaks", 4, function() { | |
63 const jsonData = { | |
64 steps: [ | |
65 { | |
66 isFinished: true, | |
67 isStarted: true, | |
68 name: 'layout-test', | |
69 results: [ | |
70 2, | |
71 [ | |
72 "2178 total leaks found!", | |
73 "2 test cases (<1%) had incorrect layout", | |
74 "2 test cases (<1%) crashed", | |
75 ], | |
76 ], | |
77 times: [ | |
78 1310599204.1231229, | |
79 1310600152.973659, | |
80 ] | |
81 }, | |
82 ], | |
83 }; | |
84 | |
85 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
86 equal(failureCount, 4); | |
87 equal(tooManyFailures, false); | |
88 }); | |
89 }); | |
90 | |
91 test("getNumberOfFailingTests detects spurious run-webkit-tests failures", 4, fu
nction() { | |
92 const jsonData = { | |
93 steps: [ | |
94 { | |
95 isFinished: true, | |
96 isStarted: true, | |
97 name: "layout-test", | |
98 results: [ | |
99 2, | |
100 [ | |
101 "layout-test" | |
102 ] | |
103 ], | |
104 step_number: 7, | |
105 text: [ | |
106 "layout-test" | |
107 ], | |
108 times: [ | |
109 1310437736.00494, | |
110 1310440118.038023 | |
111 ], | |
112 }, | |
113 ], | |
114 }; | |
115 | |
116 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
117 equal(failureCount, -1); | |
118 equal(tooManyFailures, false); | |
119 }); | |
120 }); | |
121 | |
122 test("getNumberOfFailingTests understands NRWT exiting early due to too many fai
lures", 4, function() { | |
123 const jsonData = { | |
124 steps: [ | |
125 { | |
126 isFinished: true, | |
127 isStarted: true, | |
128 name: "layout-test", | |
129 results: [ | |
130 2, | |
131 [ | |
132 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exit
ing early after 20 crashes and 0 timeouts. 2251 tests run.", | |
133 "20 failed" | |
134 ] | |
135 ], | |
136 step_number: 4, | |
137 text: [ | |
138 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exitin
g early after 20 crashes and 0 timeouts. 2251 tests run.", | |
139 "20 failed" | |
140 ], | |
141 times: [ | |
142 1310557115.793082, | |
143 1310557129.832104 | |
144 ] | |
145 }, | |
146 ], | |
147 }; | |
148 | |
149 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
150 equal(failureCount, 20); | |
151 equal(tooManyFailures, true); | |
152 }); | |
153 }); | |
154 | |
155 test("getNumberOfFailingTests treats build step interruptions as errors", 4, fun
ction() { | |
156 const jsonData = { | |
157 steps: [ | |
158 { | |
159 isFinished: true, | |
160 isStarted: true, | |
161 name: "layout-test", | |
162 results: [ | |
163 4, | |
164 [ | |
165 "interrupted", | |
166 ] | |
167 ], | |
168 step_number: 5, | |
169 text: [ | |
170 "layout-test", | |
171 "interrupted", | |
172 ], | |
173 times: [ | |
174 1310599204.1231229, | |
175 1310600152.973659, | |
176 ] | |
177 }, | |
178 ], | |
179 }; | |
180 | |
181 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
182 equal(failureCount, -1); | |
183 equal(tooManyFailures, false); | |
184 }); | |
185 }); | |
186 | |
187 test("getNumberOfFailingTests treats unfinished test runs as errors", 4, functio
n() { | |
188 const jsonData = { | |
189 steps: [ | |
190 { | |
191 isStarted: true, | |
192 name: "layout-test", | |
193 step_number: 5, | |
194 text: [ | |
195 "layout-tests running" | |
196 ], | |
197 times: [ | |
198 1312989295.518481, | |
199 null | |
200 ] | |
201 }, | |
202 ], | |
203 }; | |
204 | |
205 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
206 equal(failureCount, -1); | |
207 equal(tooManyFailures, false); | |
208 }); | |
209 }); | |
210 | |
211 test("getNumberOfFailingTests treats successful but unbelievably short test runs
as errors", 4, function() { | |
212 const jsonData = { | |
213 steps: [ | |
214 { | |
215 isFinished: true, | |
216 isStarted: true, | |
217 name: "layout-test", | |
218 step_number: 7, | |
219 text: [ | |
220 "layout-test" | |
221 ], | |
222 times: [ | |
223 1311288797.7207019, | |
224 1311288802.7791941 | |
225 ] | |
226 }, | |
227 ], | |
228 }; | |
229 | |
230 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
231 equal(failureCount, -1); | |
232 equal(tooManyFailures, false); | |
233 }); | |
234 }); | |
235 | |
236 test("getNumberOfFailingTests doesn't care if a failing run is unbelievably shor
t", 4, function() { | |
237 const jsonData = { | |
238 steps: [ | |
239 { | |
240 isFinished: true, | |
241 isStarted: true, | |
242 name: "layout-test", | |
243 results: [ | |
244 2, | |
245 [ | |
246 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exit
ing early after 20 crashes and 0 timeouts. 2251 tests run.", | |
247 "20 failed" | |
248 ] | |
249 ], | |
250 step_number: 4, | |
251 text: [ | |
252 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exitin
g early after 20 crashes and 0 timeouts. 2251 tests run.", | |
253 "20 failed" | |
254 ], | |
255 times: [ | |
256 1310557115.793082, | |
257 1310557119.832104 | |
258 ] | |
259 }, | |
260 ], | |
261 }; | |
262 | |
263 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
264 equal(failureCount, 20); | |
265 equal(tooManyFailures, true); | |
266 }); | |
267 }); | |
268 | |
269 })(); | |
OLD | NEW |