| OLD | NEW |
| (Empty) |
| 1 /* | |
| 2 * Copyright (C) 2011 Apple Inc. All rights reserved. | |
| 3 * | |
| 4 * Redistribution and use in source and binary forms, with or without | |
| 5 * modification, are permitted provided that the following conditions | |
| 6 * are met: | |
| 7 * 1. Redistributions of source code must retain the above copyright | |
| 8 * notice, this list of conditions and the following disclaimer. | |
| 9 * 2. Redistributions in binary form must reproduce the above copyright | |
| 10 * notice, this list of conditions and the following disclaimer in the | |
| 11 * documentation and/or other materials provided with the distribution. | |
| 12 * | |
| 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | |
| 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
| 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | |
| 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
| 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
| 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |
| 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |
| 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |
| 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | |
| 23 * THE POSSIBILITY OF SUCH DAMAGE. | |
| 24 */ | |
| 25 | |
| 26 (function() { | |
| 27 | |
| 28 module("Builder"); | |
| 29 | |
| 30 function runGetNumberOfFailingTestsTest(jsonData, callback) { | |
| 31 var mockBuildbot = {}; | |
| 32 mockBuildbot.baseURL = 'http://example.com/'; | |
| 33 | |
| 34 var builder = new Builder('test builder', mockBuildbot); | |
| 35 | |
| 36 var realGetResource = window.getResource; | |
| 37 window.getResource = function(url, successCallback, errorCallback) { | |
| 38 var mockXHR = {}; | |
| 39 mockXHR.responseText = JSON.stringify(jsonData); | |
| 40 | |
| 41 // FIXME: It would be better for this callback to happen | |
| 42 // asynchronously, to match what happens for real requests. | |
| 43 successCallback(mockXHR); | |
| 44 }; | |
| 45 | |
| 46 // FIXME: It's lame to be modifying singletons like this. We should get rid | |
| 47 // of our singleton usage entirely! | |
| 48 ok(!("PersistentCache" in window)); | |
| 49 window.PersistentCache = { | |
| 50 contains: function() { return false; }, | |
| 51 set: function() { }, | |
| 52 get: function() { }, | |
| 53 }; | |
| 54 | |
| 55 builder.getNumberOfFailingTests(1, callback); | |
| 56 | |
| 57 window.getResource = realGetResource; | |
| 58 equal(window.getResource, realGetResource); | |
| 59 delete window.PersistentCache; | |
| 60 } | |
| 61 | |
| 62 test("getNumberOfFailingTests shouldn't include leaks", 4, function() { | |
| 63 const jsonData = { | |
| 64 steps: [ | |
| 65 { | |
| 66 isFinished: true, | |
| 67 isStarted: true, | |
| 68 name: 'layout-test', | |
| 69 results: [ | |
| 70 2, | |
| 71 [ | |
| 72 "2178 total leaks found!", | |
| 73 "2 test cases (<1%) had incorrect layout", | |
| 74 "2 test cases (<1%) crashed", | |
| 75 ], | |
| 76 ], | |
| 77 times: [ | |
| 78 1310599204.1231229, | |
| 79 1310600152.973659, | |
| 80 ] | |
| 81 }, | |
| 82 ], | |
| 83 }; | |
| 84 | |
| 85 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 86 equal(failureCount, 4); | |
| 87 equal(tooManyFailures, false); | |
| 88 }); | |
| 89 }); | |
| 90 | |
| 91 test("getNumberOfFailingTests detects spurious run-webkit-tests failures", 4, fu
nction() { | |
| 92 const jsonData = { | |
| 93 steps: [ | |
| 94 { | |
| 95 isFinished: true, | |
| 96 isStarted: true, | |
| 97 name: "layout-test", | |
| 98 results: [ | |
| 99 2, | |
| 100 [ | |
| 101 "layout-test" | |
| 102 ] | |
| 103 ], | |
| 104 step_number: 7, | |
| 105 text: [ | |
| 106 "layout-test" | |
| 107 ], | |
| 108 times: [ | |
| 109 1310437736.00494, | |
| 110 1310440118.038023 | |
| 111 ], | |
| 112 }, | |
| 113 ], | |
| 114 }; | |
| 115 | |
| 116 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 117 equal(failureCount, -1); | |
| 118 equal(tooManyFailures, false); | |
| 119 }); | |
| 120 }); | |
| 121 | |
| 122 test("getNumberOfFailingTests understands NRWT exiting early due to too many fai
lures", 4, function() { | |
| 123 const jsonData = { | |
| 124 steps: [ | |
| 125 { | |
| 126 isFinished: true, | |
| 127 isStarted: true, | |
| 128 name: "layout-test", | |
| 129 results: [ | |
| 130 2, | |
| 131 [ | |
| 132 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exit
ing early after 20 crashes and 0 timeouts. 2251 tests run.", | |
| 133 "20 failed" | |
| 134 ] | |
| 135 ], | |
| 136 step_number: 4, | |
| 137 text: [ | |
| 138 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exitin
g early after 20 crashes and 0 timeouts. 2251 tests run.", | |
| 139 "20 failed" | |
| 140 ], | |
| 141 times: [ | |
| 142 1310557115.793082, | |
| 143 1310557129.832104 | |
| 144 ] | |
| 145 }, | |
| 146 ], | |
| 147 }; | |
| 148 | |
| 149 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 150 equal(failureCount, 20); | |
| 151 equal(tooManyFailures, true); | |
| 152 }); | |
| 153 }); | |
| 154 | |
| 155 test("getNumberOfFailingTests treats build step interruptions as errors", 4, fun
ction() { | |
| 156 const jsonData = { | |
| 157 steps: [ | |
| 158 { | |
| 159 isFinished: true, | |
| 160 isStarted: true, | |
| 161 name: "layout-test", | |
| 162 results: [ | |
| 163 4, | |
| 164 [ | |
| 165 "interrupted", | |
| 166 ] | |
| 167 ], | |
| 168 step_number: 5, | |
| 169 text: [ | |
| 170 "layout-test", | |
| 171 "interrupted", | |
| 172 ], | |
| 173 times: [ | |
| 174 1310599204.1231229, | |
| 175 1310600152.973659, | |
| 176 ] | |
| 177 }, | |
| 178 ], | |
| 179 }; | |
| 180 | |
| 181 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 182 equal(failureCount, -1); | |
| 183 equal(tooManyFailures, false); | |
| 184 }); | |
| 185 }); | |
| 186 | |
| 187 test("getNumberOfFailingTests treats unfinished test runs as errors", 4, functio
n() { | |
| 188 const jsonData = { | |
| 189 steps: [ | |
| 190 { | |
| 191 isStarted: true, | |
| 192 name: "layout-test", | |
| 193 step_number: 5, | |
| 194 text: [ | |
| 195 "layout-tests running" | |
| 196 ], | |
| 197 times: [ | |
| 198 1312989295.518481, | |
| 199 null | |
| 200 ] | |
| 201 }, | |
| 202 ], | |
| 203 }; | |
| 204 | |
| 205 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 206 equal(failureCount, -1); | |
| 207 equal(tooManyFailures, false); | |
| 208 }); | |
| 209 }); | |
| 210 | |
| 211 test("getNumberOfFailingTests treats successful but unbelievably short test runs
as errors", 4, function() { | |
| 212 const jsonData = { | |
| 213 steps: [ | |
| 214 { | |
| 215 isFinished: true, | |
| 216 isStarted: true, | |
| 217 name: "layout-test", | |
| 218 step_number: 7, | |
| 219 text: [ | |
| 220 "layout-test" | |
| 221 ], | |
| 222 times: [ | |
| 223 1311288797.7207019, | |
| 224 1311288802.7791941 | |
| 225 ] | |
| 226 }, | |
| 227 ], | |
| 228 }; | |
| 229 | |
| 230 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 231 equal(failureCount, -1); | |
| 232 equal(tooManyFailures, false); | |
| 233 }); | |
| 234 }); | |
| 235 | |
| 236 test("getNumberOfFailingTests doesn't care if a failing run is unbelievably shor
t", 4, function() { | |
| 237 const jsonData = { | |
| 238 steps: [ | |
| 239 { | |
| 240 isFinished: true, | |
| 241 isStarted: true, | |
| 242 name: "layout-test", | |
| 243 results: [ | |
| 244 2, | |
| 245 [ | |
| 246 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exit
ing early after 20 crashes and 0 timeouts. 2251 tests run.", | |
| 247 "20 failed" | |
| 248 ] | |
| 249 ], | |
| 250 step_number: 4, | |
| 251 text: [ | |
| 252 "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exitin
g early after 20 crashes and 0 timeouts. 2251 tests run.", | |
| 253 "20 failed" | |
| 254 ], | |
| 255 times: [ | |
| 256 1310557115.793082, | |
| 257 1310557119.832104 | |
| 258 ] | |
| 259 }, | |
| 260 ], | |
| 261 }; | |
| 262 | |
| 263 runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailu
res) { | |
| 264 equal(failureCount, 20); | |
| 265 equal(tooManyFailures, true); | |
| 266 }); | |
| 267 }); | |
| 268 | |
| 269 })(); | |
| OLD | NEW |