OLD | NEW |
1 /* | 1 /* |
2 * Copyright (C) 2011 Google Inc. All rights reserved. | 2 * Copyright (C) 2011 Google Inc. All rights reserved. |
3 * | 3 * |
4 * Redistribution and use in source and binary forms, with or without | 4 * Redistribution and use in source and binary forms, with or without |
5 * modification, are permitted provided that the following conditions | 5 * modification, are permitted provided that the following conditions |
6 * are met: | 6 * are met: |
7 * 1. Redistributions of source code must retain the above copyright | 7 * 1. Redistributions of source code must retain the above copyright |
8 * notice, this list of conditions and the following disclaimer. | 8 * notice, this list of conditions and the following disclaimer. |
9 * 2. Redistributions in binary form must reproduce the above copyright | 9 * 2. Redistributions in binary form must reproduce the above copyright |
10 * notice, this list of conditions and the following disclaimer in the | 10 * notice, this list of conditions and the following disclaimer in the |
11 * documentation and/or other materials provided with the distribution. | 11 * documentation and/or other materials provided with the distribution. |
12 * | 12 * |
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' |
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS |
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF |
23 * THE POSSIBILITY OF SUCH DAMAGE. | 23 * THE POSSIBILITY OF SUCH DAMAGE. |
24 */ | 24 */ |
25 | 25 |
26 var results = results || {}; | 26 var results = results || {}; |
27 | 27 |
28 (function() { | 28 (function() { |
29 | 29 |
30 var kResultsName = 'failing_results.json'; | |
31 | |
32 var PASS = 'PASS'; | 30 var PASS = 'PASS'; |
33 var TIMEOUT = 'TIMEOUT'; | 31 var TIMEOUT = 'TIMEOUT'; |
34 var TEXT = 'TEXT'; | 32 var TEXT = 'TEXT'; |
35 var CRASH = 'CRASH'; | 33 var CRASH = 'CRASH'; |
36 var IMAGE = 'IMAGE'; | 34 var IMAGE = 'IMAGE'; |
37 var IMAGE_TEXT = 'IMAGE+TEXT'; | 35 var IMAGE_TEXT = 'IMAGE+TEXT'; |
38 var AUDIO = 'AUDIO'; | 36 var AUDIO = 'AUDIO'; |
39 var MISSING = 'MISSING'; | 37 var MISSING = 'MISSING'; |
40 | 38 |
41 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO]; | 39 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO]; |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
88 { | 86 { |
89 suffixList.push(kExpectedAudioSuffix); | 87 suffixList.push(kExpectedAudioSuffix); |
90 suffixList.push(kActualAudioSuffix); | 88 suffixList.push(kActualAudioSuffix); |
91 } | 89 } |
92 | 90 |
93 function pushTextSuffixes() | 91 function pushTextSuffixes() |
94 { | 92 { |
95 suffixList.push(kActualTextSuffix); | 93 suffixList.push(kActualTextSuffix); |
96 suffixList.push(kExpectedTextSuffix); | 94 suffixList.push(kExpectedTextSuffix); |
97 suffixList.push(kDiffTextSuffix); | 95 suffixList.push(kDiffTextSuffix); |
98 // '-wdiff.html', | |
99 // '-pretty-diff.html', | |
100 } | 96 } |
101 | 97 |
102 failureTypeList.forEach(function(failureType) { | 98 failureTypeList.forEach(function(failureType) { |
103 switch(failureType) { | 99 switch(failureType) { |
104 case IMAGE: | 100 case IMAGE: |
105 pushImageSuffixes(); | 101 pushImageSuffixes(); |
106 break; | 102 break; |
107 case TEXT: | 103 case TEXT: |
108 pushTextSuffixes(); | 104 pushTextSuffixes(); |
109 break; | 105 break; |
(...skipping 23 matching lines...) Expand all Loading... |
133 return suffixList.unique(); | 129 return suffixList.unique(); |
134 } | 130 } |
135 | 131 |
136 function failureTypeList(failureBlob) | 132 function failureTypeList(failureBlob) |
137 { | 133 { |
138 return failureBlob.split(' '); | 134 return failureBlob.split(' '); |
139 }; | 135 }; |
140 | 136 |
141 function resultsDirectoryURL(builderName) | 137 function resultsDirectoryURL(builderName) |
142 { | 138 { |
143 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBu
ilderName(builderName) + '/results/layout-test-results/'; | 139 return config.layoutTestResultsURL + '/' + builderName.replace(/[ .()]/g, '_
') + '/results/layout-test-results/'; |
144 } | 140 } |
145 | 141 |
146 function resultsDirectoryURLForBuildNumber(builderName, buildNumber) | |
147 { | |
148 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBu
ilderName(builderName) + '/' + buildNumber + '/' ; | |
149 } | |
150 | |
151 function resultsSummaryURL(builderName) | |
152 { | |
153 return resultsDirectoryURL(builderName) + kResultsName; | |
154 } | |
155 | |
156 var g_resultsCache = new base.AsynchronousCache(function(key) { | |
157 return net.jsonp(key); | |
158 }); | |
159 | |
160 results.ResultAnalyzer = function(resultNode) | |
161 { | |
162 this._isUnexpected = resultNode.is_unexpected; | |
163 this._actual = resultNode ? failureTypeList(resultNode.actual) : []; | |
164 this._expected = resultNode ? this._addImpliedExpectations(failureTypeList(r
esultNode.expected)) : []; | |
165 }; | |
166 | |
167 results.ResultAnalyzer.prototype = { | |
168 _addImpliedExpectations: function(resultsList) | |
169 { | |
170 if (resultsList.indexOf('FAIL') == -1) | |
171 return resultsList; | |
172 return resultsList.concat(kFailingResults); | |
173 }, | |
174 _hasPass: function(results) | |
175 { | |
176 return results.indexOf(PASS) != -1; | |
177 }, | |
178 unexpectedResults: function() | |
179 { | |
180 return this._actual.filter(function(result) { | |
181 return this._expected.indexOf(result) == -1; | |
182 }, this); | |
183 }, | |
184 succeeded: function() | |
185 { | |
186 return this._hasPass(this._actual); | |
187 }, | |
188 flaky: function() | |
189 { | |
190 return this._actual.length > 1; | |
191 }, | |
192 wontfix: function() | |
193 { | |
194 return this._expected.indexOf('WONTFIX') != -1; | |
195 }, | |
196 hasUnexpectedFailures: function() | |
197 { | |
198 return this._isUnexpected; | |
199 }, | |
200 }; | |
201 | |
202 function isUnexpectedFailure(resultNode) | |
203 { | |
204 var analyzer = new results.ResultAnalyzer(resultNode); | |
205 return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyze
r.flaky() && !analyzer.wontfix(); | |
206 } | |
207 | |
208 function isResultNode(node) | |
209 { | |
210 return !!node.actual; | |
211 } | |
212 | |
213 results._joinPath = function(parent, child) | |
214 { | |
215 if (parent.length == 0) | |
216 return child; | |
217 return parent + '/' + child; | |
218 }; | |
219 | |
220 results._filterTree = function(tree, isLeaf, predicate) | |
221 { | |
222 var filteredTree = {}; | |
223 | |
224 function walkSubtree(subtree, directory) | |
225 { | |
226 for (var childName in subtree) { | |
227 var child = subtree[childName]; | |
228 var childPath = results._joinPath(directory, childName); | |
229 if (isLeaf(child)) { | |
230 if (predicate(child)) | |
231 filteredTree[childPath] = child; | |
232 continue; | |
233 } | |
234 walkSubtree(child, childPath); | |
235 } | |
236 } | |
237 | |
238 walkSubtree(tree, ''); | |
239 return filteredTree; | |
240 }; | |
241 | |
242 results.unexpectedFailures = function(resultsTree) | |
243 { | |
244 return results._filterTree(resultsTree.tests, isResultNode, isUnexpectedFail
ure); | |
245 }; | |
246 | |
247 function resultsByTest(resultsByBuilder, filter) | |
248 { | |
249 var resultsByTest = {}; | |
250 | |
251 Object.keys(resultsByBuilder, function(builderName, resultsTree) { | |
252 Object.keys(filter(resultsTree), function(testName, resultNode) { | |
253 resultsByTest[testName] = resultsByTest[testName] || {}; | |
254 resultsByTest[testName][builderName] = resultNode; | |
255 }); | |
256 }); | |
257 | |
258 return resultsByTest; | |
259 } | |
260 | |
261 results.unexpectedFailuresByTest = function(resultsByBuilder) | |
262 { | |
263 return resultsByTest(resultsByBuilder, results.unexpectedFailures); | |
264 }; | |
265 | |
266 results.failureInfo = function(testName, builderName, result) | 142 results.failureInfo = function(testName, builderName, result) |
267 { | 143 { |
268 return { | 144 return { |
269 'testName': testName, | 145 'testName': testName, |
270 'builderName': builderName, | 146 'builderName': builderName, |
271 'failureTypeList': failureTypeList(result), | 147 'failureTypeList': failureTypeList(result), |
272 }; | 148 }; |
273 } | 149 } |
274 | 150 |
275 // Callback data is [{ buildNumber:, url: }] | |
276 function historicalResultsLocations(builderName) | |
277 { | |
278 return builders.mostRecentBuildForBuilder(builderName).then(function (mostRe
centBuildNumber) { | |
279 var resultsLocations = []; | |
280 // Return the builds in reverse chronological order in order to load the
most recent data first. | |
281 for (var buildNumber = mostRecentBuildNumber; buildNumber > mostRecentBu
ildNumber - 100; --buildNumber) { | |
282 resultsLocations.push({ | |
283 'buildNumber': buildNumber, | |
284 'url': resultsDirectoryURLForBuildNumber(builderName, buildNumbe
r) + "failing_results.json" | |
285 }); | |
286 } | |
287 return resultsLocations; | |
288 }); | |
289 } | |
290 | |
291 // This will repeatedly call continueCallback(revision, resultNode) until it ret
urns false. | |
292 function walkHistory(builderName, testName, continueCallback) | |
293 { | |
294 var indexOfNextKeyToFetch = 0; | |
295 var keyList = []; | |
296 | |
297 function continueWalk() | |
298 { | |
299 if (indexOfNextKeyToFetch >= keyList.length) { | |
300 processResultNode(0, null); | |
301 return; | |
302 } | |
303 | |
304 var resultsURL = keyList[indexOfNextKeyToFetch].url; | |
305 ++indexOfNextKeyToFetch; | |
306 g_resultsCache.get(resultsURL).then(function(resultsTree) { | |
307 if (!Object.size(resultsTree)) { | |
308 continueWalk(); | |
309 return; | |
310 } | |
311 var resultNode = results.resultNodeForTest(resultsTree, testName); | |
312 var revision = parseInt(resultsTree['blink_revision']); | |
313 if (isNaN(revision)) | |
314 revision = 0; | |
315 processResultNode(revision, resultNode); | |
316 }); | |
317 } | |
318 | |
319 function processResultNode(revision, resultNode) | |
320 { | |
321 var shouldContinue = continueCallback(revision, resultNode); | |
322 if (!shouldContinue) | |
323 return; | |
324 continueWalk(); | |
325 } | |
326 | |
327 historicalResultsLocations(builderName).then(function(resultsLocations) { | |
328 keyList = resultsLocations; | |
329 continueWalk(); | |
330 }); | |
331 } | |
332 | |
333 results.regressionRangeForFailure = function(builderName, testName) { | |
334 return new Promise(function(resolve, reject) { | |
335 var oldestFailingRevision = 0; | |
336 var newestPassingRevision = 0; | |
337 | |
338 walkHistory(builderName, testName, function(revision, resultNode) { | |
339 if (!revision) { | |
340 resolve([oldestFailingRevision, newestPassingRevision]); | |
341 return false; | |
342 } | |
343 if (!resultNode) { | |
344 newestPassingRevision = revision; | |
345 resolve([oldestFailingRevision, newestPassingRevision]); | |
346 return false; | |
347 } | |
348 if (isUnexpectedFailure(resultNode)) { | |
349 oldestFailingRevision = revision; | |
350 return true; | |
351 } | |
352 if (!oldestFailingRevision) | |
353 return true; // We need to keep looking for a failing revision. | |
354 newestPassingRevision = revision; | |
355 resolve([oldestFailingRevision, newestPassingRevision]); | |
356 return false; | |
357 }); | |
358 }); | |
359 }; | |
360 | |
361 function mergeRegressionRanges(regressionRanges) | |
362 { | |
363 var mergedRange = {}; | |
364 | |
365 mergedRange.oldestFailingRevision = 0; | |
366 mergedRange.newestPassingRevision = 0; | |
367 | |
368 Object.keys(regressionRanges, function(builderName, range) { | |
369 if (!range.oldestFailingRevision && !range.newestPassingRevision) | |
370 return | |
371 | |
372 if (!mergedRange.oldestFailingRevision) | |
373 mergedRange.oldestFailingRevision = range.oldestFailingRevision; | |
374 if (!mergedRange.newestPassingRevision) | |
375 mergedRange.newestPassingRevision = range.newestPassingRevision; | |
376 | |
377 if (range.oldestFailingRevision && range.oldestFailingRevision < mergedR
ange.oldestFailingRevision) | |
378 mergedRange.oldestFailingRevision = range.oldestFailingRevision; | |
379 if (range.newestPassingRevision > mergedRange.newestPassingRevision) | |
380 mergedRange.newestPassingRevision = range.newestPassingRevision; | |
381 }); | |
382 | |
383 return mergedRange; | |
384 } | |
385 | |
386 results.unifyRegressionRanges = function(builderNameList, testName) { | |
387 var regressionRanges = {}; | |
388 | |
389 var rangePromises = []; | |
390 builderNameList.forEach(function(builderName) { | |
391 rangePromises.push(results.regressionRangeForFailure(builderName, testNa
me) | |
392 .then(function(result) { | |
393 var oldestFailingRevision = result[0]; | |
394 var newestPassingRevision = result[1]; | |
395 var range = {}; | |
396 range.oldestFailingRevision = oldestFailingRevisi
on; | |
397 range.newestPassingRevision = newestPassingRevisi
on; | |
398 regressionRanges[builderName] = range; | |
399 })); | |
400 }); | |
401 return Promise.all(rangePromises).then(function() { | |
402 var mergedRange = mergeRegressionRanges(regressionRanges); | |
403 return [mergedRange.oldestFailingRevision, mergedRange.newestPassingRevi
sion]; | |
404 }); | |
405 }; | |
406 | |
407 results.resultNodeForTest = function(resultsTree, testName) | |
408 { | |
409 var testNamePath = testName.split('/'); | |
410 var currentNode = resultsTree['tests']; | |
411 testNamePath.forEach(function(segmentName) { | |
412 if (!currentNode) | |
413 return; | |
414 currentNode = (segmentName in currentNode) ? currentNode[segmentName] :
null; | |
415 }); | |
416 return currentNode; | |
417 }; | |
418 | |
419 results.resultKind = function(url) | 151 results.resultKind = function(url) |
420 { | 152 { |
421 if (/-actual\.[a-z]+$/.test(url)) | 153 if (/-actual\.[a-z]+$/.test(url)) |
422 return results.kActualKind; | 154 return results.kActualKind; |
423 else if (/-expected\.[a-z]+$/.test(url)) | 155 else if (/-expected\.[a-z]+$/.test(url)) |
424 return results.kExpectedKind; | 156 return results.kExpectedKind; |
425 else if (/diff\.[a-z]+$/.test(url)) | 157 else if (/diff\.[a-z]+$/.test(url)) |
426 return results.kDiffKind; | 158 return results.kDiffKind; |
427 return results.kUnknownKind; | 159 return results.kUnknownKind; |
428 } | 160 } |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
473 function() { | 205 function() { |
474 resultURLs.push(url); | 206 resultURLs.push(url); |
475 }, | 207 }, |
476 function() {})); | 208 function() {})); |
477 }); | 209 }); |
478 return Promise.all(probePromises).then(function() { | 210 return Promise.all(probePromises).then(function() { |
479 return sortResultURLsBySuffix(resultURLs); | 211 return sortResultURLsBySuffix(resultURLs); |
480 }); | 212 }); |
481 }; | 213 }; |
482 | 214 |
483 results.fetchResultsByBuilder = function(builderNameList) | |
484 { | |
485 var resultsByBuilder = {}; | |
486 var fetchPromises = []; | |
487 builderNameList.forEach(function(builderName) { | |
488 var resultsURL = resultsSummaryURL(builderName); | |
489 fetchPromises.push(net.jsonp(resultsURL).then(function(resultsTree) { | |
490 resultsByBuilder[builderName] = resultsTree; | |
491 })); | |
492 }); | |
493 return Promise.all(fetchPromises).then(function() { | |
494 return resultsByBuilder; | |
495 }); | |
496 }; | |
497 | |
498 })(); | 215 })(); |
OLD | NEW |