| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2011 Google Inc. All rights reserved. | 2 * Copyright (C) 2011 Google Inc. All rights reserved. |
| 3 * | 3 * |
| 4 * Redistribution and use in source and binary forms, with or without | 4 * Redistribution and use in source and binary forms, with or without |
| 5 * modification, are permitted provided that the following conditions | 5 * modification, are permitted provided that the following conditions |
| 6 * are met: | 6 * are met: |
| 7 * 1. Redistributions of source code must retain the above copyright | 7 * 1. Redistributions of source code must retain the above copyright |
| 8 * notice, this list of conditions and the following disclaimer. | 8 * notice, this list of conditions and the following disclaimer. |
| 9 * 2. Redistributions in binary form must reproduce the above copyright | 9 * 2. Redistributions in binary form must reproduce the above copyright |
| 10 * notice, this list of conditions and the following disclaimer in the | 10 * notice, this list of conditions and the following disclaimer in the |
| 11 * documentation and/or other materials provided with the distribution. | 11 * documentation and/or other materials provided with the distribution. |
| 12 * | 12 * |
| 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' | 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' |
| 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
| 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
| 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS | 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS |
| 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF | 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF |
| 23 * THE POSSIBILITY OF SUCH DAMAGE. | 23 * THE POSSIBILITY OF SUCH DAMAGE. |
| 24 */ | 24 */ |
| 25 | 25 |
| 26 var results = results || {}; | 26 var results = results || {}; |
| 27 | 27 |
| 28 (function() { | 28 (function() { |
| 29 | 29 |
| 30 var kResultsName = 'failing_results.json'; | |
| 31 | |
| 32 var PASS = 'PASS'; | 30 var PASS = 'PASS'; |
| 33 var TIMEOUT = 'TIMEOUT'; | 31 var TIMEOUT = 'TIMEOUT'; |
| 34 var TEXT = 'TEXT'; | 32 var TEXT = 'TEXT'; |
| 35 var CRASH = 'CRASH'; | 33 var CRASH = 'CRASH'; |
| 36 var IMAGE = 'IMAGE'; | 34 var IMAGE = 'IMAGE'; |
| 37 var IMAGE_TEXT = 'IMAGE+TEXT'; | 35 var IMAGE_TEXT = 'IMAGE+TEXT'; |
| 38 var AUDIO = 'AUDIO'; | 36 var AUDIO = 'AUDIO'; |
| 39 var MISSING = 'MISSING'; | 37 var MISSING = 'MISSING'; |
| 40 | 38 |
| 41 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO]; | 39 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO]; |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 88 { | 86 { |
| 89 suffixList.push(kExpectedAudioSuffix); | 87 suffixList.push(kExpectedAudioSuffix); |
| 90 suffixList.push(kActualAudioSuffix); | 88 suffixList.push(kActualAudioSuffix); |
| 91 } | 89 } |
| 92 | 90 |
| 93 function pushTextSuffixes() | 91 function pushTextSuffixes() |
| 94 { | 92 { |
| 95 suffixList.push(kActualTextSuffix); | 93 suffixList.push(kActualTextSuffix); |
| 96 suffixList.push(kExpectedTextSuffix); | 94 suffixList.push(kExpectedTextSuffix); |
| 97 suffixList.push(kDiffTextSuffix); | 95 suffixList.push(kDiffTextSuffix); |
| 98 // '-wdiff.html', | |
| 99 // '-pretty-diff.html', | |
| 100 } | 96 } |
| 101 | 97 |
| 102 failureTypeList.forEach(function(failureType) { | 98 failureTypeList.forEach(function(failureType) { |
| 103 switch(failureType) { | 99 switch(failureType) { |
| 104 case IMAGE: | 100 case IMAGE: |
| 105 pushImageSuffixes(); | 101 pushImageSuffixes(); |
| 106 break; | 102 break; |
| 107 case TEXT: | 103 case TEXT: |
| 108 pushTextSuffixes(); | 104 pushTextSuffixes(); |
| 109 break; | 105 break; |
| (...skipping 23 matching lines...) Expand all Loading... |
| 133 return suffixList.unique(); | 129 return suffixList.unique(); |
| 134 } | 130 } |
| 135 | 131 |
| 136 function failureTypeList(failureBlob) | 132 function failureTypeList(failureBlob) |
| 137 { | 133 { |
| 138 return failureBlob.split(' '); | 134 return failureBlob.split(' '); |
| 139 }; | 135 }; |
| 140 | 136 |
| 141 function resultsDirectoryURL(builderName) | 137 function resultsDirectoryURL(builderName) |
| 142 { | 138 { |
| 143 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBu
ilderName(builderName) + '/results/layout-test-results/'; | 139 return config.layoutTestResultsURL + '/' + builderName.replace(/[ .()]/g, '_
') + '/results/layout-test-results/'; |
| 144 } | 140 } |
| 145 | 141 |
| 146 function resultsDirectoryURLForBuildNumber(builderName, buildNumber) | |
| 147 { | |
| 148 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBu
ilderName(builderName) + '/' + buildNumber + '/' ; | |
| 149 } | |
| 150 | |
| 151 function resultsSummaryURL(builderName) | |
| 152 { | |
| 153 return resultsDirectoryURL(builderName) + kResultsName; | |
| 154 } | |
| 155 | |
| 156 var g_resultsCache = new base.AsynchronousCache(function(key) { | |
| 157 return net.jsonp(key); | |
| 158 }); | |
| 159 | |
| 160 results.ResultAnalyzer = function(resultNode) | |
| 161 { | |
| 162 this._isUnexpected = resultNode.is_unexpected; | |
| 163 this._actual = resultNode ? failureTypeList(resultNode.actual) : []; | |
| 164 this._expected = resultNode ? this._addImpliedExpectations(failureTypeList(r
esultNode.expected)) : []; | |
| 165 }; | |
| 166 | |
| 167 results.ResultAnalyzer.prototype = { | |
| 168 _addImpliedExpectations: function(resultsList) | |
| 169 { | |
| 170 if (resultsList.indexOf('FAIL') == -1) | |
| 171 return resultsList; | |
| 172 return resultsList.concat(kFailingResults); | |
| 173 }, | |
| 174 _hasPass: function(results) | |
| 175 { | |
| 176 return results.indexOf(PASS) != -1; | |
| 177 }, | |
| 178 unexpectedResults: function() | |
| 179 { | |
| 180 return this._actual.filter(function(result) { | |
| 181 return this._expected.indexOf(result) == -1; | |
| 182 }, this); | |
| 183 }, | |
| 184 succeeded: function() | |
| 185 { | |
| 186 return this._hasPass(this._actual); | |
| 187 }, | |
| 188 flaky: function() | |
| 189 { | |
| 190 return this._actual.length > 1; | |
| 191 }, | |
| 192 wontfix: function() | |
| 193 { | |
| 194 return this._expected.indexOf('WONTFIX') != -1; | |
| 195 }, | |
| 196 hasUnexpectedFailures: function() | |
| 197 { | |
| 198 return this._isUnexpected; | |
| 199 }, | |
| 200 }; | |
| 201 | |
| 202 function isUnexpectedFailure(resultNode) | |
| 203 { | |
| 204 var analyzer = new results.ResultAnalyzer(resultNode); | |
| 205 return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyze
r.flaky() && !analyzer.wontfix(); | |
| 206 } | |
| 207 | |
| 208 function isResultNode(node) | |
| 209 { | |
| 210 return !!node.actual; | |
| 211 } | |
| 212 | |
| 213 results._joinPath = function(parent, child) | |
| 214 { | |
| 215 if (parent.length == 0) | |
| 216 return child; | |
| 217 return parent + '/' + child; | |
| 218 }; | |
| 219 | |
| 220 results._filterTree = function(tree, isLeaf, predicate) | |
| 221 { | |
| 222 var filteredTree = {}; | |
| 223 | |
| 224 function walkSubtree(subtree, directory) | |
| 225 { | |
| 226 for (var childName in subtree) { | |
| 227 var child = subtree[childName]; | |
| 228 var childPath = results._joinPath(directory, childName); | |
| 229 if (isLeaf(child)) { | |
| 230 if (predicate(child)) | |
| 231 filteredTree[childPath] = child; | |
| 232 continue; | |
| 233 } | |
| 234 walkSubtree(child, childPath); | |
| 235 } | |
| 236 } | |
| 237 | |
| 238 walkSubtree(tree, ''); | |
| 239 return filteredTree; | |
| 240 }; | |
| 241 | |
| 242 results.unexpectedFailures = function(resultsTree) | |
| 243 { | |
| 244 return results._filterTree(resultsTree.tests, isResultNode, isUnexpectedFail
ure); | |
| 245 }; | |
| 246 | |
| 247 function resultsByTest(resultsByBuilder, filter) | |
| 248 { | |
| 249 var resultsByTest = {}; | |
| 250 | |
| 251 Object.keys(resultsByBuilder, function(builderName, resultsTree) { | |
| 252 Object.keys(filter(resultsTree), function(testName, resultNode) { | |
| 253 resultsByTest[testName] = resultsByTest[testName] || {}; | |
| 254 resultsByTest[testName][builderName] = resultNode; | |
| 255 }); | |
| 256 }); | |
| 257 | |
| 258 return resultsByTest; | |
| 259 } | |
| 260 | |
| 261 results.unexpectedFailuresByTest = function(resultsByBuilder) | |
| 262 { | |
| 263 return resultsByTest(resultsByBuilder, results.unexpectedFailures); | |
| 264 }; | |
| 265 | |
| 266 results.failureInfo = function(testName, builderName, result) | 142 results.failureInfo = function(testName, builderName, result) |
| 267 { | 143 { |
| 268 return { | 144 return { |
| 269 'testName': testName, | 145 'testName': testName, |
| 270 'builderName': builderName, | 146 'builderName': builderName, |
| 271 'failureTypeList': failureTypeList(result), | 147 'failureTypeList': failureTypeList(result), |
| 272 }; | 148 }; |
| 273 } | 149 } |
| 274 | 150 |
| 275 // Callback data is [{ buildNumber:, url: }] | |
| 276 function historicalResultsLocations(builderName) | |
| 277 { | |
| 278 return builders.mostRecentBuildForBuilder(builderName).then(function (mostRe
centBuildNumber) { | |
| 279 var resultsLocations = []; | |
| 280 // Return the builds in reverse chronological order in order to load the
most recent data first. | |
| 281 for (var buildNumber = mostRecentBuildNumber; buildNumber > mostRecentBu
ildNumber - 100; --buildNumber) { | |
| 282 resultsLocations.push({ | |
| 283 'buildNumber': buildNumber, | |
| 284 'url': resultsDirectoryURLForBuildNumber(builderName, buildNumbe
r) + "failing_results.json" | |
| 285 }); | |
| 286 } | |
| 287 return resultsLocations; | |
| 288 }); | |
| 289 } | |
| 290 | |
| 291 // This will repeatedly call continueCallback(revision, resultNode) until it ret
urns false. | |
| 292 function walkHistory(builderName, testName, continueCallback) | |
| 293 { | |
| 294 var indexOfNextKeyToFetch = 0; | |
| 295 var keyList = []; | |
| 296 | |
| 297 function continueWalk() | |
| 298 { | |
| 299 if (indexOfNextKeyToFetch >= keyList.length) { | |
| 300 processResultNode(0, null); | |
| 301 return; | |
| 302 } | |
| 303 | |
| 304 var resultsURL = keyList[indexOfNextKeyToFetch].url; | |
| 305 ++indexOfNextKeyToFetch; | |
| 306 g_resultsCache.get(resultsURL).then(function(resultsTree) { | |
| 307 if (!Object.size(resultsTree)) { | |
| 308 continueWalk(); | |
| 309 return; | |
| 310 } | |
| 311 var resultNode = results.resultNodeForTest(resultsTree, testName); | |
| 312 var revision = parseInt(resultsTree['blink_revision']); | |
| 313 if (isNaN(revision)) | |
| 314 revision = 0; | |
| 315 processResultNode(revision, resultNode); | |
| 316 }); | |
| 317 } | |
| 318 | |
| 319 function processResultNode(revision, resultNode) | |
| 320 { | |
| 321 var shouldContinue = continueCallback(revision, resultNode); | |
| 322 if (!shouldContinue) | |
| 323 return; | |
| 324 continueWalk(); | |
| 325 } | |
| 326 | |
| 327 historicalResultsLocations(builderName).then(function(resultsLocations) { | |
| 328 keyList = resultsLocations; | |
| 329 continueWalk(); | |
| 330 }); | |
| 331 } | |
| 332 | |
| 333 results.regressionRangeForFailure = function(builderName, testName) { | |
| 334 return new Promise(function(resolve, reject) { | |
| 335 var oldestFailingRevision = 0; | |
| 336 var newestPassingRevision = 0; | |
| 337 | |
| 338 walkHistory(builderName, testName, function(revision, resultNode) { | |
| 339 if (!revision) { | |
| 340 resolve([oldestFailingRevision, newestPassingRevision]); | |
| 341 return false; | |
| 342 } | |
| 343 if (!resultNode) { | |
| 344 newestPassingRevision = revision; | |
| 345 resolve([oldestFailingRevision, newestPassingRevision]); | |
| 346 return false; | |
| 347 } | |
| 348 if (isUnexpectedFailure(resultNode)) { | |
| 349 oldestFailingRevision = revision; | |
| 350 return true; | |
| 351 } | |
| 352 if (!oldestFailingRevision) | |
| 353 return true; // We need to keep looking for a failing revision. | |
| 354 newestPassingRevision = revision; | |
| 355 resolve([oldestFailingRevision, newestPassingRevision]); | |
| 356 return false; | |
| 357 }); | |
| 358 }); | |
| 359 }; | |
| 360 | |
| 361 function mergeRegressionRanges(regressionRanges) | |
| 362 { | |
| 363 var mergedRange = {}; | |
| 364 | |
| 365 mergedRange.oldestFailingRevision = 0; | |
| 366 mergedRange.newestPassingRevision = 0; | |
| 367 | |
| 368 Object.keys(regressionRanges, function(builderName, range) { | |
| 369 if (!range.oldestFailingRevision && !range.newestPassingRevision) | |
| 370 return | |
| 371 | |
| 372 if (!mergedRange.oldestFailingRevision) | |
| 373 mergedRange.oldestFailingRevision = range.oldestFailingRevision; | |
| 374 if (!mergedRange.newestPassingRevision) | |
| 375 mergedRange.newestPassingRevision = range.newestPassingRevision; | |
| 376 | |
| 377 if (range.oldestFailingRevision && range.oldestFailingRevision < mergedR
ange.oldestFailingRevision) | |
| 378 mergedRange.oldestFailingRevision = range.oldestFailingRevision; | |
| 379 if (range.newestPassingRevision > mergedRange.newestPassingRevision) | |
| 380 mergedRange.newestPassingRevision = range.newestPassingRevision; | |
| 381 }); | |
| 382 | |
| 383 return mergedRange; | |
| 384 } | |
| 385 | |
| 386 results.unifyRegressionRanges = function(builderNameList, testName) { | |
| 387 var regressionRanges = {}; | |
| 388 | |
| 389 var rangePromises = []; | |
| 390 builderNameList.forEach(function(builderName) { | |
| 391 rangePromises.push(results.regressionRangeForFailure(builderName, testNa
me) | |
| 392 .then(function(result) { | |
| 393 var oldestFailingRevision = result[0]; | |
| 394 var newestPassingRevision = result[1]; | |
| 395 var range = {}; | |
| 396 range.oldestFailingRevision = oldestFailingRevisi
on; | |
| 397 range.newestPassingRevision = newestPassingRevisi
on; | |
| 398 regressionRanges[builderName] = range; | |
| 399 })); | |
| 400 }); | |
| 401 return Promise.all(rangePromises).then(function() { | |
| 402 var mergedRange = mergeRegressionRanges(regressionRanges); | |
| 403 return [mergedRange.oldestFailingRevision, mergedRange.newestPassingRevi
sion]; | |
| 404 }); | |
| 405 }; | |
| 406 | |
| 407 results.resultNodeForTest = function(resultsTree, testName) | |
| 408 { | |
| 409 var testNamePath = testName.split('/'); | |
| 410 var currentNode = resultsTree['tests']; | |
| 411 testNamePath.forEach(function(segmentName) { | |
| 412 if (!currentNode) | |
| 413 return; | |
| 414 currentNode = (segmentName in currentNode) ? currentNode[segmentName] :
null; | |
| 415 }); | |
| 416 return currentNode; | |
| 417 }; | |
| 418 | |
| 419 results.resultKind = function(url) | 151 results.resultKind = function(url) |
| 420 { | 152 { |
| 421 if (/-actual\.[a-z]+$/.test(url)) | 153 if (/-actual\.[a-z]+$/.test(url)) |
| 422 return results.kActualKind; | 154 return results.kActualKind; |
| 423 else if (/-expected\.[a-z]+$/.test(url)) | 155 else if (/-expected\.[a-z]+$/.test(url)) |
| 424 return results.kExpectedKind; | 156 return results.kExpectedKind; |
| 425 else if (/diff\.[a-z]+$/.test(url)) | 157 else if (/diff\.[a-z]+$/.test(url)) |
| 426 return results.kDiffKind; | 158 return results.kDiffKind; |
| 427 return results.kUnknownKind; | 159 return results.kUnknownKind; |
| 428 } | 160 } |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 473 function() { | 205 function() { |
| 474 resultURLs.push(url); | 206 resultURLs.push(url); |
| 475 }, | 207 }, |
| 476 function() {})); | 208 function() {})); |
| 477 }); | 209 }); |
| 478 return Promise.all(probePromises).then(function() { | 210 return Promise.all(probePromises).then(function() { |
| 479 return sortResultURLsBySuffix(resultURLs); | 211 return sortResultURLsBySuffix(resultURLs); |
| 480 }); | 212 }); |
| 481 }; | 213 }; |
| 482 | 214 |
| 483 results.fetchResultsByBuilder = function(builderNameList) | |
| 484 { | |
| 485 var resultsByBuilder = {}; | |
| 486 var fetchPromises = []; | |
| 487 builderNameList.forEach(function(builderName) { | |
| 488 var resultsURL = resultsSummaryURL(builderName); | |
| 489 fetchPromises.push(net.jsonp(resultsURL).then(function(resultsTree) { | |
| 490 resultsByBuilder[builderName] = resultsTree; | |
| 491 })); | |
| 492 }); | |
| 493 return Promise.all(fetchPromises).then(function() { | |
| 494 return resultsByBuilder; | |
| 495 }); | |
| 496 }; | |
| 497 | |
| 498 })(); | 215 })(); |
| OLD | NEW |