| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Retrieve passing and failing WebKit revision numbers from canaries. | |
| 7 | |
| 8 From each canary, | |
| 9 - the last WebKit revision number for which all the tests have passed, | |
| 10 - the last WebKit revision number for which the tests were run, and | |
| 11 - the names of failing layout tests | |
| 12 are retrieved and printed. | |
| 13 """ | |
| 14 | |
| 15 | |
| 16 import json | |
| 17 import optparse | |
| 18 import re | |
| 19 import sys | |
| 20 import urllib2 | |
| 21 | |
| 22 _WEBKIT_REVISION_IN_DEPS_RE = re.compile(r'"webkit_revision"\s*:\s*"(\d+)"') | |
| 23 _DEPS_FILE_URL = "http://src.chromium.org/viewvc/chrome/trunk/src/DEPS" | |
| 24 _DEFAULT_BUILDERS = [ | |
| 25 "Webkit Win", | |
| 26 "Webkit Vista", | |
| 27 "Webkit Win7", | |
| 28 "Webkit Win (dbg)(1)", | |
| 29 "Webkit Win (dbg)(2)", | |
| 30 "Webkit Mac10.5 (CG)", | |
| 31 "Webkit Mac10.6 (CG)", | |
| 32 "Webkit Mac10.5 (CG)(dbg)(1)", | |
| 33 "Webkit Mac10.5 (CG)(dbg)(2)", | |
| 34 "Webkit Mac10.6 (CG)(dbg)", | |
| 35 "Webkit Linux", | |
| 36 "Webkit Linux 32", | |
| 37 "Webkit Linux (dbg)(1)", | |
| 38 "Webkit Linux (dbg)(2)", | |
| 39 ] | |
| 40 _DEFAULT_MAX_BUILDS = 10 | |
| 41 _TEST_PREFIX = "&tests=" | |
| 42 _TEST_SUFFIX = '">' | |
| 43 _WEBKIT_TESTS = "webkit_tests" | |
| 44 | |
| 45 | |
| 46 def _OpenUrl(url): | |
| 47 """Opens a URL. | |
| 48 | |
| 49 Returns: | |
| 50 A file-like object in case of success, an empty list otherwise. | |
| 51 """ | |
| 52 try: | |
| 53 return urllib2.urlopen(url) | |
| 54 except urllib2.URLError, url_error: | |
| 55 message = "" | |
| 56 # Surprisingly, urllib2.URLError has different attributes based on the | |
| 57 # kinds of errors -- "code" for HTTP-level errors, "reason" for others. | |
| 58 if hasattr(url_error, "code"): | |
| 59 message = "Status code: %d" % url_error.code | |
| 60 if hasattr(url_error, "reason"): | |
| 61 message = url_error.reason | |
| 62 print >>sys.stderr, "Failed to open %s: %s" % (url, message) | |
| 63 return [] | |
| 64 | |
| 65 | |
| 66 def _WebkitRevisionInDeps(): | |
| 67 """Returns the WebKit revision specified in DEPS file. | |
| 68 | |
| 69 Returns: | |
| 70 Revision number as int. -1 in case of error. | |
| 71 """ | |
| 72 for line in _OpenUrl(_DEPS_FILE_URL): | |
| 73 match = _WEBKIT_REVISION_IN_DEPS_RE.search(line) | |
| 74 if match: | |
| 75 return int(match.group(1)) | |
| 76 return -1 | |
| 77 | |
| 78 | |
| 79 class _BuildResult(object): | |
| 80 """Build result for a builder. | |
| 81 | |
| 82 Holds builder name, the last passing revision, the last run revision, and | |
| 83 a list of names of failing tests. Revision nubmer 0 is used to represent | |
| 84 that the revision doesn't exist. | |
| 85 """ | |
| 86 def __init__(self, builder, last_passing_revision, last_run_revision, | |
| 87 failing_tests): | |
| 88 """Constructs build results.""" | |
| 89 self.builder = builder | |
| 90 self.last_passing_revision = last_passing_revision | |
| 91 self.last_run_revision = last_run_revision | |
| 92 self.failing_tests = failing_tests | |
| 93 | |
| 94 | |
| 95 def _BuilderUrlFor(builder, max_builds): | |
| 96 """Constructs the URL for a builder to retrieve the last results.""" | |
| 97 url = ("http://build.chromium.org/p/chromium.webkit/json/builders/%s/builds" % | |
| 98 urllib2.quote(builder)) | |
| 99 if max_builds == -1: | |
| 100 return url + "/_all?as_text=1" | |
| 101 return (url + "?as_text=1&" + | |
| 102 '&'.join(["select=%d" % -i for i in range(1, 1 + max_builds)])) | |
| 103 | |
| 104 | |
| 105 def _ExtractFailingTests(build): | |
| 106 """Extracts failing test names from a build result entry JSON object.""" | |
| 107 failing_tests = [] | |
| 108 for step in build["steps"]: | |
| 109 if step["name"] == _WEBKIT_TESTS: | |
| 110 for text in step["text"]: | |
| 111 prefix = text.find(_TEST_PREFIX) | |
| 112 suffix = text.find(_TEST_SUFFIX) | |
| 113 if prefix != -1 and suffix != -1: | |
| 114 failing_tests += sorted( | |
| 115 text[prefix + len(_TEST_PREFIX): suffix].split(",")) | |
| 116 elif "results" in step: | |
| 117 # Existence of "results" entry seems to mean failure. | |
| 118 failing_tests.append(" ".join(step["text"])) | |
| 119 return failing_tests | |
| 120 | |
| 121 | |
| 122 def _RetrieveBuildResult(builder, max_builds, oldest_revision_to_check): | |
| 123 """Retrieves build results for a builder. | |
| 124 | |
| 125 Checks the last passing revision, the last run revision, and failing tests | |
| 126 for the last builds of a builder. | |
| 127 | |
| 128 Args: | |
| 129 builder: Builder name. | |
| 130 max_builds: Maximum number of builds to check. | |
| 131 oldest_revision_to_check: Oldest WebKit revision to check. | |
| 132 | |
| 133 Returns: | |
| 134 _BuildResult instance. | |
| 135 """ | |
| 136 last_run_revision = 0 | |
| 137 failing_tests = [] | |
| 138 succeeded = False | |
| 139 builds_json = _OpenUrl(_BuilderUrlFor(builder, max_builds)) | |
| 140 if not builds_json: | |
| 141 return _BuildResult(builder, 0, 0, failing_tests) | |
| 142 builds = [(int(value["number"]), value) for unused_key, value | |
| 143 in json.loads(''.join(builds_json)).items() | |
| 144 if value.has_key("number")] | |
| 145 builds.sort() | |
| 146 builds.reverse() | |
| 147 for unused_key, build in builds: | |
| 148 if not build.has_key("text"): | |
| 149 continue | |
| 150 if len(build["text"]) < 2: | |
| 151 continue | |
| 152 if not build.has_key("sourceStamp"): | |
| 153 continue | |
| 154 if build["text"][1] == "successful": | |
| 155 succeeded = True | |
| 156 elif not failing_tests: | |
| 157 failing_tests = _ExtractFailingTests(build) | |
| 158 revision = 0 | |
| 159 if build["sourceStamp"]["branch"] == "trunk": | |
| 160 revision = int(build["sourceStamp"]["changes"][-1]["revision"]) | |
| 161 if revision and not last_run_revision: | |
| 162 last_run_revision = revision | |
| 163 if revision and revision < oldest_revision_to_check: | |
| 164 break | |
| 165 if not succeeded or not revision: | |
| 166 continue | |
| 167 return _BuildResult(builder, revision, last_run_revision, failing_tests) | |
| 168 return _BuildResult(builder, 0, last_run_revision, failing_tests) | |
| 169 | |
| 170 | |
| 171 def _PrintPassingRevisions(results, unused_verbose): | |
| 172 """Prints passing revisions and the range of such revisions. | |
| 173 | |
| 174 Args: | |
| 175 results: A list of build results. | |
| 176 """ | |
| 177 print "**** Passing revisions *****" | |
| 178 min_passing_revision = sys.maxint | |
| 179 max_passing_revision = 0 | |
| 180 for result in results: | |
| 181 if result.last_passing_revision: | |
| 182 min_passing_revision = min(min_passing_revision, | |
| 183 result.last_passing_revision) | |
| 184 max_passing_revision = max(max_passing_revision, | |
| 185 result.last_passing_revision) | |
| 186 print 'The last passing run was at r%d on "%s"' % ( | |
| 187 result.last_passing_revision, result.builder) | |
| 188 else: | |
| 189 print 'No passing runs on "%s"' % result.builder | |
| 190 if max_passing_revision: | |
| 191 print "Passing revision range: r%d - r%d" % ( | |
| 192 min_passing_revision, max_passing_revision) | |
| 193 | |
| 194 | |
| 195 def _PrintFailingRevisions(results, verbose): | |
| 196 """Prints failing revisions and the failing tests. | |
| 197 | |
| 198 Args: | |
| 199 results: A list of build results. | |
| 200 """ | |
| 201 failing_test_to_builders = {} | |
| 202 print "**** Failing revisions *****" | |
| 203 for result in results: | |
| 204 if result.last_run_revision and result.failing_tests: | |
| 205 print ('The last run was at r%d on "%s" and the following %d tests' | |
| 206 ' failed' % (result.last_run_revision, result.builder, | |
| 207 len(result.failing_tests))) | |
| 208 for test in result.failing_tests: | |
| 209 print " " + test | |
| 210 failing_test_to_builders.setdefault(test, set()).add(result.builder) | |
| 211 if verbose: | |
| 212 _PrintFailingTestsForBuilderSubsets(failing_test_to_builders) | |
| 213 | |
| 214 | |
| 215 class _FailingTestsForBuilderSubset(object): | |
| 216 def __init__(self, subset_size): | |
| 217 self._subset_size = subset_size | |
| 218 self._tests = [] | |
| 219 | |
| 220 def SubsetSize(self): | |
| 221 return self._subset_size | |
| 222 | |
| 223 def Tests(self): | |
| 224 return self._tests | |
| 225 | |
| 226 | |
| 227 def _PrintFailingTestsForBuilderSubsets(failing_test_to_builders): | |
| 228 """Prints failing test for builder subsets. | |
| 229 | |
| 230 Prints failing tests for each subset of builders, in descending order of the | |
| 231 set size. | |
| 232 """ | |
| 233 print "**** Failing tests ****" | |
| 234 builders_to_tests = {} | |
| 235 for test in failing_test_to_builders: | |
| 236 builders = sorted(failing_test_to_builders[test]) | |
| 237 subset_name = ", ".join(builders) | |
| 238 tests = builders_to_tests.setdefault( | |
| 239 subset_name, _FailingTestsForBuilderSubset(len(builders))).Tests() | |
| 240 tests.append(test) | |
| 241 # Sort subsets in descending order of size and then name. | |
| 242 builder_subsets = [(builders_to_tests[subset_name].SubsetSize(), subset_name) | |
| 243 for subset_name in builders_to_tests] | |
| 244 for subset_size, subset_name in reversed(sorted(builder_subsets)): | |
| 245 print "** Tests failing for %d builders: %s **" % (subset_size, | |
| 246 subset_name) | |
| 247 for test in sorted(builders_to_tests[subset_name].Tests()): | |
| 248 print test | |
| 249 | |
| 250 | |
| 251 def _ParseOptions(): | |
| 252 """Parses command-line options.""" | |
| 253 parser = optparse.OptionParser(usage="%prog [options] [builders]") | |
| 254 parser.add_option("-m", "--max_builds", type="int", | |
| 255 default=-1, | |
| 256 help="Maximum number of builds to check for each builder." | |
| 257 " Defaults to all builds for which record is" | |
| 258 " available. Checking is ended either when the maximum" | |
| 259 " number is reached, the remaining builds are older" | |
| 260 " than the DEPS WebKit revision, or a passing" | |
| 261 " revision is found.") | |
| 262 parser.add_option("-v", "--verbose", action="store_true", default=False, | |
| 263 dest="verbose") | |
| 264 return parser.parse_args() | |
| 265 | |
| 266 | |
| 267 def _Main(): | |
| 268 """The main function.""" | |
| 269 options, builders = _ParseOptions() | |
| 270 if not builders: | |
| 271 builders = _DEFAULT_BUILDERS | |
| 272 oldest_revision_to_check = _WebkitRevisionInDeps() | |
| 273 if options.max_builds == -1 and oldest_revision_to_check == -1: | |
| 274 options.max_builds = _DEFAULT_MAX_BUILDS | |
| 275 if options.max_builds != -1: | |
| 276 print "Maxium number of builds to check: %d" % options.max_builds | |
| 277 if oldest_revision_to_check != -1: | |
| 278 print "Oldest revision to check: %d" % oldest_revision_to_check | |
| 279 sys.stdout.flush() | |
| 280 results = [] | |
| 281 for builder in builders: | |
| 282 print '"%s"' % builder | |
| 283 sys.stdout.flush() | |
| 284 results.append(_RetrieveBuildResult( | |
| 285 builder, options.max_builds, oldest_revision_to_check)) | |
| 286 _PrintFailingRevisions(results, options.verbose) | |
| 287 _PrintPassingRevisions(results, options.verbose) | |
| 288 | |
| 289 | |
| 290 if __name__ == "__main__": | |
| 291 _Main() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 520373002:
webkit: Remove canary-webkit-revisions.py script. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master