| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2009 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 import logging | |
| 6 import os | |
| 7 import subprocess | |
| 8 import sys | |
| 9 import time | |
| 10 import urllib2 | |
| 11 import xml.dom.minidom | |
| 12 | |
| 13 from layout_package import path_utils | |
| 14 from layout_package import test_expectations | |
| 15 | |
| 16 sys.path.append(path_utils.PathFromBase('third_party')) | |
| 17 import simplejson | |
| 18 | |
| 19 | |
| 20 class JSONResultsGenerator(object): | |
| 21 | |
| 22 MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 | |
| 23 # Min time (seconds) that will be added to the JSON. | |
| 24 MIN_TIME = 1 | |
| 25 JSON_PREFIX = "ADD_RESULTS(" | |
| 26 JSON_SUFFIX = ");" | |
| 27 PASS_RESULT = "P" | |
| 28 SKIP_RESULT = "X" | |
| 29 NO_DATA_RESULT = "N" | |
| 30 VERSION = 3 | |
| 31 VERSION_KEY = "version" | |
| 32 RESULTS = "results" | |
| 33 TIMES = "times" | |
| 34 BUILD_NUMBERS = "buildNumbers" | |
| 35 WEBKIT_SVN = "webkitRevision" | |
| 36 CHROME_SVN = "chromeRevision" | |
| 37 TIME = "secondsSinceEpoch" | |
| 38 TESTS = "tests" | |
| 39 | |
| 40 FIXABLE_COUNT = "fixableCount" | |
| 41 FIXABLE = "fixableCounts" | |
| 42 ALL_FIXABLE_COUNT = "allFixableCount" | |
| 43 | |
| 44 # Note that we omit test_expectations.FAIL from this list because | |
| 45 # it should never show up (it's a legacy input expectation, never | |
| 46 # an output expectation). | |
| 47 FAILURE_TO_CHAR = {test_expectations.CRASH: "C", | |
| 48 test_expectations.TIMEOUT: "T", | |
| 49 test_expectations.IMAGE: "I", | |
| 50 test_expectations.TEXT: "F", | |
| 51 test_expectations.MISSING: "O", | |
| 52 test_expectations.IMAGE_PLUS_TEXT: "Z"} | |
| 53 FAILURE_CHARS = FAILURE_TO_CHAR.values() | |
| 54 | |
| 55 RESULTS_FILENAME = "results.json" | |
| 56 | |
| 57 def __init__(self, builder_name, build_name, build_number, | |
| 58 results_file_base_path, builder_base_url, | |
| 59 test_timings, failures, passed_tests, skipped_tests, all_tests): | |
| 60 """Modifies the results.json file. Grabs it off the archive directory | |
| 61 if it is not found locally. | |
| 62 | |
| 63 Args | |
| 64 builder_name: the builder name (e.g. Webkit). | |
| 65 build_name: the build name (e.g. webkit-rel). | |
| 66 build_number: the build number. | |
| 67 results_file_base_path: Absolute path to the directory containing the | |
| 68 results json file. | |
| 69 builder_base_url: the URL where we have the archived test results. | |
| 70 test_timings: Map of test name to a test_run-time. | |
| 71 failures: Map of test name to a failure type (of test_expectations). | |
| 72 passed_tests: A set containing all the passed tests. | |
| 73 skipped_tests: A set containing all the skipped tests. | |
| 74 all_tests: List of all the tests that were run. This should not | |
| 75 include skipped tests. | |
| 76 """ | |
| 77 self._builder_name = builder_name | |
| 78 self._build_name = build_name | |
| 79 self._build_number = build_number | |
| 80 self._builder_base_url = builder_base_url | |
| 81 self._results_file_path = os.path.join(results_file_base_path, | |
| 82 self.RESULTS_FILENAME) | |
| 83 self._test_timings = test_timings | |
| 84 self._failures = failures | |
| 85 self._passed_tests = passed_tests | |
| 86 self._skipped_tests = skipped_tests | |
| 87 self._all_tests = all_tests | |
| 88 | |
| 89 self._GenerateJSONOutput() | |
| 90 | |
| 91 def _GenerateJSONOutput(self): | |
| 92 """Generates the JSON output file.""" | |
| 93 json = self._GetJSON() | |
| 94 if json: | |
| 95 results_file = open(self._results_file_path, "w") | |
| 96 results_file.write(json) | |
| 97 results_file.close() | |
| 98 | |
| 99 def _GetSVNRevision(self, in_directory=None): | |
| 100 """Returns the svn revision for the given directory. | |
| 101 | |
| 102 Args: | |
| 103 in_directory: The directory where svn is to be run. | |
| 104 """ | |
| 105 output = subprocess.Popen(["svn", "info", "--xml"], | |
| 106 cwd=in_directory, | |
| 107 shell=(sys.platform == 'win32'), | |
| 108 stdout=subprocess.PIPE).communicate()[0] | |
| 109 try: | |
| 110 dom = xml.dom.minidom.parseString(output) | |
| 111 return dom.getElementsByTagName('entry')[0].getAttribute( | |
| 112 'revision') | |
| 113 except xml.parsers.expat.ExpatError: | |
| 114 return "" | |
| 115 | |
| 116 def _GetArchivedJSONResults(self): | |
| 117 """Reads old results JSON file if it exists. | |
| 118 Returns (archived_results, error) tuple where error is None if results | |
| 119 were successfully read. | |
| 120 """ | |
| 121 results_json = {} | |
| 122 old_results = None | |
| 123 error = None | |
| 124 | |
| 125 if os.path.exists(self._results_file_path): | |
| 126 old_results_file = open(self._results_file_path, "r") | |
| 127 old_results = old_results_file.read() | |
| 128 elif self._builder_base_url: | |
| 129 # Check if we have the archived JSON file on the buildbot server. | |
| 130 results_file_url = (self._builder_base_url + | |
| 131 self._build_name + "/" + self.RESULTS_FILENAME) | |
| 132 logging.error("Local results.json file does not exist. Grabbing " | |
| 133 "it off the archive at " + results_file_url) | |
| 134 | |
| 135 try: | |
| 136 results_file = urllib2.urlopen(results_file_url) | |
| 137 info = results_file.info() | |
| 138 old_results = results_file.read() | |
| 139 except urllib2.HTTPError, http_error: | |
| 140 # A non-4xx status code means the bot is hosed for some reason | |
| 141 # and we can't grab the results.json file off of it. | |
| 142 if (http_error.code < 400 and http_error.code >= 500): | |
| 143 error = http_error | |
| 144 except urllib2.URLError, url_error: | |
| 145 error = url_error | |
| 146 | |
| 147 if old_results: | |
| 148 # Strip the prefix and suffix so we can get the actual JSON object. | |
| 149 old_results = old_results[len(self.JSON_PREFIX): | |
| 150 len(old_results) - len(self.JSON_SUFFIX)] | |
| 151 | |
| 152 try: | |
| 153 results_json = simplejson.loads(old_results) | |
| 154 except: | |
| 155 logging.debug("results.json was not valid JSON. Clobbering.") | |
| 156 # The JSON file is not valid JSON. Just clobber the results. | |
| 157 results_json = {} | |
| 158 else: | |
| 159 logging.debug('Old JSON results do not exist. Starting fresh.') | |
| 160 results_json = {} | |
| 161 | |
| 162 return results_json, error | |
| 163 | |
| 164 def _GetJSON(self): | |
| 165 """Gets the results for the results.json file.""" | |
| 166 results_json, error = self._GetArchivedJSONResults() | |
| 167 if error: | |
| 168 # If there was an error don't write a results.json | |
| 169 # file at all as it would lose all the information on the bot. | |
| 170 logging.error("Archive directory is inaccessible. Not modifying " | |
| 171 "or clobbering the results.json file: " + str(error)) | |
| 172 return None | |
| 173 | |
| 174 builder_name = self._builder_name | |
| 175 if results_json and builder_name not in results_json: | |
| 176 logging.debug("Builder name (%s) is not in the results.json file." | |
| 177 % builder_name) | |
| 178 | |
| 179 self._ConvertJSONToCurrentVersion(results_json) | |
| 180 | |
| 181 if builder_name not in results_json: | |
| 182 results_json[builder_name] = self._CreateResultsForBuilderJSON() | |
| 183 | |
| 184 results_for_builder = results_json[builder_name] | |
| 185 | |
| 186 self._InsertGenericMetadata(results_for_builder) | |
| 187 | |
| 188 self._InsertFailureSummaries(results_for_builder) | |
| 189 | |
| 190 # Update the all failing tests with result type and time. | |
| 191 tests = results_for_builder[self.TESTS] | |
| 192 all_failing_tests = set(self._failures.iterkeys()) | |
| 193 all_failing_tests.update(tests.iterkeys()) | |
| 194 for test in all_failing_tests: | |
| 195 self._InsertTestTimeAndResult(test, tests) | |
| 196 | |
| 197 # Specify separators in order to get compact encoding. | |
| 198 results_str = simplejson.dumps(results_json, separators=(',', ':')) | |
| 199 return self.JSON_PREFIX + results_str + self.JSON_SUFFIX | |
| 200 | |
| 201 def _InsertFailureSummaries(self, results_for_builder): | |
| 202 """Inserts aggregate pass/failure statistics into the JSON. | |
| 203 This method reads self._skipped_tests, self._passed_tests and | |
| 204 self._failures and inserts FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT | |
| 205 entries. | |
| 206 | |
| 207 Args: | |
| 208 results_for_builder: Dictionary containing the test results for a | |
| 209 single builder. | |
| 210 """ | |
| 211 # Insert the number of tests that failed. | |
| 212 self._InsertItemIntoRawList(results_for_builder, | |
| 213 len(set(self._failures.keys()) | self._skipped_tests), | |
| 214 self.FIXABLE_COUNT) | |
| 215 | |
| 216 # Create a pass/skip/failure summary dictionary. | |
| 217 entry = {} | |
| 218 entry[self.SKIP_RESULT] = len(self._skipped_tests) | |
| 219 entry[self.PASS_RESULT] = len(self._passed_tests) | |
| 220 get = entry.get | |
| 221 for failure_type in self._failures.values(): | |
| 222 failure_char = self.FAILURE_TO_CHAR[failure_type] | |
| 223 entry[failure_char] = get(failure_char, 0) + 1 | |
| 224 | |
| 225 # Insert the pass/skip/failure summary dictionary. | |
| 226 self._InsertItemIntoRawList(results_for_builder, entry, self.FIXABLE) | |
| 227 | |
| 228 # Insert the number of all the tests that are supposed to pass. | |
| 229 self._InsertItemIntoRawList(results_for_builder, | |
| 230 len(self._skipped_tests | self._all_tests), | |
| 231 self.ALL_FIXABLE_COUNT) | |
| 232 | |
| 233 def _InsertItemIntoRawList(self, results_for_builder, item, key): | |
| 234 """Inserts the item into the list with the given key in the results for | |
| 235 this builder. Creates the list if no such list exists. | |
| 236 | |
| 237 Args: | |
| 238 results_for_builder: Dictionary containing the test results for a | |
| 239 single builder. | |
| 240 item: Number or string to insert into the list. | |
| 241 key: Key in results_for_builder for the list to insert into. | |
| 242 """ | |
| 243 if key in results_for_builder: | |
| 244 raw_list = results_for_builder[key] | |
| 245 else: | |
| 246 raw_list = [] | |
| 247 | |
| 248 raw_list.insert(0, item) | |
| 249 raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] | |
| 250 results_for_builder[key] = raw_list | |
| 251 | |
| 252 def _InsertItemRunLengthEncoded(self, item, encoded_results): | |
| 253 """Inserts the item into the run-length encoded results. | |
| 254 | |
| 255 Args: | |
| 256 item: String or number to insert. | |
| 257 encoded_results: run-length encoded results. An array of arrays, e.g. | |
| 258 [[3,'A'],[1,'Q']] encodes AAAQ. | |
| 259 """ | |
| 260 if len(encoded_results) and item == encoded_results[0][1]: | |
| 261 num_results = encoded_results[0][0] | |
| 262 if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: | |
| 263 encoded_results[0][0] = num_results + 1 | |
| 264 else: | |
| 265 # Use a list instead of a class for the run-length encoding since | |
| 266 # we want the serialized form to be concise. | |
| 267 encoded_results.insert(0, [1, item]) | |
| 268 | |
| 269 def _InsertGenericMetadata(self, results_for_builder): | |
| 270 """ Inserts generic metadata (such as version number, current time etc) | |
| 271 into the JSON. | |
| 272 | |
| 273 Args: | |
| 274 results_for_builder: Dictionary containing the test results for | |
| 275 a single builder. | |
| 276 """ | |
| 277 self._InsertItemIntoRawList(results_for_builder, | |
| 278 self._build_number, self.BUILD_NUMBERS) | |
| 279 | |
| 280 path_to_webkit = path_utils.PathFromBase('third_party', 'WebKit', | |
| 281 'WebCore') | |
| 282 self._InsertItemIntoRawList(results_for_builder, | |
| 283 self._GetSVNRevision(path_to_webkit), | |
| 284 self.WEBKIT_SVN) | |
| 285 | |
| 286 path_to_chrome_base = path_utils.PathFromBase() | |
| 287 self._InsertItemIntoRawList(results_for_builder, | |
| 288 self._GetSVNRevision(path_to_chrome_base), | |
| 289 self.CHROME_SVN) | |
| 290 | |
| 291 self._InsertItemIntoRawList(results_for_builder, | |
| 292 int(time.time()), | |
| 293 self.TIME) | |
| 294 | |
| 295 def _InsertTestTimeAndResult(self, test_name, tests): | |
| 296 """ Insert a test item with its results to the given tests dictionary. | |
| 297 | |
| 298 Args: | |
| 299 tests: Dictionary containing test result entries. | |
| 300 """ | |
| 301 | |
| 302 result = JSONResultsGenerator.PASS_RESULT | |
| 303 time = 0 | |
| 304 | |
| 305 if test_name not in self._all_tests: | |
| 306 result = JSONResultsGenerator.NO_DATA_RESULT | |
| 307 | |
| 308 if test_name in self._failures: | |
| 309 result = self.FAILURE_TO_CHAR[self._failures[test_name]] | |
| 310 | |
| 311 if test_name in self._test_timings: | |
| 312 # Floor for now to get time in seconds. | |
| 313 time = int(self._test_timings[test_name]) | |
| 314 | |
| 315 if test_name not in tests: | |
| 316 tests[test_name] = self._CreateResultsAndTimesJSON() | |
| 317 | |
| 318 thisTest = tests[test_name] | |
| 319 self._InsertItemRunLengthEncoded(result, thisTest[self.RESULTS]) | |
| 320 self._InsertItemRunLengthEncoded(time, thisTest[self.TIMES]) | |
| 321 self._NormalizeResultsJSON(thisTest, test_name, tests) | |
| 322 | |
| 323 def _ConvertJSONToCurrentVersion(self, results_json): | |
| 324 """If the JSON does not match the current version, converts it to the | |
| 325 current version and adds in the new version number. | |
| 326 """ | |
| 327 if (self.VERSION_KEY in results_json and | |
| 328 results_json[self.VERSION_KEY] == self.VERSION): | |
| 329 return | |
| 330 | |
| 331 results_json[self.VERSION_KEY] = self.VERSION | |
| 332 | |
| 333 def _CreateResultsAndTimesJSON(self): | |
| 334 results_and_times = {} | |
| 335 results_and_times[self.RESULTS] = [] | |
| 336 results_and_times[self.TIMES] = [] | |
| 337 return results_and_times | |
| 338 | |
| 339 def _CreateResultsForBuilderJSON(self): | |
| 340 results_for_builder = {} | |
| 341 results_for_builder[self.TESTS] = {} | |
| 342 return results_for_builder | |
| 343 | |
| 344 def _RemoveItemsOverMaxNumberOfBuilds(self, encoded_list): | |
| 345 """Removes items from the run-length encoded list after the final | |
| 346 item that exceeds the max number of builds to track. | |
| 347 | |
| 348 Args: | |
| 349 encoded_results: run-length encoded results. An array of arrays, e.g. | |
| 350 [[3,'A'],[1,'Q']] encodes AAAQ. | |
| 351 """ | |
| 352 num_builds = 0 | |
| 353 index = 0 | |
| 354 for result in encoded_list: | |
| 355 num_builds = num_builds + result[0] | |
| 356 index = index + 1 | |
| 357 if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: | |
| 358 return encoded_list[:index] | |
| 359 return encoded_list | |
| 360 | |
| 361 def _NormalizeResultsJSON(self, test, test_name, tests): | |
| 362 """ Prune tests where all runs pass or tests that no longer exist and | |
| 363 truncate all results to maxNumberOfBuilds. | |
| 364 | |
| 365 Args: | |
| 366 test: ResultsAndTimes object for this test. | |
| 367 test_name: Name of the test. | |
| 368 tests: The JSON object with all the test results for this builder. | |
| 369 """ | |
| 370 test[self.RESULTS] = self._RemoveItemsOverMaxNumberOfBuilds( | |
| 371 test[self.RESULTS]) | |
| 372 test[self.TIMES] = self._RemoveItemsOverMaxNumberOfBuilds( | |
| 373 test[self.TIMES]) | |
| 374 | |
| 375 is_all_pass = self._IsResultsAllOfType(test[self.RESULTS], | |
| 376 self.PASS_RESULT) | |
| 377 is_all_no_data = self._IsResultsAllOfType(test[self.RESULTS], | |
| 378 self.NO_DATA_RESULT) | |
| 379 max_time = max([time[1] for time in test[self.TIMES]]) | |
| 380 | |
| 381 # Remove all passes/no-data from the results to reduce noise and | |
| 382 # filesize. If a test passes every run, but takes > MIN_TIME to run, | |
| 383 # don't throw away the data. | |
| 384 if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME): | |
| 385 del tests[test_name] | |
| 386 | |
| 387 def _IsResultsAllOfType(self, results, type): | |
| 388 """Returns whether all the results are of the given type | |
| 389 (e.g. all passes).""" | |
| 390 return len(results) == 1 and results[0][1] == type | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 545145:
Move the layout test scripts into a 'webkitpy' subdirectory in preparation... (Closed)
Base URL: svn://chrome-svn/chrome/trunk/src/