OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python |
| 2 |
| 3 """ |
| 4 Copyright 2013 Google Inc. |
| 5 |
| 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. |
| 8 |
| 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
| 10 """ |
| 11 |
| 12 # System-level imports |
| 13 import argparse |
| 14 import fnmatch |
| 15 import logging |
| 16 import os |
| 17 import time |
| 18 |
| 19 # Must fix up PYTHONPATH before importing from within Skia |
| 20 import rs_fixpypath # pylint: disable=W0611 |
| 21 |
| 22 # Imports from within Skia |
| 23 from py.utils import url_utils |
| 24 import column |
| 25 import gm_json |
| 26 import imagediffdb |
| 27 import imagepair |
| 28 import imagepairset |
| 29 import results |
| 30 |
| 31 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ |
| 32 results.KEY__EXPECTATIONS__BUGS, |
| 33 results.KEY__EXPECTATIONS__IGNOREFAILURE, |
| 34 results.KEY__EXPECTATIONS__REVIEWED, |
| 35 ] |
| 36 FREEFORM_COLUMN_IDS = [ |
| 37 results.KEY__EXTRACOLUMNS__BUILDER, |
| 38 results.KEY__EXTRACOLUMNS__TEST, |
| 39 ] |
| 40 ORDERED_COLUMN_IDS = [ |
| 41 results.KEY__EXTRACOLUMNS__RESULT_TYPE, |
| 42 results.KEY__EXTRACOLUMNS__BUILDER, |
| 43 results.KEY__EXTRACOLUMNS__TEST, |
| 44 results.KEY__EXTRACOLUMNS__CONFIG, |
| 45 ] |
| 46 |
| 47 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
| 48 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') |
| 49 DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' |
| 50 |
| 51 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') |
| 52 |
| 53 |
| 54 class ExpectationComparisons(results.BaseComparisons): |
| 55 """Loads actual and expected GM results into an ImagePairSet. |
| 56 |
| 57 Loads actual and expected results from all builders, except for those skipped |
| 58 by _ignore_builder(). |
| 59 |
| 60 Once this object has been constructed, the results (in self._results[]) |
| 61 are immutable. If you want to update the results based on updated JSON |
| 62 file contents, you will need to create a new ExpectationComparisons object.""" |
| 63 |
| 64 def __init__(self, image_diff_db, actuals_root=results.DEFAULT_ACTUALS_DIR, |
| 65 expected_root=DEFAULT_EXPECTATIONS_DIR, |
| 66 ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE, |
| 67 diff_base_url=None, builder_regex_list=None): |
| 68 """ |
| 69 Args: |
| 70 image_diff_db: instance of ImageDiffDB we use to cache the image diffs |
| 71 actuals_root: root directory containing all actual-results.json files |
| 72 expected_root: root directory containing all expected-results.json files |
| 73 ignore_failures_file: if a file with this name is found within |
| 74 expected_root, ignore failures for any tests listed in the file |
| 75 diff_base_url: base URL within which the client should look for diff |
| 76 images; if not specified, defaults to a "file:///" URL representation |
| 77 of image_diff_db's storage_root |
| 78 builder_regex_list: List of regular expressions specifying which builders |
| 79 we will process. If None, process all builders. |
| 80 """ |
| 81 super(ExpectationComparisons, self).__init__() |
| 82 time_start = int(time.time()) |
| 83 if builder_regex_list != None: |
| 84 self.set_match_builders_pattern_list(builder_regex_list) |
| 85 self._image_diff_db = image_diff_db |
| 86 self._diff_base_url = ( |
| 87 diff_base_url or |
| 88 url_utils.create_filepath_url(image_diff_db.storage_root)) |
| 89 self._actuals_root = actuals_root |
| 90 self._expected_root = expected_root |
| 91 self._ignore_failures_on_these_tests = [] |
| 92 if ignore_failures_file: |
| 93 self._ignore_failures_on_these_tests = ( |
| 94 ExpectationComparisons._read_noncomment_lines( |
| 95 os.path.join(expected_root, ignore_failures_file))) |
| 96 self._load_actual_and_expected() |
| 97 self._timestamp = int(time.time()) |
| 98 logging.info('Results complete; took %d seconds.' % |
| 99 (self._timestamp - time_start)) |
| 100 |
| 101 def edit_expectations(self, modifications): |
| 102 """Edit the expectations stored within this object and write them back |
| 103 to disk. |
| 104 |
| 105 Note that this will NOT update the results stored in self._results[] ; |
| 106 in order to see those updates, you must instantiate a new |
| 107 ExpectationComparisons object based on the (now updated) files on disk. |
| 108 |
| 109 Args: |
| 110 modifications: a list of dictionaries, one for each expectation to update: |
| 111 |
| 112 [ |
| 113 { |
| 114 imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: { |
| 115 results.KEY__EXPECTATIONS__BUGS: [123, 456], |
| 116 results.KEY__EXPECTATIONS__IGNOREFAILURE: false, |
| 117 results.KEY__EXPECTATIONS__REVIEWED: true, |
| 118 }, |
| 119 imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: { |
| 120 results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeFo
rce320M-x86-Debug', |
| 121 results.KEY__EXTRACOLUMNS__CONFIG: '8888', |
| 122 results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix', |
| 123 }, |
| 124 results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10
894408024079689926.png', |
| 125 }, |
| 126 ... |
| 127 ] |
| 128 |
| 129 """ |
| 130 expected_builder_dicts = self._read_builder_dicts_from_root( |
| 131 self._expected_root) |
| 132 for mod in modifications: |
| 133 image_name = results.IMAGE_FILENAME_FORMATTER % ( |
| 134 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] |
| 135 [results.KEY__EXTRACOLUMNS__TEST], |
| 136 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] |
| 137 [results.KEY__EXTRACOLUMNS__CONFIG]) |
| 138 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( |
| 139 mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL]) |
| 140 allowed_digests = [[hash_type, int(hash_digest)]] |
| 141 new_expectations = { |
| 142 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, |
| 143 } |
| 144 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: |
| 145 value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field) |
| 146 if value is not None: |
| 147 new_expectations[field] = value |
| 148 builder_dict = expected_builder_dicts[ |
| 149 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] |
| 150 [results.KEY__EXTRACOLUMNS__BUILDER]] |
| 151 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
| 152 if not builder_expectations: |
| 153 builder_expectations = {} |
| 154 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations |
| 155 builder_expectations[image_name] = new_expectations |
| 156 ExpectationComparisons._write_dicts_to_root( |
| 157 expected_builder_dicts, self._expected_root) |
| 158 |
| 159 @staticmethod |
| 160 def _write_dicts_to_root(meta_dict, root, pattern='*.json'): |
| 161 """Write all per-builder dictionaries within meta_dict to files under |
| 162 the root path. |
| 163 |
| 164 Security note: this will only write to files that already exist within |
| 165 the root path (as found by os.walk() within root), so we don't need to |
| 166 worry about malformed content writing to disk outside of root. |
| 167 However, the data written to those files is not double-checked, so it |
| 168 could contain poisonous data. |
| 169 |
| 170 Args: |
| 171 meta_dict: a builder-keyed meta-dictionary containing all the JSON |
| 172 dictionaries we want to write out |
| 173 root: path to root of directory tree within which to write files |
| 174 pattern: which files to write within root (fnmatch-style pattern) |
| 175 |
| 176 Raises: |
| 177 IOError if root does not refer to an existing directory |
| 178 KeyError if the set of per-builder dictionaries written out was |
| 179 different than expected |
| 180 """ |
| 181 if not os.path.isdir(root): |
| 182 raise IOError('no directory found at path %s' % root) |
| 183 actual_builders_written = [] |
| 184 for dirpath, _, filenames in os.walk(root): |
| 185 for matching_filename in fnmatch.filter(filenames, pattern): |
| 186 builder = os.path.basename(dirpath) |
| 187 per_builder_dict = meta_dict.get(builder) |
| 188 if per_builder_dict is not None: |
| 189 fullpath = os.path.join(dirpath, matching_filename) |
| 190 gm_json.WriteToFile(per_builder_dict, fullpath) |
| 191 actual_builders_written.append(builder) |
| 192 |
| 193 # Check: did we write out the set of per-builder dictionaries we |
| 194 # expected to? |
| 195 expected_builders_written = sorted(meta_dict.keys()) |
| 196 actual_builders_written.sort() |
| 197 if expected_builders_written != actual_builders_written: |
| 198 raise KeyError( |
| 199 'expected to write dicts for builders %s, but actually wrote them ' |
| 200 'for builders %s' % ( |
| 201 expected_builders_written, actual_builders_written)) |
| 202 |
| 203 def _load_actual_and_expected(self): |
| 204 """Loads the results of all tests, across all builders (based on the |
| 205 files within self._actuals_root and self._expected_root), |
| 206 and stores them in self._results. |
| 207 """ |
| 208 logging.info('Reading actual-results JSON files from %s...' % |
| 209 self._actuals_root) |
| 210 actual_builder_dicts = self._read_builder_dicts_from_root( |
| 211 self._actuals_root) |
| 212 logging.info('Reading expected-results JSON files from %s...' % |
| 213 self._expected_root) |
| 214 expected_builder_dicts = self._read_builder_dicts_from_root( |
| 215 self._expected_root) |
| 216 |
| 217 all_image_pairs = imagepairset.ImagePairSet( |
| 218 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, |
| 219 diff_base_url=self._diff_base_url) |
| 220 failing_image_pairs = imagepairset.ImagePairSet( |
| 221 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, |
| 222 diff_base_url=self._diff_base_url) |
| 223 |
| 224 # Override settings for columns that should be filtered using freeform text. |
| 225 for column_id in FREEFORM_COLUMN_IDS: |
| 226 factory = column.ColumnHeaderFactory( |
| 227 header_text=column_id, use_freeform_filter=True) |
| 228 all_image_pairs.set_column_header_factory( |
| 229 column_id=column_id, column_header_factory=factory) |
| 230 failing_image_pairs.set_column_header_factory( |
| 231 column_id=column_id, column_header_factory=factory) |
| 232 |
| 233 all_image_pairs.ensure_extra_column_values_in_summary( |
| 234 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ |
| 235 results.KEY__RESULT_TYPE__FAILED, |
| 236 results.KEY__RESULT_TYPE__FAILUREIGNORED, |
| 237 results.KEY__RESULT_TYPE__NOCOMPARISON, |
| 238 results.KEY__RESULT_TYPE__SUCCEEDED, |
| 239 ]) |
| 240 failing_image_pairs.ensure_extra_column_values_in_summary( |
| 241 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ |
| 242 results.KEY__RESULT_TYPE__FAILED, |
| 243 results.KEY__RESULT_TYPE__FAILUREIGNORED, |
| 244 results.KEY__RESULT_TYPE__NOCOMPARISON, |
| 245 ]) |
| 246 |
| 247 # Only consider builders we have both expected and actual results for. |
| 248 # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results |
| 249 # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug |
| 250 # builder') |
| 251 actual_builder_set = set(actual_builder_dicts.keys()) |
| 252 expected_builder_set = set(expected_builder_dicts.keys()) |
| 253 builders = sorted(actual_builder_set.intersection(expected_builder_set)) |
| 254 |
| 255 num_builders = len(builders) |
| 256 builder_num = 0 |
| 257 for builder in builders: |
| 258 builder_num += 1 |
| 259 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % |
| 260 (builder_num, num_builders, builder)) |
| 261 actual_results_for_this_builder = ( |
| 262 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) |
| 263 for result_type in sorted(actual_results_for_this_builder.keys()): |
| 264 results_of_this_type = actual_results_for_this_builder[result_type] |
| 265 if not results_of_this_type: |
| 266 continue |
| 267 for image_name in sorted(results_of_this_type.keys()): |
| 268 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() |
| 269 actual_image_relative_url = ( |
| 270 ExpectationComparisons._create_relative_url( |
| 271 hashtype_and_digest=results_of_this_type[image_name], |
| 272 test_name=test)) |
| 273 |
| 274 # Default empty expectations; overwrite these if we find any real ones |
| 275 expectations_per_test = None |
| 276 expected_image_relative_url = None |
| 277 expectations_dict = None |
| 278 try: |
| 279 expectations_per_test = ( |
| 280 expected_builder_dicts |
| 281 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) |
| 282 # TODO(epoger): assumes a single allowed digest per test, which is |
| 283 # fine; see https://code.google.com/p/skia/issues/detail?id=1787 |
| 284 expected_image_hashtype_and_digest = ( |
| 285 expectations_per_test |
| 286 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) |
| 287 expected_image_relative_url = ( |
| 288 ExpectationComparisons._create_relative_url( |
| 289 hashtype_and_digest=expected_image_hashtype_and_digest, |
| 290 test_name=test)) |
| 291 expectations_dict = {} |
| 292 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: |
| 293 expectations_dict[field] = expectations_per_test.get(field) |
| 294 except (KeyError, TypeError): |
| 295 # There are several cases in which we would expect to find |
| 296 # no expectations for a given test: |
| 297 # |
| 298 # 1. result_type == NOCOMPARISON |
| 299 # There are no expectations for this test yet! |
| 300 # |
| 301 # 2. alternate rendering mode failures (e.g. serialized) |
| 302 # In cases like |
| 303 # https://code.google.com/p/skia/issues/detail?id=1684 |
| 304 # ('tileimagefilter GM test failing in serialized render mode'), |
| 305 # the gm-actuals will list a failure for the alternate |
| 306 # rendering mode even though we don't have explicit expectations |
| 307 # for the test (the implicit expectation is that it must |
| 308 # render the same in all rendering modes). |
| 309 # |
| 310 # Don't log type 1, because it is common. |
| 311 # Log other types, because they are rare and we should know about |
| 312 # them, but don't throw an exception, because we need to keep our |
| 313 # tools working in the meanwhile! |
| 314 if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: |
| 315 logging.warning('No expectations found for test: %s' % { |
| 316 results.KEY__EXTRACOLUMNS__BUILDER: builder, |
| 317 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, |
| 318 'image_name': image_name, |
| 319 }) |
| 320 |
| 321 # If this test was recently rebaselined, it will remain in |
| 322 # the 'failed' set of actuals until all the bots have |
| 323 # cycled (although the expectations have indeed been set |
| 324 # from the most recent actuals). Treat these as successes |
| 325 # instead of failures. |
| 326 # |
| 327 # TODO(epoger): Do we need to do something similar in |
| 328 # other cases, such as when we have recently marked a test |
| 329 # as ignoreFailure but it still shows up in the 'failed' |
| 330 # category? Maybe we should not rely on the result_type |
| 331 # categories recorded within the gm_actuals AT ALL, and |
| 332 # instead evaluate the result_type ourselves based on what |
| 333 # we see in expectations vs actual checksum? |
| 334 if expected_image_relative_url == actual_image_relative_url: |
| 335 updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED |
| 336 elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and |
| 337 (test in self._ignore_failures_on_these_tests)): |
| 338 updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED |
| 339 else: |
| 340 updated_result_type = result_type |
| 341 extra_columns_dict = { |
| 342 results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type, |
| 343 results.KEY__EXTRACOLUMNS__BUILDER: builder, |
| 344 results.KEY__EXTRACOLUMNS__TEST: test, |
| 345 results.KEY__EXTRACOLUMNS__CONFIG: config, |
| 346 } |
| 347 try: |
| 348 image_pair = imagepair.ImagePair( |
| 349 image_diff_db=self._image_diff_db, |
| 350 imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
| 351 imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
| 352 imageA_relative_url=expected_image_relative_url, |
| 353 imageB_relative_url=actual_image_relative_url, |
| 354 expectations=expectations_dict, |
| 355 extra_columns=extra_columns_dict) |
| 356 all_image_pairs.add_image_pair(image_pair) |
| 357 if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: |
| 358 failing_image_pairs.add_image_pair(image_pair) |
| 359 except Exception: |
| 360 logging.exception('got exception while creating new ImagePair') |
| 361 |
| 362 # pylint: disable=W0201 |
| 363 self._results = { |
| 364 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict( |
| 365 column_ids_in_order=ORDERED_COLUMN_IDS), |
| 366 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict( |
| 367 column_ids_in_order=ORDERED_COLUMN_IDS), |
| 368 } |
| 369 |
| 370 |
| 371 def main(): |
| 372 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', |
| 373 datefmt='%m/%d/%Y %H:%M:%S', |
| 374 level=logging.INFO) |
| 375 parser = argparse.ArgumentParser() |
| 376 parser.add_argument( |
| 377 '--actuals', default=results.DEFAULT_ACTUALS_DIR, |
| 378 help='Directory containing all actual-result JSON files; defaults to ' |
| 379 '\'%(default)s\' .') |
| 380 parser.add_argument( |
| 381 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, |
| 382 help='Directory containing all expected-result JSON files; defaults to ' |
| 383 '\'%(default)s\' .') |
| 384 parser.add_argument( |
| 385 '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE, |
| 386 help='If a file with this name is found within the EXPECTATIONS dir, ' |
| 387 'ignore failures for any tests listed in the file; defaults to ' |
| 388 '\'%(default)s\' .') |
| 389 parser.add_argument( |
| 390 '--outfile', required=True, |
| 391 help='File to write result summary into, in JSON format.') |
| 392 parser.add_argument( |
| 393 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, |
| 394 help='Which result types to include. Defaults to \'%(default)s\'; ' |
| 395 'must be one of ' + |
| 396 str([results.KEY__HEADER__RESULTS_FAILURES, |
| 397 results.KEY__HEADER__RESULTS_ALL])) |
| 398 parser.add_argument( |
| 399 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, |
| 400 help='Directory within which to download images and generate diffs; ' |
| 401 'defaults to \'%(default)s\' .') |
| 402 args = parser.parse_args() |
| 403 image_diff_db = imagediffdb.ImageDiffDB(storage_root=args.workdir) |
| 404 results_obj = ExpectationComparisons( |
| 405 image_diff_db=image_diff_db, |
| 406 actuals_root=args.actuals, |
| 407 expected_root=args.expectations, |
| 408 ignore_failures_file=args.ignore_failures_file) |
| 409 gm_json.WriteToFile( |
| 410 results_obj.get_packaged_results_of_type(results_type=args.results), |
| 411 args.outfile) |
| 412 |
| 413 |
| 414 if __name__ == '__main__': |
| 415 main() |
OLD | NEW |