OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python |
| 2 |
| 3 """ |
| 4 Copyright 2013 Google Inc. |
| 5 |
| 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. |
| 8 |
| 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
| 10 """ |
| 11 |
| 12 # System-level imports |
| 13 import fnmatch |
| 14 import os |
| 15 import re |
| 16 |
| 17 # Must fix up PYTHONPATH before importing from within Skia |
| 18 import rs_fixpypath # pylint: disable=W0611 |
| 19 |
| 20 # Imports from within Skia |
| 21 import gm_json |
| 22 import imagepairset |
| 23 |
| 24 # Keys used to link an image to a particular GM test. |
| 25 # NOTE: Keep these in sync with static/constants.js |
| 26 VALUE__HEADER__SCHEMA_VERSION = 5 |
| 27 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS |
| 28 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE |
| 29 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED |
| 30 KEY__EXTRACOLUMNS__BUILDER = 'builder' |
| 31 KEY__EXTRACOLUMNS__CONFIG = 'config' |
| 32 KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType' |
| 33 KEY__EXTRACOLUMNS__TEST = 'test' |
| 34 KEY__HEADER__DATAHASH = 'dataHash' |
| 35 KEY__HEADER__IS_EDITABLE = 'isEditable' |
| 36 KEY__HEADER__IS_EXPORTED = 'isExported' |
| 37 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' |
| 38 KEY__HEADER__RESULTS_ALL = 'all' |
| 39 KEY__HEADER__RESULTS_FAILURES = 'failures' |
| 40 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' |
| 41 KEY__HEADER__SET_A_DESCRIPTIONS = 'setA' |
| 42 KEY__HEADER__SET_B_DESCRIPTIONS = 'setB' |
| 43 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' |
| 44 KEY__HEADER__TIME_UPDATED = 'timeUpdated' |
| 45 KEY__HEADER__TYPE = 'type' |
| 46 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED |
| 47 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED |
| 48 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON |
| 49 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED |
| 50 KEY__SET_DESCRIPTIONS__DIR = 'dir' |
| 51 KEY__SET_DESCRIPTIONS__REPO_REVISION = 'repoRevision' |
| 52 KEY__SET_DESCRIPTIONS__SECTION = 'section' |
| 53 |
| 54 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
| 55 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) |
| 56 |
| 57 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) |
| 58 DEFAULT_ACTUALS_DIR = '.gm-actuals' |
| 59 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( |
| 60 PARENT_DIRECTORY, '.generated-images') |
| 61 |
| 62 # Define the default set of builders we will process expectations/actuals for. |
| 63 # This allows us to ignore builders for which we don't maintain expectations |
| 64 # (trybots, Valgrind, ASAN, TSAN), and avoid problems like |
| 65 # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server |
| 66 # produces error when trying to add baselines for ASAN/TSAN builders') |
| 67 DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*'] |
| 68 DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [ |
| 69 '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*'] |
| 70 |
| 71 |
| 72 class BaseComparisons(object): |
| 73 """Base class for generating summary of comparisons between two image sets. |
| 74 """ |
| 75 |
| 76 def __init__(self): |
| 77 """Base constructor; most subclasses will override.""" |
| 78 self._setA_descriptions = None |
| 79 self._setB_descriptions = None |
| 80 |
| 81 def get_results_of_type(self, results_type): |
| 82 """Return results of some/all tests (depending on 'results_type' parameter). |
| 83 |
| 84 Args: |
| 85 results_type: string describing which types of results to include; must |
| 86 be one of the RESULTS_* constants |
| 87 |
| 88 Results are returned in a dictionary as output by ImagePairSet.as_dict(). |
| 89 """ |
| 90 return self._results[results_type] |
| 91 |
| 92 def get_packaged_results_of_type(self, results_type, reload_seconds=None, |
| 93 is_editable=False, is_exported=True): |
| 94 """Package the results of some/all tests as a complete response_dict. |
| 95 |
| 96 Args: |
| 97 results_type: string indicating which set of results to return; |
| 98 must be one of the RESULTS_* constants |
| 99 reload_seconds: if specified, note that new results may be available once |
| 100 these results are reload_seconds old |
| 101 is_editable: whether clients are allowed to submit new baselines |
| 102 is_exported: whether these results are being made available to other |
| 103 network hosts |
| 104 """ |
| 105 response_dict = self._results[results_type] |
| 106 time_updated = self.get_timestamp() |
| 107 header_dict = { |
| 108 KEY__HEADER__SCHEMA_VERSION: ( |
| 109 VALUE__HEADER__SCHEMA_VERSION), |
| 110 |
| 111 # Timestamps: |
| 112 # 1. when this data was last updated |
| 113 # 2. when the caller should check back for new data (if ever) |
| 114 KEY__HEADER__TIME_UPDATED: time_updated, |
| 115 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
| 116 (time_updated+reload_seconds) if reload_seconds else None), |
| 117 |
| 118 # The type we passed to get_results_of_type() |
| 119 KEY__HEADER__TYPE: results_type, |
| 120 |
| 121 # Hash of dataset, which the client must return with any edits-- |
| 122 # this ensures that the edits were made to a particular dataset. |
| 123 KEY__HEADER__DATAHASH: str(hash(repr( |
| 124 response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))), |
| 125 |
| 126 # Whether the server will accept edits back. |
| 127 KEY__HEADER__IS_EDITABLE: is_editable, |
| 128 |
| 129 # Whether the service is accessible from other hosts. |
| 130 KEY__HEADER__IS_EXPORTED: is_exported, |
| 131 } |
| 132 if self._setA_descriptions: |
| 133 header_dict[KEY__HEADER__SET_A_DESCRIPTIONS] = self._setA_descriptions |
| 134 if self._setB_descriptions: |
| 135 header_dict[KEY__HEADER__SET_B_DESCRIPTIONS] = self._setB_descriptions |
| 136 response_dict[imagepairset.KEY__ROOT__HEADER] = header_dict |
| 137 return response_dict |
| 138 |
| 139 def get_timestamp(self): |
| 140 """Return the time at which this object was created, in seconds past epoch |
| 141 (UTC). |
| 142 """ |
| 143 return self._timestamp |
| 144 |
| 145 _match_builders_pattern_list = [ |
| 146 re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST] |
| 147 _skip_builders_pattern_list = [ |
| 148 re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST] |
| 149 |
| 150 def set_match_builders_pattern_list(self, pattern_list): |
| 151 """Override the default set of builders we should process. |
| 152 |
| 153 The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST . |
| 154 |
| 155 Note that skip_builders_pattern_list overrides this; regardless of whether a |
| 156 builder is in the "match" list, if it's in the "skip" list, we will skip it. |
| 157 |
| 158 Args: |
| 159 pattern_list: list of regex patterns; process builders that match any |
| 160 entry within this list |
| 161 """ |
| 162 if pattern_list == None: |
| 163 pattern_list = [] |
| 164 self._match_builders_pattern_list = [re.compile(p) for p in pattern_list] |
| 165 |
| 166 def set_skip_builders_pattern_list(self, pattern_list): |
| 167 """Override the default set of builders we should skip while processing. |
| 168 |
| 169 The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST . |
| 170 |
| 171 This overrides match_builders_pattern_list; regardless of whether a |
| 172 builder is in the "match" list, if it's in the "skip" list, we will skip it. |
| 173 |
| 174 Args: |
| 175 pattern_list: list of regex patterns; skip builders that match any |
| 176 entry within this list |
| 177 """ |
| 178 if pattern_list == None: |
| 179 pattern_list = [] |
| 180 self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list] |
| 181 |
| 182 def _ignore_builder(self, builder): |
| 183 """Returns True if we should skip processing this builder. |
| 184 |
| 185 Args: |
| 186 builder: name of this builder, as a string |
| 187 |
| 188 Returns: |
| 189 True if we should ignore expectations and actuals for this builder. |
| 190 """ |
| 191 for pattern in self._skip_builders_pattern_list: |
| 192 if pattern.match(builder): |
| 193 return True |
| 194 for pattern in self._match_builders_pattern_list: |
| 195 if pattern.match(builder): |
| 196 return False |
| 197 return True |
| 198 |
| 199 def _read_builder_dicts_from_root(self, root, pattern='*.json'): |
| 200 """Read all JSON dictionaries within a directory tree. |
| 201 |
| 202 Skips any dictionaries belonging to a builder we have chosen to ignore. |
| 203 |
| 204 Args: |
| 205 root: path to root of directory tree |
| 206 pattern: which files to read within root (fnmatch-style pattern) |
| 207 |
| 208 Returns: |
| 209 A meta-dictionary containing all the JSON dictionaries found within |
| 210 the directory tree, keyed by builder name (the basename of the directory |
| 211 where each JSON dictionary was found). |
| 212 |
| 213 Raises: |
| 214 IOError if root does not refer to an existing directory |
| 215 """ |
| 216 # I considered making this call read_dicts_from_root(), but I decided |
| 217 # it was better to prune out the ignored builders within the os.walk(). |
| 218 if not os.path.isdir(root): |
| 219 raise IOError('no directory found at path %s' % root) |
| 220 meta_dict = {} |
| 221 for dirpath, _, filenames in os.walk(root): |
| 222 for matching_filename in fnmatch.filter(filenames, pattern): |
| 223 builder = os.path.basename(dirpath) |
| 224 if self._ignore_builder(builder): |
| 225 continue |
| 226 full_path = os.path.join(dirpath, matching_filename) |
| 227 meta_dict[builder] = gm_json.LoadFromFile(full_path) |
| 228 return meta_dict |
| 229 |
| 230 @staticmethod |
| 231 def read_dicts_from_root(root, pattern='*.json'): |
| 232 """Read all JSON dictionaries within a directory tree. |
| 233 |
| 234 TODO(stephana): Factor this out into a utility module, as a standalone |
| 235 function (not part of a class). |
| 236 |
| 237 Args: |
| 238 root: path to root of directory tree |
| 239 pattern: which files to read within root (fnmatch-style pattern) |
| 240 |
| 241 Returns: |
| 242 A meta-dictionary containing all the JSON dictionaries found within |
| 243 the directory tree, keyed by the pathname (relative to root) of each JSON |
| 244 dictionary. |
| 245 |
| 246 Raises: |
| 247 IOError if root does not refer to an existing directory |
| 248 """ |
| 249 if not os.path.isdir(root): |
| 250 raise IOError('no directory found at path %s' % root) |
| 251 meta_dict = {} |
| 252 for abs_dirpath, _, filenames in os.walk(root): |
| 253 rel_dirpath = os.path.relpath(abs_dirpath, root) |
| 254 for matching_filename in fnmatch.filter(filenames, pattern): |
| 255 abs_path = os.path.join(abs_dirpath, matching_filename) |
| 256 rel_path = os.path.join(rel_dirpath, matching_filename) |
| 257 meta_dict[rel_path] = gm_json.LoadFromFile(abs_path) |
| 258 return meta_dict |
| 259 |
| 260 @staticmethod |
| 261 def _read_noncomment_lines(path): |
| 262 """Return a list of all noncomment lines within a file. |
| 263 |
| 264 (A "noncomment" line is one that does not start with a '#'.) |
| 265 |
| 266 Args: |
| 267 path: path to file |
| 268 """ |
| 269 lines = [] |
| 270 with open(path, 'r') as fh: |
| 271 for line in fh: |
| 272 if not line.startswith('#'): |
| 273 lines.append(line.strip()) |
| 274 return lines |
| 275 |
| 276 @staticmethod |
| 277 def _create_relative_url(hashtype_and_digest, test_name): |
| 278 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. |
| 279 |
| 280 If we don't have a record of this image, returns None. |
| 281 |
| 282 Args: |
| 283 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we |
| 284 don't have a record of this image |
| 285 test_name: string; name of the GM test that created this image |
| 286 """ |
| 287 if not hashtype_and_digest: |
| 288 return None |
| 289 return gm_json.CreateGmRelativeUrl( |
| 290 test_name=test_name, |
| 291 hash_type=hashtype_and_digest[0], |
| 292 hash_digest=hashtype_and_digest[1]) |
| 293 |
| 294 @staticmethod |
| 295 def combine_subdicts(input_dict): |
| 296 """ Flatten out a dictionary structure by one level. |
| 297 |
| 298 Input: |
| 299 { |
| 300 KEY_A1 : { |
| 301 KEY_B1 : VALUE_B1, |
| 302 }, |
| 303 KEY_A2 : { |
| 304 KEY_B2 : VALUE_B2, |
| 305 } |
| 306 } |
| 307 |
| 308 Output: |
| 309 { |
| 310 KEY_B1 : VALUE_B1, |
| 311 KEY_B2 : VALUE_B2, |
| 312 } |
| 313 |
| 314 If this would result in any repeated keys, it will raise an Exception. |
| 315 """ |
| 316 output_dict = {} |
| 317 for subdict in input_dict.values(): |
| 318 for subdict_key, subdict_value in subdict.iteritems(): |
| 319 if subdict_key in output_dict: |
| 320 raise Exception('duplicate key %s in combine_subdicts' % subdict_key) |
| 321 output_dict[subdict_key] = subdict_value |
| 322 return output_dict |
| 323 |
| 324 @staticmethod |
| 325 def get_default(input_dict, default_value, *keys): |
| 326 """Returns input_dict[key1][key2][...], or default_value. |
| 327 |
| 328 If input_dict is None, or any key is missing along the way, this returns |
| 329 default_value. |
| 330 |
| 331 Args: |
| 332 input_dict: dictionary to look within |
| 333 key: key indicating which value to return from input_dict |
| 334 default_value: value to return if input_dict is None or any key cannot |
| 335 be found along the way |
| 336 """ |
| 337 if input_dict == None: |
| 338 return default_value |
| 339 for key in keys: |
| 340 input_dict = input_dict.get(key, None) |
| 341 if input_dict == None: |
| 342 return default_value |
| 343 return input_dict |
OLD | NEW |