OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python |
| 2 |
| 3 """ |
| 4 Copyright 2014 Google Inc. |
| 5 |
| 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. |
| 8 |
| 9 Compare GM results for two configs, across all builders. |
| 10 """ |
| 11 |
| 12 # System-level imports |
| 13 import argparse |
| 14 import logging |
| 15 import time |
| 16 |
| 17 # Must fix up PYTHONPATH before importing from within Skia |
| 18 import rs_fixpypath # pylint: disable=W0611 |
| 19 |
| 20 # Imports from within Skia |
| 21 from py.utils import url_utils |
| 22 import gm_json |
| 23 import imagediffdb |
| 24 import imagepair |
| 25 import imagepairset |
| 26 import results |
| 27 |
| 28 |
| 29 class ConfigComparisons(results.BaseComparisons): |
| 30 """Loads results from two different configurations into an ImagePairSet. |
| 31 |
| 32 Loads actual and expected results from all builders, except for those skipped |
| 33 by _ignore_builder(). |
| 34 """ |
| 35 |
| 36 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, |
| 37 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, |
| 38 diff_base_url=None, builder_regex_list=None): |
| 39 """ |
| 40 Args: |
| 41 configs: (string, string) tuple; pair of configs to compare |
| 42 actuals_root: root directory containing all actual-results.json files |
| 43 generated_images_root: directory within which to create all pixel diffs; |
| 44 if this directory does not yet exist, it will be created |
| 45 diff_base_url: base URL within which the client should look for diff |
| 46 images; if not specified, defaults to a "file:///" URL representation |
| 47 of generated_images_root |
| 48 builder_regex_list: List of regular expressions specifying which builders |
| 49 we will process. If None, process all builders. |
| 50 """ |
| 51 super(ConfigComparisons, self).__init__() |
| 52 time_start = int(time.time()) |
| 53 if builder_regex_list != None: |
| 54 self.set_match_builders_pattern_list(builder_regex_list) |
| 55 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) |
| 56 self._diff_base_url = ( |
| 57 diff_base_url or |
| 58 url_utils.create_filepath_url(generated_images_root)) |
| 59 self._actuals_root = actuals_root |
| 60 self._load_config_pairs(configs) |
| 61 self._timestamp = int(time.time()) |
| 62 logging.info('Results complete; took %d seconds.' % |
| 63 (self._timestamp - time_start)) |
| 64 |
| 65 def _load_config_pairs(self, configs): |
| 66 """Loads the results of all tests, across all builders (based on the |
| 67 files within self._actuals_root), compares them across two configs, |
| 68 and stores the summary in self._results. |
| 69 |
| 70 Args: |
| 71 configs: tuple of strings; pair of configs to compare |
| 72 """ |
| 73 logging.info('Reading actual-results JSON files from %s...' % |
| 74 self._actuals_root) |
| 75 actual_builder_dicts = self._read_builder_dicts_from_root( |
| 76 self._actuals_root) |
| 77 configA, configB = configs |
| 78 logging.info('Comparing configs %s and %s...' % (configA, configB)) |
| 79 |
| 80 all_image_pairs = imagepairset.ImagePairSet( |
| 81 descriptions=configs, |
| 82 diff_base_url=self._diff_base_url) |
| 83 failing_image_pairs = imagepairset.ImagePairSet( |
| 84 descriptions=configs, |
| 85 diff_base_url=self._diff_base_url) |
| 86 |
| 87 all_image_pairs.ensure_extra_column_values_in_summary( |
| 88 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ |
| 89 results.KEY__RESULT_TYPE__FAILED, |
| 90 results.KEY__RESULT_TYPE__NOCOMPARISON, |
| 91 results.KEY__RESULT_TYPE__SUCCEEDED, |
| 92 ]) |
| 93 failing_image_pairs.ensure_extra_column_values_in_summary( |
| 94 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ |
| 95 results.KEY__RESULT_TYPE__FAILED, |
| 96 results.KEY__RESULT_TYPE__NOCOMPARISON, |
| 97 ]) |
| 98 |
| 99 builders = sorted(actual_builder_dicts.keys()) |
| 100 num_builders = len(builders) |
| 101 builder_num = 0 |
| 102 for builder in builders: |
| 103 builder_num += 1 |
| 104 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % |
| 105 (builder_num, num_builders, builder)) |
| 106 actual_results_for_this_builder = ( |
| 107 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) |
| 108 for result_type in sorted(actual_results_for_this_builder.keys()): |
| 109 results_of_this_type = actual_results_for_this_builder[result_type] |
| 110 if not results_of_this_type: |
| 111 continue |
| 112 |
| 113 tests_found = set() |
| 114 for image_name in sorted(results_of_this_type.keys()): |
| 115 (test, _) = results.IMAGE_FILENAME_RE.match(image_name).groups() |
| 116 tests_found.add(test) |
| 117 |
| 118 for test in tests_found: |
| 119 # Get image_relative_url (or None) for each of configA, configB |
| 120 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) |
| 121 configA_image_relative_url = ConfigComparisons._create_relative_url( |
| 122 hashtype_and_digest=results_of_this_type.get(image_name_A), |
| 123 test_name=test) |
| 124 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) |
| 125 configB_image_relative_url = ConfigComparisons._create_relative_url( |
| 126 hashtype_and_digest=results_of_this_type.get(image_name_B), |
| 127 test_name=test) |
| 128 |
| 129 # If we have images for at least one of these two configs, |
| 130 # add them to our list. |
| 131 if configA_image_relative_url or configB_image_relative_url: |
| 132 if configA_image_relative_url == configB_image_relative_url: |
| 133 result_type = results.KEY__RESULT_TYPE__SUCCEEDED |
| 134 elif not configA_image_relative_url: |
| 135 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON |
| 136 elif not configB_image_relative_url: |
| 137 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON |
| 138 else: |
| 139 result_type = results.KEY__RESULT_TYPE__FAILED |
| 140 |
| 141 extra_columns_dict = { |
| 142 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, |
| 143 results.KEY__EXTRACOLUMNS__BUILDER: builder, |
| 144 results.KEY__EXTRACOLUMNS__TEST: test, |
| 145 # TODO(epoger): Right now, the client UI crashes if it receives |
| 146 # results that do not include a 'config' column. |
| 147 # Until we fix that, keep the client happy. |
| 148 results.KEY__EXTRACOLUMNS__CONFIG: 'TODO', |
| 149 } |
| 150 |
| 151 try: |
| 152 image_pair = imagepair.ImagePair( |
| 153 image_diff_db=self._image_diff_db, |
| 154 imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
| 155 imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
| 156 imageA_relative_url=configA_image_relative_url, |
| 157 imageB_relative_url=configB_image_relative_url, |
| 158 extra_columns=extra_columns_dict) |
| 159 all_image_pairs.add_image_pair(image_pair) |
| 160 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: |
| 161 failing_image_pairs.add_image_pair(image_pair) |
| 162 except (KeyError, TypeError): |
| 163 logging.exception( |
| 164 'got exception while creating ImagePair for test ' |
| 165 '"%s", builder "%s"' % (test, builder)) |
| 166 |
| 167 # pylint: disable=W0201 |
| 168 self._results = { |
| 169 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), |
| 170 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), |
| 171 } |
| 172 |
| 173 |
| 174 def main(): |
| 175 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', |
| 176 datefmt='%m/%d/%Y %H:%M:%S', |
| 177 level=logging.INFO) |
| 178 parser = argparse.ArgumentParser() |
| 179 parser.add_argument( |
| 180 '--actuals', default=results.DEFAULT_ACTUALS_DIR, |
| 181 help='Directory containing all actual-result JSON files; defaults to ' |
| 182 '\'%(default)s\' .') |
| 183 parser.add_argument( |
| 184 'config', nargs=2, |
| 185 help='Two configurations to compare (8888, gpu, etc.).') |
| 186 parser.add_argument( |
| 187 '--outfile', required=True, |
| 188 help='File to write result summary into, in JSON format.') |
| 189 parser.add_argument( |
| 190 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, |
| 191 help='Which result types to include. Defaults to \'%(default)s\'; ' |
| 192 'must be one of ' + |
| 193 str([results.KEY__HEADER__RESULTS_FAILURES, |
| 194 results.KEY__HEADER__RESULTS_ALL])) |
| 195 parser.add_argument( |
| 196 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, |
| 197 help='Directory within which to download images and generate diffs; ' |
| 198 'defaults to \'%(default)s\' .') |
| 199 args = parser.parse_args() |
| 200 results_obj = ConfigComparisons(configs=args.config, |
| 201 actuals_root=args.actuals, |
| 202 generated_images_root=args.workdir) |
| 203 gm_json.WriteToFile( |
| 204 results_obj.get_packaged_results_of_type(results_type=args.results), |
| 205 args.outfile) |
| 206 |
| 207 |
| 208 if __name__ == '__main__': |
| 209 main() |
OLD | NEW |