OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Compare GM results for two configs, across all builders. | |
10 """ | |
11 | |
12 # System-level imports | |
13 import argparse | |
14 import logging | |
15 import time | |
16 | |
17 # Must fix up PYTHONPATH before importing from within Skia | |
18 import rs_fixpypath # pylint: disable=W0611 | |
19 | |
20 # Imports from within Skia | |
21 from py.utils import url_utils | |
22 import gm_json | |
23 import imagediffdb | |
24 import imagepair | |
25 import imagepairset | |
26 import results | |
27 | |
28 | |
29 class ConfigComparisons(results.BaseComparisons): | |
30 """Loads results from two different configurations into an ImagePairSet. | |
31 | |
32 Loads actual and expected results from all builders, except for those skipped | |
33 by _ignore_builder(). | |
34 """ | |
35 | |
36 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, | |
37 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, | |
38 diff_base_url=None, builder_regex_list=None): | |
39 """ | |
40 Args: | |
41 configs: (string, string) tuple; pair of configs to compare | |
42 actuals_root: root directory containing all actual-results.json files | |
43 generated_images_root: directory within which to create all pixel diffs; | |
44 if this directory does not yet exist, it will be created | |
45 diff_base_url: base URL within which the client should look for diff | |
46 images; if not specified, defaults to a "file:///" URL representation | |
47 of generated_images_root | |
48 builder_regex_list: List of regular expressions specifying which builders | |
49 we will process. If None, process all builders. | |
50 """ | |
51 super(ConfigComparisons, self).__init__() | |
52 time_start = int(time.time()) | |
53 if builder_regex_list != None: | |
54 self.set_match_builders_pattern_list(builder_regex_list) | |
55 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) | |
56 self._diff_base_url = ( | |
57 diff_base_url or | |
58 url_utils.create_filepath_url(generated_images_root)) | |
59 self._actuals_root = actuals_root | |
60 self._load_config_pairs(configs) | |
61 self._timestamp = int(time.time()) | |
62 logging.info('Results complete; took %d seconds.' % | |
63 (self._timestamp - time_start)) | |
64 | |
65 def _load_config_pairs(self, configs): | |
66 """Loads the results of all tests, across all builders (based on the | |
67 files within self._actuals_root), compares them across two configs, | |
68 and stores the summary in self._results. | |
69 | |
70 Args: | |
71 configs: tuple of strings; pair of configs to compare | |
72 """ | |
73 logging.info('Reading actual-results JSON files from %s...' % | |
74 self._actuals_root) | |
75 actual_builder_dicts = self._read_builder_dicts_from_root( | |
76 self._actuals_root) | |
77 configA, configB = configs | |
78 logging.info('Comparing configs %s and %s...' % (configA, configB)) | |
79 | |
80 all_image_pairs = imagepairset.ImagePairSet( | |
81 descriptions=configs, | |
82 diff_base_url=self._diff_base_url) | |
83 failing_image_pairs = imagepairset.ImagePairSet( | |
84 descriptions=configs, | |
85 diff_base_url=self._diff_base_url) | |
86 | |
87 all_image_pairs.ensure_extra_column_values_in_summary( | |
88 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ | |
89 results.KEY__RESULT_TYPE__FAILED, | |
90 results.KEY__RESULT_TYPE__NOCOMPARISON, | |
91 results.KEY__RESULT_TYPE__SUCCEEDED, | |
92 ]) | |
93 failing_image_pairs.ensure_extra_column_values_in_summary( | |
94 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ | |
95 results.KEY__RESULT_TYPE__FAILED, | |
96 results.KEY__RESULT_TYPE__NOCOMPARISON, | |
97 ]) | |
98 | |
99 builders = sorted(actual_builder_dicts.keys()) | |
100 num_builders = len(builders) | |
101 builder_num = 0 | |
102 for builder in builders: | |
103 builder_num += 1 | |
104 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % | |
105 (builder_num, num_builders, builder)) | |
106 actual_results_for_this_builder = ( | |
107 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) | |
108 for result_type in sorted(actual_results_for_this_builder.keys()): | |
109 results_of_this_type = actual_results_for_this_builder[result_type] | |
110 if not results_of_this_type: | |
111 continue | |
112 | |
113 tests_found = set() | |
114 for image_name in sorted(results_of_this_type.keys()): | |
115 (test, _) = results.IMAGE_FILENAME_RE.match(image_name).groups() | |
116 tests_found.add(test) | |
117 | |
118 for test in tests_found: | |
119 # Get image_relative_url (or None) for each of configA, configB | |
120 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) | |
121 configA_image_relative_url = ConfigComparisons._create_relative_url( | |
122 hashtype_and_digest=results_of_this_type.get(image_name_A), | |
123 test_name=test) | |
124 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) | |
125 configB_image_relative_url = ConfigComparisons._create_relative_url( | |
126 hashtype_and_digest=results_of_this_type.get(image_name_B), | |
127 test_name=test) | |
128 | |
129 # If we have images for at least one of these two configs, | |
130 # add them to our list. | |
131 if configA_image_relative_url or configB_image_relative_url: | |
132 if configA_image_relative_url == configB_image_relative_url: | |
133 result_type = results.KEY__RESULT_TYPE__SUCCEEDED | |
134 elif not configA_image_relative_url: | |
135 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON | |
136 elif not configB_image_relative_url: | |
137 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON | |
138 else: | |
139 result_type = results.KEY__RESULT_TYPE__FAILED | |
140 | |
141 extra_columns_dict = { | |
142 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, | |
143 results.KEY__EXTRACOLUMNS__BUILDER: builder, | |
144 results.KEY__EXTRACOLUMNS__TEST: test, | |
145 # TODO(epoger): Right now, the client UI crashes if it receives | |
146 # results that do not include a 'config' column. | |
147 # Until we fix that, keep the client happy. | |
148 results.KEY__EXTRACOLUMNS__CONFIG: 'TODO', | |
149 } | |
150 | |
151 try: | |
152 image_pair = imagepair.ImagePair( | |
153 image_diff_db=self._image_diff_db, | |
154 imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, | |
155 imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, | |
156 imageA_relative_url=configA_image_relative_url, | |
157 imageB_relative_url=configB_image_relative_url, | |
158 extra_columns=extra_columns_dict) | |
159 all_image_pairs.add_image_pair(image_pair) | |
160 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: | |
161 failing_image_pairs.add_image_pair(image_pair) | |
162 except (KeyError, TypeError): | |
163 logging.exception( | |
164 'got exception while creating ImagePair for test ' | |
165 '"%s", builder "%s"' % (test, builder)) | |
166 | |
167 # pylint: disable=W0201 | |
168 self._results = { | |
169 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), | |
170 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), | |
171 } | |
172 | |
173 | |
174 def main(): | |
175 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', | |
176 datefmt='%m/%d/%Y %H:%M:%S', | |
177 level=logging.INFO) | |
178 parser = argparse.ArgumentParser() | |
179 parser.add_argument( | |
180 '--actuals', default=results.DEFAULT_ACTUALS_DIR, | |
181 help='Directory containing all actual-result JSON files; defaults to ' | |
182 '\'%(default)s\' .') | |
183 parser.add_argument( | |
184 'config', nargs=2, | |
185 help='Two configurations to compare (8888, gpu, etc.).') | |
186 parser.add_argument( | |
187 '--outfile', required=True, | |
188 help='File to write result summary into, in JSON format.') | |
189 parser.add_argument( | |
190 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, | |
191 help='Which result types to include. Defaults to \'%(default)s\'; ' | |
192 'must be one of ' + | |
193 str([results.KEY__HEADER__RESULTS_FAILURES, | |
194 results.KEY__HEADER__RESULTS_ALL])) | |
195 parser.add_argument( | |
196 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, | |
197 help='Directory within which to download images and generate diffs; ' | |
198 'defaults to \'%(default)s\' .') | |
199 args = parser.parse_args() | |
200 results_obj = ConfigComparisons(configs=args.config, | |
201 actuals_root=args.actuals, | |
202 generated_images_root=args.workdir) | |
203 gm_json.WriteToFile( | |
204 results_obj.get_packaged_results_of_type(results_type=args.results), | |
205 args.outfile) | |
206 | |
207 | |
208 if __name__ == '__main__': | |
209 main() | |
OLD | NEW |