OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/python | |
2 | |
3 """ | |
4 Copyright 2014 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 | |
9 Compare GM results for two configs, across all builders. | |
10 """ | |
11 | |
12 # System-level imports | |
13 import argparse | |
14 import fnmatch | |
15 import json | |
16 import logging | |
17 import os | |
18 import re | |
19 import sys | |
20 import time | |
21 | |
22 # Imports from within Skia | |
23 # | |
24 # TODO(epoger): Once we move the create_filepath_url() function out of | |
25 # download_actuals into a shared utility module, we won't need to import | |
26 # download_actuals anymore. | |
27 # | |
28 # We need to add the 'gm' directory, so that we can import gm_json.py within | |
29 # that directory. That script allows us to parse the actual-results.json file | |
30 # written out by the GM tool. | |
31 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | |
32 # so any dirs that are already in the PYTHONPATH will be preferred. | |
33 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) | |
34 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) | |
35 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) | |
36 if GM_DIRECTORY not in sys.path: | |
37 sys.path.append(GM_DIRECTORY) | |
38 import download_actuals | |
39 import gm_json | |
40 import imagediffdb | |
41 import imagepair | |
42 import imagepairset | |
43 import results | |
44 | |
45 | |
46 class ConfigComparisons(results.BaseComparisons): | |
47 """ Loads results from two different configurations into an ImagePairSet. | |
48 | |
49 Loads actual and expected results from all builders, except for those skipped | |
50 by _ignore_builder(). | |
rmistry
2014/03/28 18:51:39
Change to results._ignore_builder() ?
epoger
2014/03/28 19:25:55
Changed to BaseComparisons._ignore_builder()
| |
51 """ | |
52 | |
53 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, | |
54 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, | |
55 diff_base_url=None): | |
56 """ | |
57 Args: | |
58 configs: (string, string) tuple; pair of configs to compare | |
59 actuals_root: root directory containing all actual-results.json files | |
60 generated_images_root: directory within which to create all pixel diffs; | |
61 if this directory does not yet exist, it will be created | |
62 diff_base_url: base URL within which the client should look for diff | |
63 images; if not specified, defaults to a "file:///" URL representation | |
64 of generated_images_root | |
65 """ | |
66 time_start = int(time.time()) | |
67 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) | |
68 self._diff_base_url = ( | |
69 diff_base_url or | |
70 download_actuals.create_filepath_url(generated_images_root)) | |
71 self._actuals_root = actuals_root | |
72 self._load_config_pairs(configs) | |
73 self._timestamp = int(time.time()) | |
74 logging.info('Results complete; took %d seconds.' % | |
75 (self._timestamp - time_start)) | |
76 | |
77 def _load_config_pairs(self, configs): | |
78 """ Loads the results of all tests, across all builders (based on the | |
79 files within self._actuals_root), compares them across two configs, | |
80 and stores the summary in self._results. | |
81 | |
82 Args: | |
83 configs: tuple of strings; pair of configs to compare | |
84 """ | |
85 logging.info('Reading actual-results JSON files from %s...' % | |
86 self._actuals_root) | |
87 actual_builder_dicts = ConfigComparisons._read_dicts_from_root( | |
88 self._actuals_root) | |
89 configA, configB = configs | |
90 logging.info('Comparing configs %s and %s...' % (configA, configB)) | |
91 | |
92 all_image_pairs = imagepairset.ImagePairSet( | |
93 descriptions=configs, | |
94 diff_base_url=self._diff_base_url) | |
95 failing_image_pairs = imagepairset.ImagePairSet( | |
96 descriptions=configs, | |
97 diff_base_url=self._diff_base_url) | |
98 | |
99 all_image_pairs.ensure_extra_column_values_in_summary( | |
100 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | |
101 results.KEY__RESULT_TYPE__FAILED, | |
102 results.KEY__RESULT_TYPE__NOCOMPARISON, | |
103 results.KEY__RESULT_TYPE__SUCCEEDED, | |
104 ]) | |
105 failing_image_pairs.ensure_extra_column_values_in_summary( | |
106 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | |
107 results.KEY__RESULT_TYPE__FAILED, | |
108 results.KEY__RESULT_TYPE__NOCOMPARISON, | |
109 ]) | |
110 | |
111 builders = sorted(actual_builder_dicts.keys()) | |
112 num_builders = len(builders) | |
113 builder_num = 0 | |
114 for builder in builders: | |
115 builder_num += 1 | |
116 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % | |
117 (builder_num, num_builders, builder)) | |
118 actual_results_for_this_builder = ( | |
119 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) | |
120 for result_type in sorted(actual_results_for_this_builder.keys()): | |
121 results_of_this_type = actual_results_for_this_builder[result_type] | |
122 if not results_of_this_type: | |
123 continue | |
124 | |
125 tests_found = set() | |
126 for image_name in sorted(results_of_this_type.keys()): | |
127 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() | |
128 tests_found.add(test) | |
129 | |
130 for test in tests_found: | |
131 # Get image_relative_url (or None) for each of configA, configB | |
132 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) | |
133 configA_image_relative_url = ConfigComparisons._create_relative_url( | |
134 hashtype_and_digest=results_of_this_type.get(image_name_A), | |
135 test_name=test) | |
136 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) | |
137 configB_image_relative_url = ConfigComparisons._create_relative_url( | |
138 hashtype_and_digest=results_of_this_type.get(image_name_B), | |
139 test_name=test) | |
140 | |
141 # If we have images for at least one of these two configs, | |
142 # add them to our list. | |
143 if configA_image_relative_url or configB_image_relative_url: | |
144 if configA_image_relative_url == configB_image_relative_url: | |
145 result_type = results.KEY__RESULT_TYPE__SUCCEEDED | |
146 elif not configA_image_relative_url: | |
147 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON | |
148 elif not configB_image_relative_url: | |
149 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON | |
150 else: | |
151 result_type = results.KEY__RESULT_TYPE__FAILED | |
152 | |
153 extra_columns_dict = { | |
154 results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type, | |
155 results.KEY__EXTRACOLUMN__BUILDER: builder, | |
156 results.KEY__EXTRACOLUMN__TEST: test, | |
157 # TODO(epoger): Right now, the client UI crashes if it receives | |
158 # results that do not include a 'config' column. | |
159 # Until we fix that, keep the client happy. | |
160 results.KEY__EXTRACOLUMN__CONFIG: 'TODO', | |
161 } | |
162 | |
163 try: | |
164 image_pair = imagepair.ImagePair( | |
165 image_diff_db=self._image_diff_db, | |
166 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, | |
167 imageA_relative_url=configA_image_relative_url, | |
168 imageB_relative_url=configB_image_relative_url, | |
169 extra_columns=extra_columns_dict) | |
170 all_image_pairs.add_image_pair(image_pair) | |
171 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: | |
172 failing_image_pairs.add_image_pair(image_pair) | |
173 except (KeyError, TypeError): | |
174 logging.exception( | |
175 'got exception while creating ImagePair for image_name ' | |
176 '"%s", builder "%s"' % (image_name, builder)) | |
177 | |
178 self._results = { | |
179 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), | |
180 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), | |
181 } | |
182 | |
183 | |
184 def main(): | |
185 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', | |
186 datefmt='%m/%d/%Y %H:%M:%S', | |
187 level=logging.INFO) | |
188 parser = argparse.ArgumentParser() | |
189 parser.add_argument( | |
190 '--actuals', default=results.DEFAULT_ACTUALS_DIR, | |
191 help='Directory containing all actual-result JSON files; defaults to ' | |
192 '\'%(default)s\' .') | |
193 parser.add_argument( | |
194 'config', nargs=2, | |
195 help='Two configurations to compare (8888, gpu, etc.).') | |
196 parser.add_argument( | |
197 '--outfile', required=True, | |
198 help='File to write result summary into, in JSON format.') | |
199 parser.add_argument( | |
200 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, | |
201 help='Which result types to include. Defaults to \'%(default)s\'; ' | |
202 'must be one of ' + | |
203 str([results.KEY__HEADER__RESULTS_FAILURES, | |
204 results.KEY__HEADER__RESULTS_ALL])) | |
205 parser.add_argument( | |
206 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, | |
207 help='Directory within which to download images and generate diffs; ' | |
208 'defaults to \'%(default)s\' .') | |
209 args = parser.parse_args() | |
210 results_obj = ConfigComparisons(configs=args.config, | |
211 actuals_root=args.actuals, | |
212 generated_images_root=args.workdir) | |
213 gm_json.WriteToFile( | |
214 results_obj.get_packaged_results_of_type(results_type=args.results), | |
215 args.outfile) | |
216 | |
217 | |
218 if __name__ == '__main__': | |
219 main() | |
OLD | NEW |