Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(164)

Side by Side Diff: gm/rebaseline_server/compare_configs.py

Issue 215503002: rebaseline_server: add --compare-configs option (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: rebase Created 6 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | gm/rebaseline_server/compare_configs_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/python
2
3 """
4 Copyright 2014 Google Inc.
5
6 Use of this source code is governed by a BSD-style license that can be
7 found in the LICENSE file.
8
9 Compare GM results for two configs, across all builders.
10 """
11
12 # System-level imports
13 import argparse
14 import fnmatch
15 import json
16 import logging
17 import os
18 import re
19 import sys
20 import time
21
22 # Imports from within Skia
23 #
24 # TODO(epoger): Once we move the create_filepath_url() function out of
25 # download_actuals into a shared utility module, we won't need to import
26 # download_actuals anymore.
27 #
28 # We need to add the 'gm' directory, so that we can import gm_json.py within
29 # that directory. That script allows us to parse the actual-results.json file
30 # written out by the GM tool.
31 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
32 # so any dirs that are already in the PYTHONPATH will be preferred.
33 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
34 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
35 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
36 if GM_DIRECTORY not in sys.path:
37 sys.path.append(GM_DIRECTORY)
38 import download_actuals
39 import gm_json
40 import imagediffdb
41 import imagepair
42 import imagepairset
43 import results
44
45
46 class ConfigComparisons(results.BaseComparisons):
47 """Loads results from two different configurations into an ImagePairSet.
48
49 Loads actual and expected results from all builders, except for those skipped
50 by BaseComparisons._ignore_builder().
51 """
52
53 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
54 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
55 diff_base_url=None):
56 """
57 Args:
58 configs: (string, string) tuple; pair of configs to compare
59 actuals_root: root directory containing all actual-results.json files
60 generated_images_root: directory within which to create all pixel diffs;
61 if this directory does not yet exist, it will be created
62 diff_base_url: base URL within which the client should look for diff
63 images; if not specified, defaults to a "file:///" URL representation
64 of generated_images_root
65 """
66 time_start = int(time.time())
67 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
68 self._diff_base_url = (
69 diff_base_url or
70 download_actuals.create_filepath_url(generated_images_root))
71 self._actuals_root = actuals_root
72 self._load_config_pairs(configs)
73 self._timestamp = int(time.time())
74 logging.info('Results complete; took %d seconds.' %
75 (self._timestamp - time_start))
76
77 def _load_config_pairs(self, configs):
78 """Loads the results of all tests, across all builders (based on the
79 files within self._actuals_root), compares them across two configs,
80 and stores the summary in self._results.
81
82 Args:
83 configs: tuple of strings; pair of configs to compare
84 """
85 logging.info('Reading actual-results JSON files from %s...' %
86 self._actuals_root)
87 actual_builder_dicts = ConfigComparisons._read_dicts_from_root(
88 self._actuals_root)
89 configA, configB = configs
90 logging.info('Comparing configs %s and %s...' % (configA, configB))
91
92 all_image_pairs = imagepairset.ImagePairSet(
93 descriptions=configs,
94 diff_base_url=self._diff_base_url)
95 failing_image_pairs = imagepairset.ImagePairSet(
96 descriptions=configs,
97 diff_base_url=self._diff_base_url)
98
99 all_image_pairs.ensure_extra_column_values_in_summary(
100 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
101 results.KEY__RESULT_TYPE__FAILED,
102 results.KEY__RESULT_TYPE__NOCOMPARISON,
103 results.KEY__RESULT_TYPE__SUCCEEDED,
104 ])
105 failing_image_pairs.ensure_extra_column_values_in_summary(
106 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
107 results.KEY__RESULT_TYPE__FAILED,
108 results.KEY__RESULT_TYPE__NOCOMPARISON,
109 ])
110
111 builders = sorted(actual_builder_dicts.keys())
112 num_builders = len(builders)
113 builder_num = 0
114 for builder in builders:
115 builder_num += 1
116 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
117 (builder_num, num_builders, builder))
118 actual_results_for_this_builder = (
119 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
120 for result_type in sorted(actual_results_for_this_builder.keys()):
121 results_of_this_type = actual_results_for_this_builder[result_type]
122 if not results_of_this_type:
123 continue
124
125 tests_found = set()
126 for image_name in sorted(results_of_this_type.keys()):
127 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
128 tests_found.add(test)
129
130 for test in tests_found:
131 # Get image_relative_url (or None) for each of configA, configB
132 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
133 configA_image_relative_url = ConfigComparisons._create_relative_url(
134 hashtype_and_digest=results_of_this_type.get(image_name_A),
135 test_name=test)
136 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
137 configB_image_relative_url = ConfigComparisons._create_relative_url(
138 hashtype_and_digest=results_of_this_type.get(image_name_B),
139 test_name=test)
140
141 # If we have images for at least one of these two configs,
142 # add them to our list.
143 if configA_image_relative_url or configB_image_relative_url:
144 if configA_image_relative_url == configB_image_relative_url:
145 result_type = results.KEY__RESULT_TYPE__SUCCEEDED
146 elif not configA_image_relative_url:
147 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
148 elif not configB_image_relative_url:
149 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
150 else:
151 result_type = results.KEY__RESULT_TYPE__FAILED
152
153 extra_columns_dict = {
154 results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
155 results.KEY__EXTRACOLUMN__BUILDER: builder,
156 results.KEY__EXTRACOLUMN__TEST: test,
157 # TODO(epoger): Right now, the client UI crashes if it receives
158 # results that do not include a 'config' column.
159 # Until we fix that, keep the client happy.
160 results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
161 }
162
163 try:
164 image_pair = imagepair.ImagePair(
165 image_diff_db=self._image_diff_db,
166 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
167 imageA_relative_url=configA_image_relative_url,
168 imageB_relative_url=configB_image_relative_url,
169 extra_columns=extra_columns_dict)
170 all_image_pairs.add_image_pair(image_pair)
171 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
172 failing_image_pairs.add_image_pair(image_pair)
173 except (KeyError, TypeError):
174 logging.exception(
175 'got exception while creating ImagePair for image_name '
176 '"%s", builder "%s"' % (image_name, builder))
177
178 self._results = {
179 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
180 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
181 }
182
183
184 def main():
185 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
186 datefmt='%m/%d/%Y %H:%M:%S',
187 level=logging.INFO)
188 parser = argparse.ArgumentParser()
189 parser.add_argument(
190 '--actuals', default=results.DEFAULT_ACTUALS_DIR,
191 help='Directory containing all actual-result JSON files; defaults to '
192 '\'%(default)s\' .')
193 parser.add_argument(
194 'config', nargs=2,
195 help='Two configurations to compare (8888, gpu, etc.).')
196 parser.add_argument(
197 '--outfile', required=True,
198 help='File to write result summary into, in JSON format.')
199 parser.add_argument(
200 '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
201 help='Which result types to include. Defaults to \'%(default)s\'; '
202 'must be one of ' +
203 str([results.KEY__HEADER__RESULTS_FAILURES,
204 results.KEY__HEADER__RESULTS_ALL]))
205 parser.add_argument(
206 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
207 help='Directory within which to download images and generate diffs; '
208 'defaults to \'%(default)s\' .')
209 args = parser.parse_args()
210 results_obj = ConfigComparisons(configs=args.config,
211 actuals_root=args.actuals,
212 generated_images_root=args.workdir)
213 gm_json.WriteToFile(
214 results_obj.get_packaged_results_of_type(results_type=args.results),
215 args.outfile)
216
217
218 if __name__ == '__main__':
219 main()
OLDNEW
« no previous file with comments | « no previous file | gm/rebaseline_server/compare_configs_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698