OLD | NEW |
1 # Copyright (C) 2011, Google Inc. All rights reserved. | 1 # Copyright (C) 2011, Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 10 matching lines...) Expand all Loading... |
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | 28 |
29 import copy | 29 import copy |
30 import logging | 30 import logging |
| 31 import functools |
31 | 32 |
32 from webkitpy.common.memoized import memoized | 33 from webkitpy.common.memoized import memoized |
33 from functools import reduce | |
34 | 34 |
35 _log = logging.getLogger(__name__) | 35 _log = logging.getLogger(__name__) |
36 | 36 |
37 | 37 |
38 # FIXME: Should this function be somewhere more general? | 38 # FIXME: Should this function be somewhere more general? |
39 def _invert_dictionary(dictionary): | 39 def _invert_dictionary(dictionary): |
40 inverted_dictionary = {} | 40 inverted_dictionary = {} |
41 for key, value in dictionary.items(): | 41 for key, value in dictionary.items(): |
42 if inverted_dictionary.get(value): | 42 if inverted_dictionary.get(value): |
43 inverted_dictionary[value].append(key) | 43 inverted_dictionary[value].append(key) |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
94 # to the baseline in the platform directory, we need to append just foo-
expected.png to the directory. | 94 # to the baseline in the platform directory, we need to append just foo-
expected.png to the directory. |
95 virtual_suite = self._virtual_suite(baseline_name) | 95 virtual_suite = self._virtual_suite(baseline_name) |
96 if virtual_suite: | 96 if virtual_suite: |
97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name
) + 1:] | 97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name
) + 1:] |
98 else: | 98 else: |
99 baseline_name_without_virtual = baseline_name | 99 baseline_name_without_virtual = baseline_name |
100 return self._filesystem.join(self._webkit_base, directory, baseline_name
_without_virtual) | 100 return self._filesystem.join(self._webkit_base, directory, baseline_name
_without_virtual) |
101 | 101 |
102 def read_results_by_directory(self, baseline_name): | 102 def read_results_by_directory(self, baseline_name): |
103 results_by_directory = {} | 103 results_by_directory = {} |
104 directories = reduce(set.union, map(set, [self._relative_baseline_search
_paths( | 104 directories = functools.reduce(set.union, map(set, [self._relative_basel
ine_search_paths( |
105 port, baseline_name) for port in self._ports.values()])) | 105 port, baseline_name) for port in self._ports.values()])) |
106 | 106 |
107 for directory in directories: | 107 for directory in directories: |
108 path = self._join_directory(directory, baseline_name) | 108 path = self._join_directory(directory, baseline_name) |
109 if self._filesystem.exists(path): | 109 if self._filesystem.exists(path): |
110 results_by_directory[directory] = self._filesystem.sha1(path) | 110 results_by_directory[directory] = self._filesystem.sha1(path) |
111 return results_by_directory | 111 return results_by_directory |
112 | 112 |
113 def _results_by_port_name(self, results_by_directory, baseline_name): | 113 def _results_by_port_name(self, results_by_directory, baseline_name): |
114 results_by_port_name = {} | 114 results_by_port_name = {} |
(...skipping 12 matching lines...) Expand all Loading... |
127 directories.add(directory) | 127 directories.add(directory) |
128 return directories | 128 return directories |
129 | 129 |
130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name)
: | 130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name)
: |
131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor
respond | 131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor
respond |
132 # to a specific platform. As such, it's the only one where the baseline
in fallback directories | 132 # to a specific platform. As such, it's the only one where the baseline
in fallback directories |
133 # immediately before it can be promoted up, i.e. if win and mac | 133 # immediately before it can be promoted up, i.e. if win and mac |
134 # have the same baseline, then it can be promoted up to be the LayoutTes
ts baseline. | 134 # have the same baseline, then it can be promoted up to be the LayoutTes
ts baseline. |
135 # All other baselines can only be removed if they're redundant with a ba
seline earlier | 135 # All other baselines can only be removed if they're redundant with a ba
seline earlier |
136 # in the fallback order. They can never promoted up. | 136 # in the fallback order. They can never promoted up. |
137 directories_immediately_preceding_root = self._directories_immediately_p
receding_root(baseline_name) | 137 immediately_preceding_root = self._directories_immediately_preceding_roo
t(baseline_name) |
138 | 138 |
139 shared_result = None | 139 shared_result = None |
140 root_baseline_unused = False | 140 root_baseline_unused = False |
141 for directory in directories_immediately_preceding_root: | 141 for directory in immediately_preceding_root: |
142 this_result = new_results_by_directory.get(directory) | 142 this_result = new_results_by_directory.get(directory) |
143 | 143 |
144 # If any of these directories don't have a baseline, there's no opti
mization we can do. | 144 # If any of these directories don't have a baseline, there's no opti
mization we can do. |
145 if not this_result: | 145 if not this_result: |
146 return | 146 return |
147 | 147 |
148 if not shared_result: | 148 if not shared_result: |
149 shared_result = this_result | 149 shared_result = this_result |
150 elif shared_result != this_result: | 150 elif shared_result != this_result: |
151 root_baseline_unused = True | 151 root_baseline_unused = True |
152 | 152 |
153 baseline_root = self._baseline_root(baseline_name) | 153 baseline_root = self._baseline_root(baseline_name) |
154 | 154 |
155 # The root baseline is unused if all the directories immediately precedi
ng the root | 155 # The root baseline is unused if all the directories immediately precedi
ng the root |
156 # have a baseline, but have different baselines, so the baselines can't
be promoted up. | 156 # have a baseline, but have different baselines, so the baselines can't
be promoted up. |
157 if root_baseline_unused: | 157 if root_baseline_unused: |
158 if baseline_root in new_results_by_directory: | 158 if baseline_root in new_results_by_directory: |
159 del new_results_by_directory[baseline_root] | 159 del new_results_by_directory[baseline_root] |
160 return | 160 return |
161 | 161 |
162 new_results_by_directory[baseline_root] = shared_result | 162 new_results_by_directory[baseline_root] = shared_result |
163 for directory in directories_immediately_preceding_root: | 163 for directory in immediately_preceding_root: |
164 del new_results_by_directory[directory] | 164 del new_results_by_directory[directory] |
165 | 165 |
166 def _find_optimal_result_placement(self, baseline_name): | 166 def _find_optimal_result_placement(self, baseline_name): |
167 results_by_directory = self.read_results_by_directory(baseline_name) | 167 results_by_directory = self.read_results_by_directory(baseline_name) |
168 results_by_port_name = self._results_by_port_name(results_by_directory,
baseline_name) | 168 results_by_port_name = self._results_by_port_name(results_by_directory,
baseline_name) |
169 port_names_by_result = _invert_dictionary(results_by_port_name) | |
170 | 169 |
171 new_results_by_directory = self._remove_redundant_results( | 170 new_results_by_directory = self._remove_redundant_results( |
172 results_by_directory, results_by_port_name, port_names_by_result, ba
seline_name) | 171 results_by_directory, results_by_port_name, baseline_name) |
173 self._optimize_result_for_root(new_results_by_directory, baseline_name) | 172 self._optimize_result_for_root(new_results_by_directory, baseline_name) |
174 | 173 |
175 return results_by_directory, new_results_by_directory | 174 return results_by_directory, new_results_by_directory |
176 | 175 |
177 def _remove_redundant_results(self, results_by_directory, results_by_port_na
me, port_names_by_result, baseline_name): | 176 def _remove_redundant_results(self, results_by_directory, results_by_port_na
me, baseline_name): |
178 new_results_by_directory = copy.copy(results_by_directory) | 177 new_results_by_directory = copy.copy(results_by_directory) |
179 for port_name, port in self._ports.items(): | 178 for port_name, port in self._ports.items(): |
180 current_result = results_by_port_name.get(port_name) | 179 current_result = results_by_port_name.get(port_name) |
181 | 180 |
182 # This happens if we're missing baselines for a port. | 181 # This happens if we're missing baselines for a port. |
183 if not current_result: | 182 if not current_result: |
184 continue | 183 continue |
185 | 184 |
186 fallback_path = self._relative_baseline_search_paths(port, baseline_
name) | 185 fallback_path = self._relative_baseline_search_paths(port, baseline_
name) |
187 current_index, current_directory = self._find_in_fallbackpath(fallba
ck_path, current_result, new_results_by_directory) | 186 current_index, current_directory = self._find_in_fallbackpath(fallba
ck_path, current_result, new_results_by_directory) |
188 for index in range(current_index + 1, len(fallback_path)): | 187 for index in range(current_index + 1, len(fallback_path)): |
189 new_directory = fallback_path[index] | 188 new_directory = fallback_path[index] |
190 if not new_directory in new_results_by_directory: | 189 if new_directory not in new_results_by_directory: |
191 # No result for this baseline in this directory. | 190 # No result for this baseline in this directory. |
192 continue | 191 continue |
193 elif new_results_by_directory[new_directory] == current_result: | 192 elif new_results_by_directory[new_directory] == current_result: |
194 # Result for new_directory are redundant with the result ear
lier in the fallback order. | 193 # Result for new_directory are redundant with the result ear
lier in the fallback order. |
195 if current_directory in new_results_by_directory: | 194 if current_directory in new_results_by_directory: |
196 del new_results_by_directory[current_directory] | 195 del new_results_by_directory[current_directory] |
197 else: | 196 else: |
198 # The new_directory contains a different result, so stop try
ing to push results up. | 197 # The new_directory contains a different result, so stop try
ing to push results up. |
199 break | 198 break |
200 | 199 |
(...skipping 10 matching lines...) Expand all Loading... |
211 if filename.startswith(platform_dir): | 210 if filename.startswith(platform_dir): |
212 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] | 211 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] |
213 platform_dir = self._filesystem.join(self._webkit_base, platform_dir) | 212 platform_dir = self._filesystem.join(self._webkit_base, platform_dir) |
214 if filename.startswith(platform_dir): | 213 if filename.startswith(platform_dir): |
215 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] | 214 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] |
216 return '(generic)' | 215 return '(generic)' |
217 | 216 |
218 def _move_baselines(self, baseline_name, results_by_directory, new_results_b
y_directory): | 217 def _move_baselines(self, baseline_name, results_by_directory, new_results_b
y_directory): |
219 data_for_result = {} | 218 data_for_result = {} |
220 for directory, result in results_by_directory.items(): | 219 for directory, result in results_by_directory.items(): |
221 if not result in data_for_result: | 220 if result not in data_for_result: |
222 source = self._join_directory(directory, baseline_name) | 221 source = self._join_directory(directory, baseline_name) |
223 data_for_result[result] = self._filesystem.read_binary_file(sour
ce) | 222 data_for_result[result] = self._filesystem.read_binary_file(sour
ce) |
224 | 223 |
225 fs_files = [] | 224 fs_files = [] |
226 for directory, result in results_by_directory.items(): | 225 for directory, result in results_by_directory.items(): |
227 if new_results_by_directory.get(directory) != result: | 226 if new_results_by_directory.get(directory) != result: |
228 file_name = self._join_directory(directory, baseline_name) | 227 file_name = self._join_directory(directory, baseline_name) |
229 if self._filesystem.exists(file_name): | 228 if self._filesystem.exists(file_name): |
230 fs_files.append(file_name) | 229 fs_files.append(file_name) |
231 | 230 |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
282 _log.debug(" %s:", basename) | 281 _log.debug(" %s:", basename) |
283 _log.debug(" Before: ") | 282 _log.debug(" Before: ") |
284 self.write_by_directory(results_by_directory, _log.debug, " ") | 283 self.write_by_directory(results_by_directory, _log.debug, " ") |
285 _log.debug(" After: ") | 284 _log.debug(" After: ") |
286 self.write_by_directory(new_results_by_directory, _log.debug, " ") | 285 self.write_by_directory(new_results_by_directory, _log.debug, " ") |
287 | 286 |
288 self._move_baselines(baseline_name, results_by_directory, new_results_by
_directory) | 287 self._move_baselines(baseline_name, results_by_directory, new_results_by
_directory) |
289 return True | 288 return True |
290 | 289 |
291 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name): | 290 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name): |
292 virtual_root_expected_baseline_path = self._filesystem.join(self._layout
_tests_dir, baseline_name) | 291 virtual_root_baseline_path = self._filesystem.join(self._layout_tests_di
r, baseline_name) |
293 if not self._filesystem.exists(virtual_root_expected_baseline_path): | 292 if not self._filesystem.exists(virtual_root_baseline_path): |
294 return | 293 return |
295 root_sha1 = self._filesystem.sha1(virtual_root_expected_baseline_path) | 294 root_sha1 = self._filesystem.sha1(virtual_root_baseline_path) |
296 | 295 |
297 results_by_directory = self.read_results_by_directory(non_virtual_baseli
ne_name) | 296 results_by_directory = self.read_results_by_directory(non_virtual_baseli
ne_name) |
298 # See if all the immediate predecessors of the virtual root have the sam
e expected result. | 297 # See if all the immediate predecessors of the virtual root have the sam
e expected result. |
299 for port in self._ports.values(): | 298 for port in self._ports.values(): |
300 directories = self._relative_baseline_search_paths(port, non_virtual
_baseline_name) | 299 directories = self._relative_baseline_search_paths(port, non_virtual
_baseline_name) |
301 for directory in directories: | 300 for directory in directories: |
302 if directory not in results_by_directory: | 301 if directory not in results_by_directory: |
303 continue | 302 continue |
304 if results_by_directory[directory] != root_sha1: | 303 if results_by_directory[directory] != root_sha1: |
305 return | 304 return |
306 break | 305 break |
307 | 306 |
308 _log.debug("Deleting redundant virtual root expected result.") | 307 _log.debug("Deleting redundant virtual root expected result.") |
309 _log.debug(" Deleting (file system): " + virtual_root_expected_baseli
ne_path) | 308 _log.debug(" Deleting (file system): " + virtual_root_baseline_path) |
310 self._filesystem.remove(virtual_root_expected_baseline_path) | 309 self._filesystem.remove(virtual_root_baseline_path) |
311 | 310 |
312 def optimize(self, baseline_name): | 311 def optimize(self, baseline_name): |
313 # The virtual fallback path is the same as the non-virtual one tacked on
to the bottom of the non-virtual path. | 312 # The virtual fallback path is the same as the non-virtual one tacked on
to the bottom of the non-virtual path. |
314 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb
UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for | 313 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb
UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for |
315 # a visual representation of this. | 314 # a visual representation of this. |
316 # | 315 # |
317 # So, we can optimize the virtual path, then the virtual root and then t
he regular path. | 316 # So, we can optimize the virtual path, then the virtual root and then t
he regular path. |
318 | 317 |
319 _log.debug("Optimizing regular fallback path.") | 318 _log.debug("Optimizing regular fallback path.") |
320 result = self._optimize_subtree(baseline_name) | 319 result = self._optimize_subtree(baseline_name) |
321 non_virtual_baseline_name = self._virtual_base(baseline_name) | 320 non_virtual_baseline_name = self._virtual_base(baseline_name) |
322 if not non_virtual_baseline_name: | 321 if not non_virtual_baseline_name: |
323 return result | 322 return result |
324 | 323 |
325 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name) | 324 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name) |
326 | 325 |
327 _log.debug("Optimizing non-virtual fallback path.") | 326 _log.debug("Optimizing non-virtual fallback path.") |
328 result |= self._optimize_subtree(non_virtual_baseline_name) | 327 result |= self._optimize_subtree(non_virtual_baseline_name) |
329 return result | 328 return result |
OLD | NEW |