| OLD | NEW |
| 1 # Copyright (C) 2011, Google Inc. All rights reserved. | 1 # Copyright (C) 2011, Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 10 matching lines...) Expand all Loading... |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 import copy | 29 import copy |
| 30 import logging | 30 import logging |
| 31 import functools |
| 31 | 32 |
| 32 from webkitpy.common.memoized import memoized | 33 from webkitpy.common.memoized import memoized |
| 33 from functools import reduce | |
| 34 | 34 |
| 35 _log = logging.getLogger(__name__) | 35 _log = logging.getLogger(__name__) |
| 36 | 36 |
| 37 | 37 |
| 38 # FIXME: Should this function be somewhere more general? | 38 # FIXME: Should this function be somewhere more general? |
| 39 def _invert_dictionary(dictionary): | 39 def _invert_dictionary(dictionary): |
| 40 inverted_dictionary = {} | 40 inverted_dictionary = {} |
| 41 for key, value in dictionary.items(): | 41 for key, value in dictionary.items(): |
| 42 if inverted_dictionary.get(value): | 42 if inverted_dictionary.get(value): |
| 43 inverted_dictionary[value].append(key) | 43 inverted_dictionary[value].append(key) |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 94 # to the baseline in the platform directory, we need to append just foo-
expected.png to the directory. | 94 # to the baseline in the platform directory, we need to append just foo-
expected.png to the directory. |
| 95 virtual_suite = self._virtual_suite(baseline_name) | 95 virtual_suite = self._virtual_suite(baseline_name) |
| 96 if virtual_suite: | 96 if virtual_suite: |
| 97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name
) + 1:] | 97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name
) + 1:] |
| 98 else: | 98 else: |
| 99 baseline_name_without_virtual = baseline_name | 99 baseline_name_without_virtual = baseline_name |
| 100 return self._filesystem.join(self._webkit_base, directory, baseline_name
_without_virtual) | 100 return self._filesystem.join(self._webkit_base, directory, baseline_name
_without_virtual) |
| 101 | 101 |
| 102 def read_results_by_directory(self, baseline_name): | 102 def read_results_by_directory(self, baseline_name): |
| 103 results_by_directory = {} | 103 results_by_directory = {} |
| 104 directories = reduce(set.union, map(set, [self._relative_baseline_search
_paths( | 104 directories = functools.reduce(set.union, map(set, [self._relative_basel
ine_search_paths( |
| 105 port, baseline_name) for port in self._ports.values()])) | 105 port, baseline_name) for port in self._ports.values()])) |
| 106 | 106 |
| 107 for directory in directories: | 107 for directory in directories: |
| 108 path = self._join_directory(directory, baseline_name) | 108 path = self._join_directory(directory, baseline_name) |
| 109 if self._filesystem.exists(path): | 109 if self._filesystem.exists(path): |
| 110 results_by_directory[directory] = self._filesystem.sha1(path) | 110 results_by_directory[directory] = self._filesystem.sha1(path) |
| 111 return results_by_directory | 111 return results_by_directory |
| 112 | 112 |
| 113 def _results_by_port_name(self, results_by_directory, baseline_name): | 113 def _results_by_port_name(self, results_by_directory, baseline_name): |
| 114 results_by_port_name = {} | 114 results_by_port_name = {} |
| (...skipping 12 matching lines...) Expand all Loading... |
| 127 directories.add(directory) | 127 directories.add(directory) |
| 128 return directories | 128 return directories |
| 129 | 129 |
| 130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name)
: | 130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name)
: |
| 131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor
respond | 131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor
respond |
| 132 # to a specific platform. As such, it's the only one where the baseline
in fallback directories | 132 # to a specific platform. As such, it's the only one where the baseline
in fallback directories |
| 133 # immediately before it can be promoted up, i.e. if win and mac | 133 # immediately before it can be promoted up, i.e. if win and mac |
| 134 # have the same baseline, then it can be promoted up to be the LayoutTes
ts baseline. | 134 # have the same baseline, then it can be promoted up to be the LayoutTes
ts baseline. |
| 135 # All other baselines can only be removed if they're redundant with a ba
seline earlier | 135 # All other baselines can only be removed if they're redundant with a ba
seline earlier |
| 136 # in the fallback order. They can never promoted up. | 136 # in the fallback order. They can never promoted up. |
| 137 directories_immediately_preceding_root = self._directories_immediately_p
receding_root(baseline_name) | 137 immediately_preceding_root = self._directories_immediately_preceding_roo
t(baseline_name) |
| 138 | 138 |
| 139 shared_result = None | 139 shared_result = None |
| 140 root_baseline_unused = False | 140 root_baseline_unused = False |
| 141 for directory in directories_immediately_preceding_root: | 141 for directory in immediately_preceding_root: |
| 142 this_result = new_results_by_directory.get(directory) | 142 this_result = new_results_by_directory.get(directory) |
| 143 | 143 |
| 144 # If any of these directories don't have a baseline, there's no opti
mization we can do. | 144 # If any of these directories don't have a baseline, there's no opti
mization we can do. |
| 145 if not this_result: | 145 if not this_result: |
| 146 return | 146 return |
| 147 | 147 |
| 148 if not shared_result: | 148 if not shared_result: |
| 149 shared_result = this_result | 149 shared_result = this_result |
| 150 elif shared_result != this_result: | 150 elif shared_result != this_result: |
| 151 root_baseline_unused = True | 151 root_baseline_unused = True |
| 152 | 152 |
| 153 baseline_root = self._baseline_root(baseline_name) | 153 baseline_root = self._baseline_root(baseline_name) |
| 154 | 154 |
| 155 # The root baseline is unused if all the directories immediately precedi
ng the root | 155 # The root baseline is unused if all the directories immediately precedi
ng the root |
| 156 # have a baseline, but have different baselines, so the baselines can't
be promoted up. | 156 # have a baseline, but have different baselines, so the baselines can't
be promoted up. |
| 157 if root_baseline_unused: | 157 if root_baseline_unused: |
| 158 if baseline_root in new_results_by_directory: | 158 if baseline_root in new_results_by_directory: |
| 159 del new_results_by_directory[baseline_root] | 159 del new_results_by_directory[baseline_root] |
| 160 return | 160 return |
| 161 | 161 |
| 162 new_results_by_directory[baseline_root] = shared_result | 162 new_results_by_directory[baseline_root] = shared_result |
| 163 for directory in directories_immediately_preceding_root: | 163 for directory in immediately_preceding_root: |
| 164 del new_results_by_directory[directory] | 164 del new_results_by_directory[directory] |
| 165 | 165 |
| 166 def _find_optimal_result_placement(self, baseline_name): | 166 def _find_optimal_result_placement(self, baseline_name): |
| 167 results_by_directory = self.read_results_by_directory(baseline_name) | 167 results_by_directory = self.read_results_by_directory(baseline_name) |
| 168 results_by_port_name = self._results_by_port_name(results_by_directory,
baseline_name) | 168 results_by_port_name = self._results_by_port_name(results_by_directory,
baseline_name) |
| 169 port_names_by_result = _invert_dictionary(results_by_port_name) | |
| 170 | 169 |
| 171 new_results_by_directory = self._remove_redundant_results( | 170 new_results_by_directory = self._remove_redundant_results( |
| 172 results_by_directory, results_by_port_name, port_names_by_result, ba
seline_name) | 171 results_by_directory, results_by_port_name, baseline_name) |
| 173 self._optimize_result_for_root(new_results_by_directory, baseline_name) | 172 self._optimize_result_for_root(new_results_by_directory, baseline_name) |
| 174 | 173 |
| 175 return results_by_directory, new_results_by_directory | 174 return results_by_directory, new_results_by_directory |
| 176 | 175 |
| 177 def _remove_redundant_results(self, results_by_directory, results_by_port_na
me, port_names_by_result, baseline_name): | 176 def _remove_redundant_results(self, results_by_directory, results_by_port_na
me, baseline_name): |
| 178 new_results_by_directory = copy.copy(results_by_directory) | 177 new_results_by_directory = copy.copy(results_by_directory) |
| 179 for port_name, port in self._ports.items(): | 178 for port_name, port in self._ports.items(): |
| 180 current_result = results_by_port_name.get(port_name) | 179 current_result = results_by_port_name.get(port_name) |
| 181 | 180 |
| 182 # This happens if we're missing baselines for a port. | 181 # This happens if we're missing baselines for a port. |
| 183 if not current_result: | 182 if not current_result: |
| 184 continue | 183 continue |
| 185 | 184 |
| 186 fallback_path = self._relative_baseline_search_paths(port, baseline_
name) | 185 fallback_path = self._relative_baseline_search_paths(port, baseline_
name) |
| 187 current_index, current_directory = self._find_in_fallbackpath(fallba
ck_path, current_result, new_results_by_directory) | 186 current_index, current_directory = self._find_in_fallbackpath(fallba
ck_path, current_result, new_results_by_directory) |
| 188 for index in range(current_index + 1, len(fallback_path)): | 187 for index in range(current_index + 1, len(fallback_path)): |
| 189 new_directory = fallback_path[index] | 188 new_directory = fallback_path[index] |
| 190 if not new_directory in new_results_by_directory: | 189 if new_directory not in new_results_by_directory: |
| 191 # No result for this baseline in this directory. | 190 # No result for this baseline in this directory. |
| 192 continue | 191 continue |
| 193 elif new_results_by_directory[new_directory] == current_result: | 192 elif new_results_by_directory[new_directory] == current_result: |
| 194 # Result for new_directory are redundant with the result ear
lier in the fallback order. | 193 # Result for new_directory are redundant with the result ear
lier in the fallback order. |
| 195 if current_directory in new_results_by_directory: | 194 if current_directory in new_results_by_directory: |
| 196 del new_results_by_directory[current_directory] | 195 del new_results_by_directory[current_directory] |
| 197 else: | 196 else: |
| 198 # The new_directory contains a different result, so stop try
ing to push results up. | 197 # The new_directory contains a different result, so stop try
ing to push results up. |
| 199 break | 198 break |
| 200 | 199 |
| (...skipping 10 matching lines...) Expand all Loading... |
| 211 if filename.startswith(platform_dir): | 210 if filename.startswith(platform_dir): |
| 212 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] | 211 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] |
| 213 platform_dir = self._filesystem.join(self._webkit_base, platform_dir) | 212 platform_dir = self._filesystem.join(self._webkit_base, platform_dir) |
| 214 if filename.startswith(platform_dir): | 213 if filename.startswith(platform_dir): |
| 215 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] | 214 return filename.replace(platform_dir, '').split(self._filesystem.sep
)[0] |
| 216 return '(generic)' | 215 return '(generic)' |
| 217 | 216 |
| 218 def _move_baselines(self, baseline_name, results_by_directory, new_results_b
y_directory): | 217 def _move_baselines(self, baseline_name, results_by_directory, new_results_b
y_directory): |
| 219 data_for_result = {} | 218 data_for_result = {} |
| 220 for directory, result in results_by_directory.items(): | 219 for directory, result in results_by_directory.items(): |
| 221 if not result in data_for_result: | 220 if result not in data_for_result: |
| 222 source = self._join_directory(directory, baseline_name) | 221 source = self._join_directory(directory, baseline_name) |
| 223 data_for_result[result] = self._filesystem.read_binary_file(sour
ce) | 222 data_for_result[result] = self._filesystem.read_binary_file(sour
ce) |
| 224 | 223 |
| 225 fs_files = [] | 224 fs_files = [] |
| 226 for directory, result in results_by_directory.items(): | 225 for directory, result in results_by_directory.items(): |
| 227 if new_results_by_directory.get(directory) != result: | 226 if new_results_by_directory.get(directory) != result: |
| 228 file_name = self._join_directory(directory, baseline_name) | 227 file_name = self._join_directory(directory, baseline_name) |
| 229 if self._filesystem.exists(file_name): | 228 if self._filesystem.exists(file_name): |
| 230 fs_files.append(file_name) | 229 fs_files.append(file_name) |
| 231 | 230 |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 282 _log.debug(" %s:", basename) | 281 _log.debug(" %s:", basename) |
| 283 _log.debug(" Before: ") | 282 _log.debug(" Before: ") |
| 284 self.write_by_directory(results_by_directory, _log.debug, " ") | 283 self.write_by_directory(results_by_directory, _log.debug, " ") |
| 285 _log.debug(" After: ") | 284 _log.debug(" After: ") |
| 286 self.write_by_directory(new_results_by_directory, _log.debug, " ") | 285 self.write_by_directory(new_results_by_directory, _log.debug, " ") |
| 287 | 286 |
| 288 self._move_baselines(baseline_name, results_by_directory, new_results_by
_directory) | 287 self._move_baselines(baseline_name, results_by_directory, new_results_by
_directory) |
| 289 return True | 288 return True |
| 290 | 289 |
| 291 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name): | 290 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name): |
| 292 virtual_root_expected_baseline_path = self._filesystem.join(self._layout
_tests_dir, baseline_name) | 291 virtual_root_baseline_path = self._filesystem.join(self._layout_tests_di
r, baseline_name) |
| 293 if not self._filesystem.exists(virtual_root_expected_baseline_path): | 292 if not self._filesystem.exists(virtual_root_baseline_path): |
| 294 return | 293 return |
| 295 root_sha1 = self._filesystem.sha1(virtual_root_expected_baseline_path) | 294 root_sha1 = self._filesystem.sha1(virtual_root_baseline_path) |
| 296 | 295 |
| 297 results_by_directory = self.read_results_by_directory(non_virtual_baseli
ne_name) | 296 results_by_directory = self.read_results_by_directory(non_virtual_baseli
ne_name) |
| 298 # See if all the immediate predecessors of the virtual root have the sam
e expected result. | 297 # See if all the immediate predecessors of the virtual root have the sam
e expected result. |
| 299 for port in self._ports.values(): | 298 for port in self._ports.values(): |
| 300 directories = self._relative_baseline_search_paths(port, non_virtual
_baseline_name) | 299 directories = self._relative_baseline_search_paths(port, non_virtual
_baseline_name) |
| 301 for directory in directories: | 300 for directory in directories: |
| 302 if directory not in results_by_directory: | 301 if directory not in results_by_directory: |
| 303 continue | 302 continue |
| 304 if results_by_directory[directory] != root_sha1: | 303 if results_by_directory[directory] != root_sha1: |
| 305 return | 304 return |
| 306 break | 305 break |
| 307 | 306 |
| 308 _log.debug("Deleting redundant virtual root expected result.") | 307 _log.debug("Deleting redundant virtual root expected result.") |
| 309 _log.debug(" Deleting (file system): " + virtual_root_expected_baseli
ne_path) | 308 _log.debug(" Deleting (file system): " + virtual_root_baseline_path) |
| 310 self._filesystem.remove(virtual_root_expected_baseline_path) | 309 self._filesystem.remove(virtual_root_baseline_path) |
| 311 | 310 |
| 312 def optimize(self, baseline_name): | 311 def optimize(self, baseline_name): |
| 313 # The virtual fallback path is the same as the non-virtual one tacked on
to the bottom of the non-virtual path. | 312 # The virtual fallback path is the same as the non-virtual one tacked on
to the bottom of the non-virtual path. |
| 314 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb
UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for | 313 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb
UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for |
| 315 # a visual representation of this. | 314 # a visual representation of this. |
| 316 # | 315 # |
| 317 # So, we can optimize the virtual path, then the virtual root and then t
he regular path. | 316 # So, we can optimize the virtual path, then the virtual root and then t
he regular path. |
| 318 | 317 |
| 319 _log.debug("Optimizing regular fallback path.") | 318 _log.debug("Optimizing regular fallback path.") |
| 320 result = self._optimize_subtree(baseline_name) | 319 result = self._optimize_subtree(baseline_name) |
| 321 non_virtual_baseline_name = self._virtual_base(baseline_name) | 320 non_virtual_baseline_name = self._virtual_base(baseline_name) |
| 322 if not non_virtual_baseline_name: | 321 if not non_virtual_baseline_name: |
| 323 return result | 322 return result |
| 324 | 323 |
| 325 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name) | 324 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name) |
| 326 | 325 |
| 327 _log.debug("Optimizing non-virtual fallback path.") | 326 _log.debug("Optimizing non-virtual fallback path.") |
| 328 result |= self._optimize_subtree(non_virtual_baseline_name) | 327 result |= self._optimize_subtree(non_virtual_baseline_name) |
| 329 return result | 328 return result |
| OLD | NEW |