Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(496)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/common/checkout/baseline_optimizer.py

Issue 2578213005: Use underscores to separate words in filenames in webkitpy. (Closed)
Patch Set: Fix check for attribute in output_capture.py. Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | third_party/WebKit/Tools/Scripts/webkitpy/common/checkout/baseline_optimizer_unittest.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (C) 2011, Google Inc. All rights reserved. 1 # Copyright (C) 2011, Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 10 matching lines...) Expand all
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 28
29 import copy 29 import copy
30 import logging 30 import logging
31 import functools
31 32
32 from webkitpy.common.memoized import memoized 33 from webkitpy.common.memoized import memoized
33 from functools import reduce
34 34
35 _log = logging.getLogger(__name__) 35 _log = logging.getLogger(__name__)
36 36
37 37
38 # FIXME: Should this function be somewhere more general? 38 # FIXME: Should this function be somewhere more general?
39 def _invert_dictionary(dictionary): 39 def _invert_dictionary(dictionary):
40 inverted_dictionary = {} 40 inverted_dictionary = {}
41 for key, value in dictionary.items(): 41 for key, value in dictionary.items():
42 if inverted_dictionary.get(value): 42 if inverted_dictionary.get(value):
43 inverted_dictionary[value].append(key) 43 inverted_dictionary[value].append(key)
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
94 # to the baseline in the platform directory, we need to append just foo- expected.png to the directory. 94 # to the baseline in the platform directory, we need to append just foo- expected.png to the directory.
95 virtual_suite = self._virtual_suite(baseline_name) 95 virtual_suite = self._virtual_suite(baseline_name)
96 if virtual_suite: 96 if virtual_suite:
97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name ) + 1:] 97 baseline_name_without_virtual = baseline_name[len(virtual_suite.name ) + 1:]
98 else: 98 else:
99 baseline_name_without_virtual = baseline_name 99 baseline_name_without_virtual = baseline_name
100 return self._filesystem.join(self._webkit_base, directory, baseline_name _without_virtual) 100 return self._filesystem.join(self._webkit_base, directory, baseline_name _without_virtual)
101 101
102 def read_results_by_directory(self, baseline_name): 102 def read_results_by_directory(self, baseline_name):
103 results_by_directory = {} 103 results_by_directory = {}
104 directories = reduce(set.union, map(set, [self._relative_baseline_search _paths( 104 directories = functools.reduce(set.union, map(set, [self._relative_basel ine_search_paths(
105 port, baseline_name) for port in self._ports.values()])) 105 port, baseline_name) for port in self._ports.values()]))
106 106
107 for directory in directories: 107 for directory in directories:
108 path = self._join_directory(directory, baseline_name) 108 path = self._join_directory(directory, baseline_name)
109 if self._filesystem.exists(path): 109 if self._filesystem.exists(path):
110 results_by_directory[directory] = self._filesystem.sha1(path) 110 results_by_directory[directory] = self._filesystem.sha1(path)
111 return results_by_directory 111 return results_by_directory
112 112
113 def _results_by_port_name(self, results_by_directory, baseline_name): 113 def _results_by_port_name(self, results_by_directory, baseline_name):
114 results_by_port_name = {} 114 results_by_port_name = {}
(...skipping 12 matching lines...) Expand all
127 directories.add(directory) 127 directories.add(directory)
128 return directories 128 return directories
129 129
130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name) : 130 def _optimize_result_for_root(self, new_results_by_directory, baseline_name) :
131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor respond 131 # The root directory (i.e. LayoutTests) is the only one that doesn't cor respond
132 # to a specific platform. As such, it's the only one where the baseline in fallback directories 132 # to a specific platform. As such, it's the only one where the baseline in fallback directories
133 # immediately before it can be promoted up, i.e. if win and mac 133 # immediately before it can be promoted up, i.e. if win and mac
134 # have the same baseline, then it can be promoted up to be the LayoutTes ts baseline. 134 # have the same baseline, then it can be promoted up to be the LayoutTes ts baseline.
135 # All other baselines can only be removed if they're redundant with a ba seline earlier 135 # All other baselines can only be removed if they're redundant with a ba seline earlier
136 # in the fallback order. They can never promoted up. 136 # in the fallback order. They can never promoted up.
137 directories_immediately_preceding_root = self._directories_immediately_p receding_root(baseline_name) 137 immediately_preceding_root = self._directories_immediately_preceding_roo t(baseline_name)
138 138
139 shared_result = None 139 shared_result = None
140 root_baseline_unused = False 140 root_baseline_unused = False
141 for directory in directories_immediately_preceding_root: 141 for directory in immediately_preceding_root:
142 this_result = new_results_by_directory.get(directory) 142 this_result = new_results_by_directory.get(directory)
143 143
144 # If any of these directories don't have a baseline, there's no opti mization we can do. 144 # If any of these directories don't have a baseline, there's no opti mization we can do.
145 if not this_result: 145 if not this_result:
146 return 146 return
147 147
148 if not shared_result: 148 if not shared_result:
149 shared_result = this_result 149 shared_result = this_result
150 elif shared_result != this_result: 150 elif shared_result != this_result:
151 root_baseline_unused = True 151 root_baseline_unused = True
152 152
153 baseline_root = self._baseline_root(baseline_name) 153 baseline_root = self._baseline_root(baseline_name)
154 154
155 # The root baseline is unused if all the directories immediately precedi ng the root 155 # The root baseline is unused if all the directories immediately precedi ng the root
156 # have a baseline, but have different baselines, so the baselines can't be promoted up. 156 # have a baseline, but have different baselines, so the baselines can't be promoted up.
157 if root_baseline_unused: 157 if root_baseline_unused:
158 if baseline_root in new_results_by_directory: 158 if baseline_root in new_results_by_directory:
159 del new_results_by_directory[baseline_root] 159 del new_results_by_directory[baseline_root]
160 return 160 return
161 161
162 new_results_by_directory[baseline_root] = shared_result 162 new_results_by_directory[baseline_root] = shared_result
163 for directory in directories_immediately_preceding_root: 163 for directory in immediately_preceding_root:
164 del new_results_by_directory[directory] 164 del new_results_by_directory[directory]
165 165
166 def _find_optimal_result_placement(self, baseline_name): 166 def _find_optimal_result_placement(self, baseline_name):
167 results_by_directory = self.read_results_by_directory(baseline_name) 167 results_by_directory = self.read_results_by_directory(baseline_name)
168 results_by_port_name = self._results_by_port_name(results_by_directory, baseline_name) 168 results_by_port_name = self._results_by_port_name(results_by_directory, baseline_name)
169 port_names_by_result = _invert_dictionary(results_by_port_name)
170 169
171 new_results_by_directory = self._remove_redundant_results( 170 new_results_by_directory = self._remove_redundant_results(
172 results_by_directory, results_by_port_name, port_names_by_result, ba seline_name) 171 results_by_directory, results_by_port_name, baseline_name)
173 self._optimize_result_for_root(new_results_by_directory, baseline_name) 172 self._optimize_result_for_root(new_results_by_directory, baseline_name)
174 173
175 return results_by_directory, new_results_by_directory 174 return results_by_directory, new_results_by_directory
176 175
177 def _remove_redundant_results(self, results_by_directory, results_by_port_na me, port_names_by_result, baseline_name): 176 def _remove_redundant_results(self, results_by_directory, results_by_port_na me, baseline_name):
178 new_results_by_directory = copy.copy(results_by_directory) 177 new_results_by_directory = copy.copy(results_by_directory)
179 for port_name, port in self._ports.items(): 178 for port_name, port in self._ports.items():
180 current_result = results_by_port_name.get(port_name) 179 current_result = results_by_port_name.get(port_name)
181 180
182 # This happens if we're missing baselines for a port. 181 # This happens if we're missing baselines for a port.
183 if not current_result: 182 if not current_result:
184 continue 183 continue
185 184
186 fallback_path = self._relative_baseline_search_paths(port, baseline_ name) 185 fallback_path = self._relative_baseline_search_paths(port, baseline_ name)
187 current_index, current_directory = self._find_in_fallbackpath(fallba ck_path, current_result, new_results_by_directory) 186 current_index, current_directory = self._find_in_fallbackpath(fallba ck_path, current_result, new_results_by_directory)
188 for index in range(current_index + 1, len(fallback_path)): 187 for index in range(current_index + 1, len(fallback_path)):
189 new_directory = fallback_path[index] 188 new_directory = fallback_path[index]
190 if not new_directory in new_results_by_directory: 189 if new_directory not in new_results_by_directory:
191 # No result for this baseline in this directory. 190 # No result for this baseline in this directory.
192 continue 191 continue
193 elif new_results_by_directory[new_directory] == current_result: 192 elif new_results_by_directory[new_directory] == current_result:
194 # Result for new_directory are redundant with the result ear lier in the fallback order. 193 # Result for new_directory are redundant with the result ear lier in the fallback order.
195 if current_directory in new_results_by_directory: 194 if current_directory in new_results_by_directory:
196 del new_results_by_directory[current_directory] 195 del new_results_by_directory[current_directory]
197 else: 196 else:
198 # The new_directory contains a different result, so stop try ing to push results up. 197 # The new_directory contains a different result, so stop try ing to push results up.
199 break 198 break
200 199
(...skipping 10 matching lines...) Expand all
211 if filename.startswith(platform_dir): 210 if filename.startswith(platform_dir):
212 return filename.replace(platform_dir, '').split(self._filesystem.sep )[0] 211 return filename.replace(platform_dir, '').split(self._filesystem.sep )[0]
213 platform_dir = self._filesystem.join(self._webkit_base, platform_dir) 212 platform_dir = self._filesystem.join(self._webkit_base, platform_dir)
214 if filename.startswith(platform_dir): 213 if filename.startswith(platform_dir):
215 return filename.replace(platform_dir, '').split(self._filesystem.sep )[0] 214 return filename.replace(platform_dir, '').split(self._filesystem.sep )[0]
216 return '(generic)' 215 return '(generic)'
217 216
218 def _move_baselines(self, baseline_name, results_by_directory, new_results_b y_directory): 217 def _move_baselines(self, baseline_name, results_by_directory, new_results_b y_directory):
219 data_for_result = {} 218 data_for_result = {}
220 for directory, result in results_by_directory.items(): 219 for directory, result in results_by_directory.items():
221 if not result in data_for_result: 220 if result not in data_for_result:
222 source = self._join_directory(directory, baseline_name) 221 source = self._join_directory(directory, baseline_name)
223 data_for_result[result] = self._filesystem.read_binary_file(sour ce) 222 data_for_result[result] = self._filesystem.read_binary_file(sour ce)
224 223
225 fs_files = [] 224 fs_files = []
226 for directory, result in results_by_directory.items(): 225 for directory, result in results_by_directory.items():
227 if new_results_by_directory.get(directory) != result: 226 if new_results_by_directory.get(directory) != result:
228 file_name = self._join_directory(directory, baseline_name) 227 file_name = self._join_directory(directory, baseline_name)
229 if self._filesystem.exists(file_name): 228 if self._filesystem.exists(file_name):
230 fs_files.append(file_name) 229 fs_files.append(file_name)
231 230
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
282 _log.debug(" %s:", basename) 281 _log.debug(" %s:", basename)
283 _log.debug(" Before: ") 282 _log.debug(" Before: ")
284 self.write_by_directory(results_by_directory, _log.debug, " ") 283 self.write_by_directory(results_by_directory, _log.debug, " ")
285 _log.debug(" After: ") 284 _log.debug(" After: ")
286 self.write_by_directory(new_results_by_directory, _log.debug, " ") 285 self.write_by_directory(new_results_by_directory, _log.debug, " ")
287 286
288 self._move_baselines(baseline_name, results_by_directory, new_results_by _directory) 287 self._move_baselines(baseline_name, results_by_directory, new_results_by _directory)
289 return True 288 return True
290 289
291 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name): 290 def _optimize_virtual_root(self, baseline_name, non_virtual_baseline_name):
292 virtual_root_expected_baseline_path = self._filesystem.join(self._layout _tests_dir, baseline_name) 291 virtual_root_baseline_path = self._filesystem.join(self._layout_tests_di r, baseline_name)
293 if not self._filesystem.exists(virtual_root_expected_baseline_path): 292 if not self._filesystem.exists(virtual_root_baseline_path):
294 return 293 return
295 root_sha1 = self._filesystem.sha1(virtual_root_expected_baseline_path) 294 root_sha1 = self._filesystem.sha1(virtual_root_baseline_path)
296 295
297 results_by_directory = self.read_results_by_directory(non_virtual_baseli ne_name) 296 results_by_directory = self.read_results_by_directory(non_virtual_baseli ne_name)
298 # See if all the immediate predecessors of the virtual root have the sam e expected result. 297 # See if all the immediate predecessors of the virtual root have the sam e expected result.
299 for port in self._ports.values(): 298 for port in self._ports.values():
300 directories = self._relative_baseline_search_paths(port, non_virtual _baseline_name) 299 directories = self._relative_baseline_search_paths(port, non_virtual _baseline_name)
301 for directory in directories: 300 for directory in directories:
302 if directory not in results_by_directory: 301 if directory not in results_by_directory:
303 continue 302 continue
304 if results_by_directory[directory] != root_sha1: 303 if results_by_directory[directory] != root_sha1:
305 return 304 return
306 break 305 break
307 306
308 _log.debug("Deleting redundant virtual root expected result.") 307 _log.debug("Deleting redundant virtual root expected result.")
309 _log.debug(" Deleting (file system): " + virtual_root_expected_baseli ne_path) 308 _log.debug(" Deleting (file system): " + virtual_root_baseline_path)
310 self._filesystem.remove(virtual_root_expected_baseline_path) 309 self._filesystem.remove(virtual_root_baseline_path)
311 310
312 def optimize(self, baseline_name): 311 def optimize(self, baseline_name):
313 # The virtual fallback path is the same as the non-virtual one tacked on to the bottom of the non-virtual path. 312 # The virtual fallback path is the same as the non-virtual one tacked on to the bottom of the non-virtual path.
314 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for 313 # See https://docs.google.com/a/chromium.org/drawings/d/1eGdsIKzJ2dxDDBb UaIABrN4aMLD1bqJTfyxNGZsTdmg/edit for
315 # a visual representation of this. 314 # a visual representation of this.
316 # 315 #
317 # So, we can optimize the virtual path, then the virtual root and then t he regular path. 316 # So, we can optimize the virtual path, then the virtual root and then t he regular path.
318 317
319 _log.debug("Optimizing regular fallback path.") 318 _log.debug("Optimizing regular fallback path.")
320 result = self._optimize_subtree(baseline_name) 319 result = self._optimize_subtree(baseline_name)
321 non_virtual_baseline_name = self._virtual_base(baseline_name) 320 non_virtual_baseline_name = self._virtual_base(baseline_name)
322 if not non_virtual_baseline_name: 321 if not non_virtual_baseline_name:
323 return result 322 return result
324 323
325 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name) 324 self._optimize_virtual_root(baseline_name, non_virtual_baseline_name)
326 325
327 _log.debug("Optimizing non-virtual fallback path.") 326 _log.debug("Optimizing non-virtual fallback path.")
328 result |= self._optimize_subtree(non_virtual_baseline_name) 327 result |= self._optimize_subtree(non_virtual_baseline_name)
329 return result 328 return result
OLDNEW
« no previous file with comments | « no previous file | third_party/WebKit/Tools/Scripts/webkitpy/common/checkout/baseline_optimizer_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698