Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(274)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations.py

Issue 301853003: Make update-flaky-tests to work again and support all bots (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Removed use of operator.attrgetter Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
128 results_json = self._results_json_for_port(port_name, builder_category) 128 results_json = self._results_json_for_port(port_name, builder_category)
129 if not results_json: 129 if not results_json:
130 return None 130 return None
131 return BotTestExpectations(results_json) 131 return BotTestExpectations(results_json)
132 132
133 133
134 class BotTestExpectations(object): 134 class BotTestExpectations(object):
135 # FIXME: Get this from the json instead of hard-coding it. 135 # FIXME: Get this from the json instead of hard-coding it.
136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] 136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y']
137 137
138 def __init__(self, results_json): 138 # specifiers arg is used in unittests to avoid the static dependency on buil ders.
139 def __init__(self, results_json, specifiers=None):
139 self.results_json = results_json 140 self.results_json = results_json
141 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu lts_json.builder_name))
140 142
141 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type s, bug_urls): 143 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type s, bug_urls):
142 line = TestExpectationLine() 144 line = TestExpectationLine()
143 line.original_string = test_path 145 line.original_string = test_path
144 line.name = test_path 146 line.name = test_path
145 line.filename = test_path 147 line.filename = test_path
146 line.specifiers = bug_urls if bug_urls else "" 148 line.path = test_path # FIXME: Should this be normpath?
149 line.matching_tests = [test_path]
150 line.bugs = bug_urls if bug_urls else ["Bug(gardener)"]
147 line.expectations = sorted(map(self.results_json.expectation_for_type, f laky_types)) 151 line.expectations = sorted(map(self.results_json.expectation_for_type, f laky_types))
152 line.specifiers = self.specifiers
148 return line 153 return line
149 154
150 def flakes_by_path(self, only_ignore_very_flaky): 155 def flakes_by_path(self, only_ignore_very_flaky):
151 """Sets test expectations to bot results if there are at least two disti nct results.""" 156 """Sets test expectations to bot results if there are at least two disti nct results."""
152 flakes_by_path = {} 157 flakes_by_path = {}
153 for test_path, entry in self.results_json.walk_results(): 158 for test_path, entry in self.results_json.walk_results():
154 results_dict = entry[self.results_json.RESULTS_KEY] 159 results_dict = entry[self.results_json.RESULTS_KEY]
155 flaky_types = self._flaky_types_in_results(results_dict, only_ignore _very_flaky) 160 flaky_types = self._flaky_types_in_results(results_dict, only_ignore _very_flaky)
156 if len(flaky_types) <= 1: 161 if len(flaky_types) <= 1:
157 continue 162 continue
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after
196 additional_expectations = set(e for e in result_exp if not expected( e)) 201 additional_expectations = set(e for e in result_exp if not expected( e))
197 202
198 # Test did not have unexpected results. 203 # Test did not have unexpected results.
199 if not additional_expectations: 204 if not additional_expectations:
200 continue 205 continue
201 206
202 expectations.update(additional_expectations) 207 expectations.update(additional_expectations)
203 unexpected_results_by_path[test_path] = sorted(map(exp_to_string, ex pectations)) 208 unexpected_results_by_path[test_path] = sorted(map(exp_to_string, ex pectations))
204 return unexpected_results_by_path 209 return unexpected_results_by_path
205 210
206 def expectation_lines(self): 211 def expectation_lines(self, only_ignore_very_flaky=False):
207 lines = [] 212 lines = []
208 for test_path, entry in self.results_json.walk_results(): 213 for test_path, entry in self.results_json.walk_results():
209 results_array = entry[self.results_json.RESULTS_KEY] 214 results_array = entry[self.results_json.RESULTS_KEY]
210 flaky_types = self._flaky_types_in_results(results_array, False) 215 flaky_types = self._flaky_types_in_results(results_array, only_ignor e_very_flaky)
211 if len(flaky_types) > 1: 216 if len(flaky_types) > 1:
212 bug_urls = entry.get(self.results_json.BUGS_KEY) 217 bug_urls = entry.get(self.results_json.BUGS_KEY)
213 line = self._line_from_test_and_flaky_types_and_bug_urls(test_pa th, flaky_types, bug_urls) 218 line = self._line_from_test_and_flaky_types_and_bug_urls(test_pa th, flaky_types, bug_urls)
214 lines.append(line) 219 lines.append(line)
215 return lines 220 return lines
216 221
217 def _flaky_types_in_results(self, run_length_encoded_results, only_ignore_ve ry_flaky=False): 222 def _flaky_types_in_results(self, run_length_encoded_results, only_ignore_ve ry_flaky=False):
218 results_map = {} 223 results_map = {}
219 seen_results = {} 224 seen_results = {}
220 225
221 for result_item in run_length_encoded_results: 226 for result_item in run_length_encoded_results:
222 _, result_type = self.results_json.occurances_and_type_from_result_i tem(result_item) 227 _, result_type = self.results_json.occurances_and_type_from_result_i tem(result_item)
223 if result_type in self.RESULT_TYPES_TO_IGNORE: 228 if result_type in self.RESULT_TYPES_TO_IGNORE:
224 continue 229 continue
225 230
226 if only_ignore_very_flaky and result_type not in seen_results: 231 if only_ignore_very_flaky and result_type not in seen_results:
227 # Only consider a short-lived result if we've seen it more than once. 232 # Only consider a short-lived result if we've seen it more than once.
228 # Otherwise, we include lots of false-positives due to tests tha t fail 233 # Otherwise, we include lots of false-positives due to tests tha t fail
229 # for a couple runs and then start passing. 234 # for a couple runs and then start passing.
230 # FIXME: Maybe we should make this more liberal and consider it a flake 235 # FIXME: Maybe we should make this more liberal and consider it a flake
231 # even if we only see that failure once. 236 # even if we only see that failure once.
232 seen_results[result_type] = True 237 seen_results[result_type] = True
233 continue 238 continue
234 239
235 results_map[result_type] = True 240 results_map[result_type] = True
236 241
237 return results_map.keys() 242 return results_map.keys()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698