Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(246)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations.py

Issue 326483002: Make update-flaky-tests work for debug builders as well. (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: add tests Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after
103 # is a detail of this class. 103 # is a detail of this class.
104 def occurances_and_type_from_result_item(self, item): 104 def occurances_and_type_from_result_item(self, item):
105 return item[self.RLE_LENGTH], item[self.RLE_VALUE] 105 return item[self.RLE_LENGTH], item[self.RLE_VALUE]
106 106
107 107
108 class BotTestExpectationsFactory(object): 108 class BotTestExpectationsFactory(object):
109 RESULTS_URL_PREFIX = 'http://test-results.appspot.com/testfile?master=Chromi umWebkit&testtype=layout-tests&name=results-small.json&builder=' 109 RESULTS_URL_PREFIX = 'http://test-results.appspot.com/testfile?master=Chromi umWebkit&testtype=layout-tests&name=results-small.json&builder='
110 110
111 def _results_json_for_port(self, port_name, builder_category): 111 def _results_json_for_port(self, port_name, builder_category):
112 if builder_category == 'deps': 112 if builder_category == 'deps':
113 builder_name = builders.deps_builder_name_for_port_name(port_name) 113 builder = builders.deps_builder_name_for_port_name(port_name)
114 else: 114 else:
115 builder_name = builders.builder_name_for_port_name(port_name) 115 builder = builders.builder_name_for_port_name(port_name)
116 116
117 if not builder_name: 117 if not builder:
118 return None 118 return None
119 results_url = self.RESULTS_URL_PREFIX + urllib.quote(builder_name) 119 return self._results_json_for_builder(builder)
120
121 def _results_json_for_builder(self, builder):
122 results_url = self.RESULTS_URL_PREFIX + urllib.quote(builder)
120 try: 123 try:
121 _log.debug('Fetching flakiness data from appengine.') 124 _log.debug('Fetching flakiness data from appengine.')
122 return ResultsJSON(builder_name, json.load(urllib2.urlopen(results_u rl))) 125 return ResultsJSON(builder, json.load(urllib2.urlopen(results_url)))
123 except urllib2.URLError as error: 126 except urllib2.URLError as error:
124 _log.warning('Could not retrieve flakiness data from the bot. url: %s', results_url) 127 _log.warning('Could not retrieve flakiness data from the bot. url: %s', results_url)
125 _log.warning(error) 128 _log.warning(error)
126 129
127 def expectations_for_port(self, port_name, builder_category='layout'): 130 def expectations_for_port(self, port_name, builder_category='layout'):
131 # FIXME: This only grabs release builder's flakiness data. If we're runn ing debug,
132 # when we should grab the debug builder's data.
133 # FIXME: What should this do if there is no debug builder for a port, e. g. we have
134 # no debug XP builder? Should it use the release bot or another Windows debug bot?
135 # At the very least, it should log an error.
128 results_json = self._results_json_for_port(port_name, builder_category) 136 results_json = self._results_json_for_port(port_name, builder_category)
129 if not results_json: 137 if not results_json:
130 return None 138 return None
131 return BotTestExpectations(results_json) 139 return BotTestExpectations(results_json)
132 140
141 def expectations_for_builder(self, builder):
142 results_json = self._results_json_for_builder(builder)
143 if not results_json:
144 return None
145 return BotTestExpectations(results_json)
133 146
134 class BotTestExpectations(object): 147 class BotTestExpectations(object):
135 # FIXME: Get this from the json instead of hard-coding it. 148 # FIXME: Get this from the json instead of hard-coding it.
136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] 149 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y']
137 150
138 # specifiers arg is used in unittests to avoid the static dependency on buil ders. 151 # specifiers arg is used in unittests to avoid the static dependency on buil ders.
139 def __init__(self, results_json, specifiers=None): 152 def __init__(self, results_json, specifiers=None):
140 self.results_json = results_json 153 self.results_json = results_json
141 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu lts_json.builder_name)) 154 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu lts_json.builder_name))
142 155
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after
233 # Otherwise, we include lots of false-positives due to tests tha t fail 246 # Otherwise, we include lots of false-positives due to tests tha t fail
234 # for a couple runs and then start passing. 247 # for a couple runs and then start passing.
235 # FIXME: Maybe we should make this more liberal and consider it a flake 248 # FIXME: Maybe we should make this more liberal and consider it a flake
236 # even if we only see that failure once. 249 # even if we only see that failure once.
237 seen_results[result_type] = True 250 seen_results[result_type] = True
238 continue 251 continue
239 252
240 results_map[result_type] = True 253 results_map[result_type] = True
241 254
242 return results_map.keys() 255 return results_map.keys()
OLDNEW
« no previous file with comments | « no previous file | Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698