OLD | NEW |
1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
103 # is a detail of this class. | 103 # is a detail of this class. |
104 def occurances_and_type_from_result_item(self, item): | 104 def occurances_and_type_from_result_item(self, item): |
105 return item[self.RLE_LENGTH], item[self.RLE_VALUE] | 105 return item[self.RLE_LENGTH], item[self.RLE_VALUE] |
106 | 106 |
107 | 107 |
108 class BotTestExpectationsFactory(object): | 108 class BotTestExpectationsFactory(object): |
109 RESULTS_URL_PREFIX = 'http://test-results.appspot.com/testfile?master=Chromi
umWebkit&testtype=layout-tests&name=results-small.json&builder=' | 109 RESULTS_URL_PREFIX = 'http://test-results.appspot.com/testfile?master=Chromi
umWebkit&testtype=layout-tests&name=results-small.json&builder=' |
110 | 110 |
111 def _results_json_for_port(self, port_name, builder_category): | 111 def _results_json_for_port(self, port_name, builder_category): |
112 if builder_category == 'deps': | 112 if builder_category == 'deps': |
113 builder_name = builders.deps_builder_name_for_port_name(port_name) | 113 builder = builders.deps_builder_name_for_port_name(port_name) |
114 else: | 114 else: |
115 builder_name = builders.builder_name_for_port_name(port_name) | 115 builder = builders.builder_name_for_port_name(port_name) |
116 | 116 |
117 if not builder_name: | 117 if not builder: |
118 return None | 118 return None |
119 results_url = self.RESULTS_URL_PREFIX + urllib.quote(builder_name) | 119 return self._results_json_for_builder(builder) |
| 120 |
| 121 def _results_json_for_builder(self, builder): |
| 122 results_url = self.RESULTS_URL_PREFIX + urllib.quote(builder) |
120 try: | 123 try: |
121 _log.debug('Fetching flakiness data from appengine.') | 124 _log.debug('Fetching flakiness data from appengine.') |
122 return ResultsJSON(builder_name, json.load(urllib2.urlopen(results_u
rl))) | 125 return ResultsJSON(builder, json.load(urllib2.urlopen(results_url))) |
123 except urllib2.URLError as error: | 126 except urllib2.URLError as error: |
124 _log.warning('Could not retrieve flakiness data from the bot. url:
%s', results_url) | 127 _log.warning('Could not retrieve flakiness data from the bot. url:
%s', results_url) |
125 _log.warning(error) | 128 _log.warning(error) |
126 | 129 |
127 def expectations_for_port(self, port_name, builder_category='layout'): | 130 def expectations_for_port(self, port_name, builder_category='layout'): |
| 131 # FIXME: This only grabs release builder's flakiness data. If we're runn
ing debug, |
| 132 # when we should grab the debug builder's data. |
| 133 # FIXME: What should this do if there is no debug builder for a port, e.
g. we have |
| 134 # no debug XP builder? Should it use the release bot or another Windows
debug bot? |
| 135 # At the very least, it should log an error. |
128 results_json = self._results_json_for_port(port_name, builder_category) | 136 results_json = self._results_json_for_port(port_name, builder_category) |
129 if not results_json: | 137 if not results_json: |
130 return None | 138 return None |
131 return BotTestExpectations(results_json) | 139 return BotTestExpectations(results_json) |
132 | 140 |
| 141 def expectations_for_builder(self, builder): |
| 142 results_json = self._results_json_for_builder(builder) |
| 143 if not results_json: |
| 144 return None |
| 145 return BotTestExpectations(results_json) |
133 | 146 |
134 class BotTestExpectations(object): | 147 class BotTestExpectations(object): |
135 # FIXME: Get this from the json instead of hard-coding it. | 148 # FIXME: Get this from the json instead of hard-coding it. |
136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] | 149 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] |
137 | 150 |
138 # specifiers arg is used in unittests to avoid the static dependency on buil
ders. | 151 # specifiers arg is used in unittests to avoid the static dependency on buil
ders. |
139 def __init__(self, results_json, specifiers=None): | 152 def __init__(self, results_json, specifiers=None): |
140 self.results_json = results_json | 153 self.results_json = results_json |
141 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu
lts_json.builder_name)) | 154 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu
lts_json.builder_name)) |
142 | 155 |
(...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
233 # Otherwise, we include lots of false-positives due to tests tha
t fail | 246 # Otherwise, we include lots of false-positives due to tests tha
t fail |
234 # for a couple runs and then start passing. | 247 # for a couple runs and then start passing. |
235 # FIXME: Maybe we should make this more liberal and consider it
a flake | 248 # FIXME: Maybe we should make this more liberal and consider it
a flake |
236 # even if we only see that failure once. | 249 # even if we only see that failure once. |
237 seen_results[result_type] = True | 250 seen_results[result_type] = True |
238 continue | 251 continue |
239 | 252 |
240 results_map[result_type] = True | 253 results_map[result_type] = True |
241 | 254 |
242 return results_map.keys() | 255 return results_map.keys() |
OLD | NEW |