OLD | NEW |
1 # Copyright (c) 2009, Google Inc. All rights reserved. | 1 # Copyright (c) 2009, Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 26 matching lines...) Expand all Loading... |
37 from webkitpy.common.net.layouttestresults import LayoutTestResults | 37 from webkitpy.common.net.layouttestresults import LayoutTestResults |
38 from webkitpy.common.net.networktransaction import NetworkTransaction | 38 from webkitpy.common.net.networktransaction import NetworkTransaction |
39 from webkitpy.common.system.logutils import get_logger | 39 from webkitpy.common.system.logutils import get_logger |
40 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup | 40 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup |
41 | 41 |
42 | 42 |
43 _log = get_logger(__file__) | 43 _log = get_logger(__file__) |
44 | 44 |
45 | 45 |
46 class Builder(object): | 46 class Builder(object): |
| 47 |
47 def __init__(self, name, buildbot): | 48 def __init__(self, name, buildbot): |
48 self._name = name | 49 self._name = name |
49 self._buildbot = buildbot | 50 self._buildbot = buildbot |
50 self._builds_cache = {} | 51 self._builds_cache = {} |
51 self._revision_to_build_number = None | 52 self._revision_to_build_number = None |
52 | 53 |
53 def name(self): | 54 def name(self): |
54 return self._name | 55 return self._name |
55 | 56 |
56 def results_url(self): | 57 def results_url(self): |
57 return config_urls.chromium_results_url_base_for_builder(self._name) | 58 return config_urls.chromium_results_url_base_for_builder(self._name) |
58 | 59 |
59 def accumulated_results_url(self): | 60 def accumulated_results_url(self): |
60 return config_urls.chromium_accumulated_results_url_base_for_builder(sel
f._name) | 61 return config_urls.chromium_accumulated_results_url_base_for_builder(sel
f._name) |
61 | 62 |
62 def latest_layout_test_results_url(self): | 63 def latest_layout_test_results_url(self): |
63 return self.accumulated_results_url() or self.latest_cached_build().resu
lts_url(); | 64 return self.accumulated_results_url() or self.latest_cached_build().resu
lts_url() |
64 | 65 |
65 @memoized | 66 @memoized |
66 def latest_layout_test_results(self): | 67 def latest_layout_test_results(self): |
67 return self.fetch_layout_test_results(self.latest_layout_test_results_ur
l()) | 68 return self.fetch_layout_test_results(self.latest_layout_test_results_ur
l()) |
68 | 69 |
69 def _fetch_file_from_results(self, results_url, file_name): | 70 def _fetch_file_from_results(self, results_url, file_name): |
70 # It seems this can return None if the url redirects and then returns 40
4. | 71 # It seems this can return None if the url redirects and then returns 40
4. |
71 result = urllib2.urlopen("%s/%s" % (results_url, file_name)) | 72 result = urllib2.urlopen("%s/%s" % (results_url, file_name)) |
72 if not result: | 73 if not result: |
73 return None | 74 return None |
74 # urlopen returns a file-like object which sometimes works fine with str
() | 75 # urlopen returns a file-like object which sometimes works fine with str
() |
75 # but sometimes is a addinfourl object. In either case calling read() i
s correct. | 76 # but sometimes is a addinfourl object. In either case calling read() i
s correct. |
76 return result.read() | 77 return result.read() |
77 | 78 |
78 def fetch_layout_test_results(self, results_url): | 79 def fetch_layout_test_results(self, results_url): |
79 # FIXME: This should cache that the result was a 404 and stop hitting th
e network. | 80 # FIXME: This should cache that the result was a 404 and stop hitting th
e network. |
80 results_file = NetworkTransaction(convert_404_to_None=True).run(lambda:
self._fetch_file_from_results(results_url, "failing_results.json")) | 81 results_file = NetworkTransaction(convert_404_to_None=True).run( |
| 82 lambda: self._fetch_file_from_results(results_url, "failing_results.
json")) |
81 return LayoutTestResults.results_from_string(results_file) | 83 return LayoutTestResults.results_from_string(results_file) |
82 | 84 |
83 def url_encoded_name(self): | 85 def url_encoded_name(self): |
84 return urllib.quote(self._name) | 86 return urllib.quote(self._name) |
85 | 87 |
86 def url(self): | 88 def url(self): |
87 return "%s/builders/%s" % (self._buildbot.buildbot_url, self.url_encoded
_name()) | 89 return "%s/builders/%s" % (self._buildbot.buildbot_url, self.url_encoded
_name()) |
88 | 90 |
89 # This provides a single place to mock | 91 # This provides a single place to mock |
90 def _fetch_build(self, build_number): | 92 def _fetch_build(self, build_number): |
91 build_dictionary = self._buildbot._fetch_build_dictionary(self, build_nu
mber) | 93 build_dictionary = self._buildbot._fetch_build_dictionary(self, build_nu
mber) |
92 if not build_dictionary: | 94 if not build_dictionary: |
93 return None | 95 return None |
94 revision_string = build_dictionary['sourceStamp']['revision'] | 96 revision_string = build_dictionary['sourceStamp']['revision'] |
95 return Build(self, | 97 return Build(self, |
96 build_number=int(build_dictionary['number']), | 98 build_number=int(build_dictionary['number']), |
97 # 'revision' may be None if a trunk build was started by the force-b
uild button on the web page. | 99 # 'revision' may be None if a trunk build was started by th
e force-build button on the web page. |
98 revision=(int(revision_string) if revision_string else None), | 100 revision=(int(revision_string) if revision_string else None
), |
99 # Buildbot uses any nubmer other than 0 to mean fail. Since we fetc
h with | 101 # Buildbot uses any nubmer other than 0 to mean fail. Sinc
e we fetch with |
100 # filter=1, passing builds may contain no 'results' value. | 102 # filter=1, passing builds may contain no 'results' value. |
101 is_green=(not build_dictionary.get('results')), | 103 is_green=(not build_dictionary.get('results')), |
102 ) | 104 ) |
103 | 105 |
104 def build(self, build_number): | 106 def build(self, build_number): |
105 if not build_number: | 107 if not build_number: |
106 return None | 108 return None |
107 cached_build = self._builds_cache.get(build_number) | 109 cached_build = self._builds_cache.get(build_number) |
108 if cached_build: | 110 if cached_build: |
109 return cached_build | 111 return cached_build |
110 | 112 |
111 build = self._fetch_build(build_number) | 113 build = self._fetch_build(build_number) |
112 self._builds_cache[build_number] = build | 114 self._builds_cache[build_number] = build |
113 return build | 115 return build |
114 | 116 |
115 def latest_cached_build(self): | 117 def latest_cached_build(self): |
116 revision_build_pairs = self.revision_build_pairs_with_results() | 118 revision_build_pairs = self.revision_build_pairs_with_results() |
117 revision_build_pairs.sort(key=lambda i: i[1]) | 119 revision_build_pairs.sort(key=lambda i: i[1]) |
118 latest_build_number = revision_build_pairs[-1][1] | 120 latest_build_number = revision_build_pairs[-1][1] |
119 return self.build(latest_build_number) | 121 return self.build(latest_build_number) |
120 | 122 |
121 file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\)
") | 123 file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\)
") |
| 124 |
122 def _revision_and_build_for_filename(self, filename): | 125 def _revision_and_build_for_filename(self, filename): |
123 # Example: "r47483 (1)/" or "r47483 (1).zip" | 126 # Example: "r47483 (1)/" or "r47483 (1).zip" |
124 match = self.file_name_regexp.match(filename) | 127 match = self.file_name_regexp.match(filename) |
125 if not match: | 128 if not match: |
126 return None | 129 return None |
127 return (int(match.group("revision")), int(match.group("build_number"))) | 130 return (int(match.group("revision")), int(match.group("build_number"))) |
128 | 131 |
129 def _fetch_revision_to_build_map(self): | 132 def _fetch_revision_to_build_map(self): |
130 # All _fetch requests go through _buildbot for easier mocking | 133 # All _fetch requests go through _buildbot for easier mocking |
131 # FIXME: This should use NetworkTransaction's 404 handling instead. | 134 # FIXME: This should use NetworkTransaction's 404 handling instead. |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
163 # This assumes there can be only one build per revision, which is false, but
we don't care for now. | 166 # This assumes there can be only one build per revision, which is false, but
we don't care for now. |
164 def build_for_revision(self, revision, allow_failed_lookups=False): | 167 def build_for_revision(self, revision, allow_failed_lookups=False): |
165 # NOTE: This lookup will fail if that exact revision was never built. | 168 # NOTE: This lookup will fail if that exact revision was never built. |
166 build_number = self._revision_to_build_map().get(int(revision)) | 169 build_number = self._revision_to_build_map().get(int(revision)) |
167 if not build_number: | 170 if not build_number: |
168 return None | 171 return None |
169 build = self.build(build_number) | 172 build = self.build(build_number) |
170 if not build and allow_failed_lookups: | 173 if not build and allow_failed_lookups: |
171 # Builds for old revisions with fail to lookup via buildbot's json a
pi. | 174 # Builds for old revisions with fail to lookup via buildbot's json a
pi. |
172 build = Build(self, | 175 build = Build(self, |
173 build_number=build_number, | 176 build_number=build_number, |
174 revision=revision, | 177 revision=revision, |
175 is_green=False, | 178 is_green=False, |
176 ) | 179 ) |
177 return build | 180 return build |
178 | 181 |
179 | 182 |
180 class Build(object): | 183 class Build(object): |
| 184 |
181 def __init__(self, builder, build_number, revision, is_green): | 185 def __init__(self, builder, build_number, revision, is_green): |
182 self._builder = builder | 186 self._builder = builder |
183 self._number = build_number | 187 self._number = build_number |
184 self._revision = revision | 188 self._revision = revision |
185 self._is_green = is_green | 189 self._is_green = is_green |
186 | 190 |
187 @staticmethod | 191 @staticmethod |
188 def build_url(builder, build_number): | 192 def build_url(builder, build_number): |
189 return "%s/builds/%s" % (builder.url(), build_number) | 193 return "%s/builds/%s" % (builder.url(), build_number) |
190 | 194 |
(...skipping 30 matching lines...) Expand all Loading... |
221 self.buildbot_url = url if url else self._default_url | 225 self.buildbot_url = url if url else self._default_url |
222 self._builder_by_name = {} | 226 self._builder_by_name = {} |
223 | 227 |
224 def _parse_last_build_cell(self, builder, cell): | 228 def _parse_last_build_cell(self, builder, cell): |
225 status_link = cell.find('a') | 229 status_link = cell.find('a') |
226 if status_link: | 230 if status_link: |
227 # Will be either a revision number or a build number | 231 # Will be either a revision number or a build number |
228 revision_string = status_link.string | 232 revision_string = status_link.string |
229 # If revision_string has non-digits assume it's not a revision numbe
r. | 233 # If revision_string has non-digits assume it's not a revision numbe
r. |
230 builder['built_revision'] = int(revision_string) \ | 234 builder['built_revision'] = int(revision_string) \ |
231 if not re.match('\D', revision_string) \ | 235 if not re.match('\D', revision_string) \ |
232 else None | 236 else None |
233 | 237 |
234 # FIXME: We treat slave lost as green even though it is not to | 238 # FIXME: We treat slave lost as green even though it is not to |
235 # work around the Qts bot being on a broken internet connection. | 239 # work around the Qts bot being on a broken internet connection. |
236 # The real fix is https://bugs.webkit.org/show_bug.cgi?id=37099 | 240 # The real fix is https://bugs.webkit.org/show_bug.cgi?id=37099 |
237 builder['is_green'] = not re.search('fail', cell.renderContents()) o
r \ | 241 builder['is_green'] = not re.search('fail', cell.renderContents()) o
r \ |
238 not not re.search('lost', cell.renderContents(
)) | 242 not not re.search('lost', cell.renderContents()) |
239 | 243 |
240 status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<buil
d_number>\d+)" | 244 status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<buil
d_number>\d+)" |
241 link_match = re.match(status_link_regexp, status_link['href']) | 245 link_match = re.match(status_link_regexp, status_link['href']) |
242 builder['build_number'] = int(link_match.group("build_number")) | 246 builder['build_number'] = int(link_match.group("build_number")) |
243 else: | 247 else: |
244 # We failed to find a link in the first cell, just give up. This | 248 # We failed to find a link in the first cell, just give up. This |
245 # can happen if a builder is just-added, the first cell will just | 249 # can happen if a builder is just-added, the first cell will just |
246 # be "no build" | 250 # be "no build" |
247 # Other parts of the code depend on is_green being present. | 251 # Other parts of the code depend on is_green being present. |
248 builder['is_green'] = False | 252 builder['is_green'] = False |
249 builder['built_revision'] = None | 253 builder['built_revision'] = None |
250 builder['build_number'] = None | 254 builder['build_number'] = None |
251 | 255 |
252 def _parse_current_build_cell(self, builder, cell): | 256 def _parse_current_build_cell(self, builder, cell): |
253 activity_lines = cell.renderContents().split("<br />") | 257 activity_lines = cell.renderContents().split("<br />") |
254 builder["activity"] = activity_lines[0] # normally "building" or "idle" | 258 builder["activity"] = activity_lines[0] # normally "building" or "idle" |
255 # The middle lines document how long left for any current builds. | 259 # The middle lines document how long left for any current builds. |
256 match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1]) | 260 match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1]) |
257 builder["pending_builds"] = int(match.group("pending_builds")) if match
else 0 | 261 builder["pending_builds"] = int(match.group("pending_builds")) if match
else 0 |
258 | 262 |
259 def _parse_builder_status_from_row(self, status_row): | 263 def _parse_builder_status_from_row(self, status_row): |
260 status_cells = status_row.findAll('td') | 264 status_cells = status_row.findAll('td') |
261 builder = {} | 265 builder = {} |
262 | 266 |
263 # First cell is the name | 267 # First cell is the name |
264 name_link = status_cells[0].find('a') | 268 name_link = status_cells[0].find('a') |
(...skipping 13 matching lines...) Expand all Loading... |
278 def _fetch_build_dictionary(self, builder, build_number): | 282 def _fetch_build_dictionary(self, builder, build_number): |
279 # Note: filter=1 will remove None and {} and '', which cuts noise but ca
n | 283 # Note: filter=1 will remove None and {} and '', which cuts noise but ca
n |
280 # cause keys to be missing which you might otherwise expect. | 284 # cause keys to be missing which you might otherwise expect. |
281 # FIXME: The bot sends a *huge* amount of data for each request, we shou
ld | 285 # FIXME: The bot sends a *huge* amount of data for each request, we shou
ld |
282 # find a way to reduce the response size further. | 286 # find a way to reduce the response size further. |
283 json_url = "%s/json/builders/%s/builds/%s?filter=1" % (self.buildbot_url
, urllib.quote(builder.name()), build_number) | 287 json_url = "%s/json/builders/%s/builds/%s?filter=1" % (self.buildbot_url
, urllib.quote(builder.name()), build_number) |
284 try: | 288 try: |
285 return json.load(urllib2.urlopen(json_url)) | 289 return json.load(urllib2.urlopen(json_url)) |
286 except urllib2.URLError, err: | 290 except urllib2.URLError, err: |
287 build_url = Build.build_url(builder, build_number) | 291 build_url = Build.build_url(builder, build_number) |
288 _log.error("Error fetching data for %s build %s (%s, json: %s): %s"
% (builder.name(), build_number, build_url, json_url, err)) | 292 _log.error("Error fetching data for %s build %s (%s, json: %s): %s"
% |
| 293 (builder.name(), build_number, build_url, json_url, err)) |
289 return None | 294 return None |
290 except ValueError, err: | 295 except ValueError, err: |
291 build_url = Build.build_url(builder, build_number) | 296 build_url = Build.build_url(builder, build_number) |
292 _log.error("Error decoding json data from %s: %s" % (build_url, err)
) | 297 _log.error("Error decoding json data from %s: %s" % (build_url, err)
) |
293 return None | 298 return None |
294 | 299 |
295 def _fetch_one_box_per_builder(self): | 300 def _fetch_one_box_per_builder(self): |
296 build_status_url = "%s/one_box_per_builder" % self.buildbot_url | 301 build_status_url = "%s/one_box_per_builder" % self.buildbot_url |
297 return urllib2.urlopen(build_status_url) | 302 return urllib2.urlopen(build_status_url) |
298 | 303 |
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
391 | 396 |
392 builders_succeeded_in_past = set() | 397 builders_succeeded_in_past = set() |
393 for past_revision in revisions_in_order[i:]: | 398 for past_revision in revisions_in_order[i:]: |
394 if not revision_statuses[past_revision]: | 399 if not revision_statuses[past_revision]: |
395 break | 400 break |
396 builders_succeeded_in_past = builders_succeeded_in_past.union(re
vision_statuses[past_revision]) | 401 builders_succeeded_in_past = builders_succeeded_in_past.union(re
vision_statuses[past_revision]) |
397 | 402 |
398 if len(builders_succeeded_in_future) == len(builder_revisions) and l
en(builders_succeeded_in_past) == len(builder_revisions): | 403 if len(builders_succeeded_in_future) == len(builder_revisions) and l
en(builders_succeeded_in_past) == len(builder_revisions): |
399 return revision | 404 return revision |
400 return None | 405 return None |
OLD | NEW |