Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(8)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py

Issue 1839193004: Run auto-formatter (autopep8) on webkitpy. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Rebased Created 4 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2009, Google Inc. All rights reserved. 1 # Copyright (c) 2009, Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 26 matching lines...) Expand all
37 from webkitpy.common.net.layouttestresults import LayoutTestResults 37 from webkitpy.common.net.layouttestresults import LayoutTestResults
38 from webkitpy.common.net.networktransaction import NetworkTransaction 38 from webkitpy.common.net.networktransaction import NetworkTransaction
39 from webkitpy.common.system.logutils import get_logger 39 from webkitpy.common.system.logutils import get_logger
40 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup 40 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
41 41
42 42
43 _log = get_logger(__file__) 43 _log = get_logger(__file__)
44 44
45 45
46 class Builder(object): 46 class Builder(object):
47
47 def __init__(self, name, buildbot): 48 def __init__(self, name, buildbot):
48 self._name = name 49 self._name = name
49 self._buildbot = buildbot 50 self._buildbot = buildbot
50 self._builds_cache = {} 51 self._builds_cache = {}
51 self._revision_to_build_number = None 52 self._revision_to_build_number = None
52 53
53 def name(self): 54 def name(self):
54 return self._name 55 return self._name
55 56
56 def results_url(self): 57 def results_url(self):
57 return config_urls.chromium_results_url_base_for_builder(self._name) 58 return config_urls.chromium_results_url_base_for_builder(self._name)
58 59
59 def accumulated_results_url(self): 60 def accumulated_results_url(self):
60 return config_urls.chromium_accumulated_results_url_base_for_builder(sel f._name) 61 return config_urls.chromium_accumulated_results_url_base_for_builder(sel f._name)
61 62
62 def latest_layout_test_results_url(self): 63 def latest_layout_test_results_url(self):
63 return self.accumulated_results_url() or self.latest_cached_build().resu lts_url(); 64 return self.accumulated_results_url() or self.latest_cached_build().resu lts_url()
64 65
65 @memoized 66 @memoized
66 def latest_layout_test_results(self): 67 def latest_layout_test_results(self):
67 return self.fetch_layout_test_results(self.latest_layout_test_results_ur l()) 68 return self.fetch_layout_test_results(self.latest_layout_test_results_ur l())
68 69
69 def _fetch_file_from_results(self, results_url, file_name): 70 def _fetch_file_from_results(self, results_url, file_name):
70 # It seems this can return None if the url redirects and then returns 40 4. 71 # It seems this can return None if the url redirects and then returns 40 4.
71 result = urllib2.urlopen("%s/%s" % (results_url, file_name)) 72 result = urllib2.urlopen("%s/%s" % (results_url, file_name))
72 if not result: 73 if not result:
73 return None 74 return None
74 # urlopen returns a file-like object which sometimes works fine with str () 75 # urlopen returns a file-like object which sometimes works fine with str ()
75 # but sometimes is a addinfourl object. In either case calling read() i s correct. 76 # but sometimes is a addinfourl object. In either case calling read() i s correct.
76 return result.read() 77 return result.read()
77 78
78 def fetch_layout_test_results(self, results_url): 79 def fetch_layout_test_results(self, results_url):
79 # FIXME: This should cache that the result was a 404 and stop hitting th e network. 80 # FIXME: This should cache that the result was a 404 and stop hitting th e network.
80 results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "failing_results.json")) 81 results_file = NetworkTransaction(convert_404_to_None=True).run(
82 lambda: self._fetch_file_from_results(results_url, "failing_results. json"))
81 return LayoutTestResults.results_from_string(results_file) 83 return LayoutTestResults.results_from_string(results_file)
82 84
83 def url_encoded_name(self): 85 def url_encoded_name(self):
84 return urllib.quote(self._name) 86 return urllib.quote(self._name)
85 87
86 def url(self): 88 def url(self):
87 return "%s/builders/%s" % (self._buildbot.buildbot_url, self.url_encoded _name()) 89 return "%s/builders/%s" % (self._buildbot.buildbot_url, self.url_encoded _name())
88 90
89 # This provides a single place to mock 91 # This provides a single place to mock
90 def _fetch_build(self, build_number): 92 def _fetch_build(self, build_number):
91 build_dictionary = self._buildbot._fetch_build_dictionary(self, build_nu mber) 93 build_dictionary = self._buildbot._fetch_build_dictionary(self, build_nu mber)
92 if not build_dictionary: 94 if not build_dictionary:
93 return None 95 return None
94 revision_string = build_dictionary['sourceStamp']['revision'] 96 revision_string = build_dictionary['sourceStamp']['revision']
95 return Build(self, 97 return Build(self,
96 build_number=int(build_dictionary['number']), 98 build_number=int(build_dictionary['number']),
97 # 'revision' may be None if a trunk build was started by the force-b uild button on the web page. 99 # 'revision' may be None if a trunk build was started by th e force-build button on the web page.
98 revision=(int(revision_string) if revision_string else None), 100 revision=(int(revision_string) if revision_string else None ),
99 # Buildbot uses any nubmer other than 0 to mean fail. Since we fetc h with 101 # Buildbot uses any nubmer other than 0 to mean fail. Sinc e we fetch with
100 # filter=1, passing builds may contain no 'results' value. 102 # filter=1, passing builds may contain no 'results' value.
101 is_green=(not build_dictionary.get('results')), 103 is_green=(not build_dictionary.get('results')),
102 ) 104 )
103 105
104 def build(self, build_number): 106 def build(self, build_number):
105 if not build_number: 107 if not build_number:
106 return None 108 return None
107 cached_build = self._builds_cache.get(build_number) 109 cached_build = self._builds_cache.get(build_number)
108 if cached_build: 110 if cached_build:
109 return cached_build 111 return cached_build
110 112
111 build = self._fetch_build(build_number) 113 build = self._fetch_build(build_number)
112 self._builds_cache[build_number] = build 114 self._builds_cache[build_number] = build
113 return build 115 return build
114 116
115 def latest_cached_build(self): 117 def latest_cached_build(self):
116 revision_build_pairs = self.revision_build_pairs_with_results() 118 revision_build_pairs = self.revision_build_pairs_with_results()
117 revision_build_pairs.sort(key=lambda i: i[1]) 119 revision_build_pairs.sort(key=lambda i: i[1])
118 latest_build_number = revision_build_pairs[-1][1] 120 latest_build_number = revision_build_pairs[-1][1]
119 return self.build(latest_build_number) 121 return self.build(latest_build_number)
120 122
121 file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\) ") 123 file_name_regexp = re.compile(r"r(?P<revision>\d+) \((?P<build_number>\d+)\) ")
124
122 def _revision_and_build_for_filename(self, filename): 125 def _revision_and_build_for_filename(self, filename):
123 # Example: "r47483 (1)/" or "r47483 (1).zip" 126 # Example: "r47483 (1)/" or "r47483 (1).zip"
124 match = self.file_name_regexp.match(filename) 127 match = self.file_name_regexp.match(filename)
125 if not match: 128 if not match:
126 return None 129 return None
127 return (int(match.group("revision")), int(match.group("build_number"))) 130 return (int(match.group("revision")), int(match.group("build_number")))
128 131
129 def _fetch_revision_to_build_map(self): 132 def _fetch_revision_to_build_map(self):
130 # All _fetch requests go through _buildbot for easier mocking 133 # All _fetch requests go through _buildbot for easier mocking
131 # FIXME: This should use NetworkTransaction's 404 handling instead. 134 # FIXME: This should use NetworkTransaction's 404 handling instead.
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
163 # This assumes there can be only one build per revision, which is false, but we don't care for now. 166 # This assumes there can be only one build per revision, which is false, but we don't care for now.
164 def build_for_revision(self, revision, allow_failed_lookups=False): 167 def build_for_revision(self, revision, allow_failed_lookups=False):
165 # NOTE: This lookup will fail if that exact revision was never built. 168 # NOTE: This lookup will fail if that exact revision was never built.
166 build_number = self._revision_to_build_map().get(int(revision)) 169 build_number = self._revision_to_build_map().get(int(revision))
167 if not build_number: 170 if not build_number:
168 return None 171 return None
169 build = self.build(build_number) 172 build = self.build(build_number)
170 if not build and allow_failed_lookups: 173 if not build and allow_failed_lookups:
171 # Builds for old revisions with fail to lookup via buildbot's json a pi. 174 # Builds for old revisions with fail to lookup via buildbot's json a pi.
172 build = Build(self, 175 build = Build(self,
173 build_number=build_number, 176 build_number=build_number,
174 revision=revision, 177 revision=revision,
175 is_green=False, 178 is_green=False,
176 ) 179 )
177 return build 180 return build
178 181
179 182
180 class Build(object): 183 class Build(object):
184
181 def __init__(self, builder, build_number, revision, is_green): 185 def __init__(self, builder, build_number, revision, is_green):
182 self._builder = builder 186 self._builder = builder
183 self._number = build_number 187 self._number = build_number
184 self._revision = revision 188 self._revision = revision
185 self._is_green = is_green 189 self._is_green = is_green
186 190
187 @staticmethod 191 @staticmethod
188 def build_url(builder, build_number): 192 def build_url(builder, build_number):
189 return "%s/builds/%s" % (builder.url(), build_number) 193 return "%s/builds/%s" % (builder.url(), build_number)
190 194
(...skipping 30 matching lines...) Expand all
221 self.buildbot_url = url if url else self._default_url 225 self.buildbot_url = url if url else self._default_url
222 self._builder_by_name = {} 226 self._builder_by_name = {}
223 227
224 def _parse_last_build_cell(self, builder, cell): 228 def _parse_last_build_cell(self, builder, cell):
225 status_link = cell.find('a') 229 status_link = cell.find('a')
226 if status_link: 230 if status_link:
227 # Will be either a revision number or a build number 231 # Will be either a revision number or a build number
228 revision_string = status_link.string 232 revision_string = status_link.string
229 # If revision_string has non-digits assume it's not a revision numbe r. 233 # If revision_string has non-digits assume it's not a revision numbe r.
230 builder['built_revision'] = int(revision_string) \ 234 builder['built_revision'] = int(revision_string) \
231 if not re.match('\D', revision_string) \ 235 if not re.match('\D', revision_string) \
232 else None 236 else None
233 237
234 # FIXME: We treat slave lost as green even though it is not to 238 # FIXME: We treat slave lost as green even though it is not to
235 # work around the Qts bot being on a broken internet connection. 239 # work around the Qts bot being on a broken internet connection.
236 # The real fix is https://bugs.webkit.org/show_bug.cgi?id=37099 240 # The real fix is https://bugs.webkit.org/show_bug.cgi?id=37099
237 builder['is_green'] = not re.search('fail', cell.renderContents()) o r \ 241 builder['is_green'] = not re.search('fail', cell.renderContents()) o r \
238 not not re.search('lost', cell.renderContents( )) 242 not not re.search('lost', cell.renderContents())
239 243
240 status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<buil d_number>\d+)" 244 status_link_regexp = r"builders/(?P<builder_name>.*)/builds/(?P<buil d_number>\d+)"
241 link_match = re.match(status_link_regexp, status_link['href']) 245 link_match = re.match(status_link_regexp, status_link['href'])
242 builder['build_number'] = int(link_match.group("build_number")) 246 builder['build_number'] = int(link_match.group("build_number"))
243 else: 247 else:
244 # We failed to find a link in the first cell, just give up. This 248 # We failed to find a link in the first cell, just give up. This
245 # can happen if a builder is just-added, the first cell will just 249 # can happen if a builder is just-added, the first cell will just
246 # be "no build" 250 # be "no build"
247 # Other parts of the code depend on is_green being present. 251 # Other parts of the code depend on is_green being present.
248 builder['is_green'] = False 252 builder['is_green'] = False
249 builder['built_revision'] = None 253 builder['built_revision'] = None
250 builder['build_number'] = None 254 builder['build_number'] = None
251 255
252 def _parse_current_build_cell(self, builder, cell): 256 def _parse_current_build_cell(self, builder, cell):
253 activity_lines = cell.renderContents().split("<br />") 257 activity_lines = cell.renderContents().split("<br />")
254 builder["activity"] = activity_lines[0] # normally "building" or "idle" 258 builder["activity"] = activity_lines[0] # normally "building" or "idle"
255 # The middle lines document how long left for any current builds. 259 # The middle lines document how long left for any current builds.
256 match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1]) 260 match = re.match("(?P<pending_builds>\d) pending", activity_lines[-1])
257 builder["pending_builds"] = int(match.group("pending_builds")) if match else 0 261 builder["pending_builds"] = int(match.group("pending_builds")) if match else 0
258 262
259 def _parse_builder_status_from_row(self, status_row): 263 def _parse_builder_status_from_row(self, status_row):
260 status_cells = status_row.findAll('td') 264 status_cells = status_row.findAll('td')
261 builder = {} 265 builder = {}
262 266
263 # First cell is the name 267 # First cell is the name
264 name_link = status_cells[0].find('a') 268 name_link = status_cells[0].find('a')
(...skipping 13 matching lines...) Expand all
278 def _fetch_build_dictionary(self, builder, build_number): 282 def _fetch_build_dictionary(self, builder, build_number):
279 # Note: filter=1 will remove None and {} and '', which cuts noise but ca n 283 # Note: filter=1 will remove None and {} and '', which cuts noise but ca n
280 # cause keys to be missing which you might otherwise expect. 284 # cause keys to be missing which you might otherwise expect.
281 # FIXME: The bot sends a *huge* amount of data for each request, we shou ld 285 # FIXME: The bot sends a *huge* amount of data for each request, we shou ld
282 # find a way to reduce the response size further. 286 # find a way to reduce the response size further.
283 json_url = "%s/json/builders/%s/builds/%s?filter=1" % (self.buildbot_url , urllib.quote(builder.name()), build_number) 287 json_url = "%s/json/builders/%s/builds/%s?filter=1" % (self.buildbot_url , urllib.quote(builder.name()), build_number)
284 try: 288 try:
285 return json.load(urllib2.urlopen(json_url)) 289 return json.load(urllib2.urlopen(json_url))
286 except urllib2.URLError, err: 290 except urllib2.URLError, err:
287 build_url = Build.build_url(builder, build_number) 291 build_url = Build.build_url(builder, build_number)
288 _log.error("Error fetching data for %s build %s (%s, json: %s): %s" % (builder.name(), build_number, build_url, json_url, err)) 292 _log.error("Error fetching data for %s build %s (%s, json: %s): %s" %
293 (builder.name(), build_number, build_url, json_url, err))
289 return None 294 return None
290 except ValueError, err: 295 except ValueError, err:
291 build_url = Build.build_url(builder, build_number) 296 build_url = Build.build_url(builder, build_number)
292 _log.error("Error decoding json data from %s: %s" % (build_url, err) ) 297 _log.error("Error decoding json data from %s: %s" % (build_url, err) )
293 return None 298 return None
294 299
295 def _fetch_one_box_per_builder(self): 300 def _fetch_one_box_per_builder(self):
296 build_status_url = "%s/one_box_per_builder" % self.buildbot_url 301 build_status_url = "%s/one_box_per_builder" % self.buildbot_url
297 return urllib2.urlopen(build_status_url) 302 return urllib2.urlopen(build_status_url)
298 303
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after
391 396
392 builders_succeeded_in_past = set() 397 builders_succeeded_in_past = set()
393 for past_revision in revisions_in_order[i:]: 398 for past_revision in revisions_in_order[i:]:
394 if not revision_statuses[past_revision]: 399 if not revision_statuses[past_revision]:
395 break 400 break
396 builders_succeeded_in_past = builders_succeeded_in_past.union(re vision_statuses[past_revision]) 401 builders_succeeded_in_past = builders_succeeded_in_past.union(re vision_statuses[past_revision])
397 402
398 if len(builders_succeeded_in_future) == len(builder_revisions) and l en(builders_succeeded_in_past) == len(builder_revisions): 403 if len(builders_succeeded_in_future) == len(builder_revisions) and l en(builders_succeeded_in_past) == len(builder_revisions):
399 return revision 404 return revision
400 return None 405 return None
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698