OLD | NEW |
| (Empty) |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """ Skia's override of buildbot.status.web.console """ | |
6 | |
7 | |
8 from buildbot import util | |
9 from buildbot.changes import changes as changes_module | |
10 from buildbot.status import builder as builder_status | |
11 from buildbot.status.web.base import HtmlResource | |
12 from buildbot.status.web.console import ANYBRANCH, \ | |
13 CacheStatus, \ | |
14 DevBuild, \ | |
15 DevRevision, \ | |
16 DoesNotPassFilter, \ | |
17 getInProgressResults, \ | |
18 getResultsClass, \ | |
19 TimeRevisionComparator, \ | |
20 IntegerRevisionComparator | |
21 from buildbot.status.web.status_json import JsonResource | |
22 from skia_master_scripts import utils | |
23 from twisted.internet import defer | |
24 | |
25 import builder_name_schema | |
26 import re | |
27 import skia_vars | |
28 import time | |
29 import urllib | |
30 | |
31 | |
32 class ConsoleJsonStatusResource(JsonResource): | |
33 """JSON interface for the console page.""" | |
34 | |
35 def __init__(self, status, order_by_time=False): | |
36 JsonResource.__init__(self, status) | |
37 | |
38 self.cache = CacheStatus() | |
39 | |
40 if order_by_time: | |
41 self.comparator = TimeRevisionComparator() | |
42 else: | |
43 self.comparator = IntegerRevisionComparator() | |
44 | |
45 def asDict(self, request): | |
46 cxt = {} | |
47 status = request.site.buildbot_service.getStatus() | |
48 | |
49 # get url parameters | |
50 # Categories to show information for. | |
51 categories = request.args.get("category", []) | |
52 # List of all builders to show on the page. | |
53 builders = request.args.get("builder", []) | |
54 # Repo used to filter the changes shown. | |
55 repository = request.args.get("repository", [None])[0] | |
56 # Branch used to filter the changes shown. | |
57 branch = request.args.get("branch", [ANYBRANCH])[0] | |
58 # List of all the committers name to display on the page. | |
59 dev_name = request.args.get("name", []) | |
60 | |
61 # Debug information to display at the end of the page. | |
62 debug_info = cxt['debuginfo'] = dict() | |
63 debug_info["load_time"] = time.time() | |
64 | |
65 # Keep only the revisions we care about. | |
66 # By default we process the last 40 revisions. | |
67 # If a dev name is passed, we look for the changes by this person in the | |
68 # last 160 revisions. | |
69 num_revs = int(request.args.get("revs", [40])[0]) | |
70 if dev_name: | |
71 num_revs *= 4 | |
72 num_builds = num_revs | |
73 | |
74 # Get all changes we can find. This is a DB operation, so it must use | |
75 # a deferred. | |
76 d = self.getAllChanges(request, status, debug_info) | |
77 def got_changes(all_changes): | |
78 debug_info["source_all"] = len(all_changes) | |
79 | |
80 rev_filter = {} | |
81 if branch != ANYBRANCH: | |
82 rev_filter['branch'] = branch | |
83 if dev_name: | |
84 rev_filter['who'] = dev_name | |
85 rev_filter['repository'] = skia_vars.GetGlobalVariable('skia_git_url') | |
86 revisions = list(self.filterRevisions(all_changes, max_revs=num_revs, | |
87 rev_filter=rev_filter)) | |
88 debug_info["revision_final"] = len(revisions) | |
89 | |
90 # Fetch all the builds for all builders until we get the next build | |
91 # after last_revision. | |
92 builder_list = None | |
93 all_builds = None | |
94 if revisions: | |
95 last_revision = revisions[len(revisions) - 1].revision | |
96 debug_info["last_revision"] = last_revision | |
97 | |
98 (builder_list, all_builds) = self.getAllBuildsForRevision(status, | |
99 request, | |
100 last_revision, | |
101 num_builds, | |
102 categories, | |
103 builders, | |
104 debug_info) | |
105 | |
106 debug_info["added_blocks"] = 0 | |
107 debug_info["from_cache"] = 0 | |
108 | |
109 if request.args.get("display_cache", None): | |
110 data = "" | |
111 data += "\nGlobal Cache\n" | |
112 data += self.cache.display() | |
113 return data | |
114 | |
115 cxt.update(self.displayPage(request, status, builder_list, | |
116 all_builds, revisions, categories, | |
117 repository, branch, debug_info)) | |
118 # Clean up the cache. | |
119 if debug_info["added_blocks"]: | |
120 self.cache.trim() | |
121 return {'builders': cxt['builders'], | |
122 'revisions': cxt['revisions']} | |
123 d.addCallback(got_changes) | |
124 return d | |
125 | |
126 ## | |
127 ## Data gathering functions | |
128 ## | |
129 | |
130 def getHeadBuild(self, builder): | |
131 """Get the most recent build for the given builder. | |
132 """ | |
133 build = builder.getBuild(-1) | |
134 | |
135 # HACK: Work around #601, the head build may be None if it is | |
136 # locked. | |
137 if build is None: | |
138 build = builder.getBuild(-2) | |
139 | |
140 return build | |
141 | |
142 def fetchChangesFromHistory(self, status, max_depth, max_builds, debug_info): | |
143 """Look at the history of the builders and try to fetch as many changes | |
144 as possible. We need this when the main source does not contain enough | |
145 sourcestamps. | |
146 | |
147 max_depth defines how many builds we will parse for a given builder. | |
148 max_builds defines how many builds total we want to parse. This is to | |
149 limit the amount of time we spend in this function. | |
150 | |
151 This function is sub-optimal, but the information returned by this | |
152 function is cached, so this function won't be called more than once. | |
153 """ | |
154 | |
155 all_changes = list() | |
156 build_count = 0 | |
157 for builder_name in status.getBuilderNames()[:]: | |
158 if build_count > max_builds: | |
159 break | |
160 | |
161 builder = status.getBuilder(builder_name) | |
162 build = self.getHeadBuild(builder) | |
163 depth = 0 | |
164 while build and depth < max_depth and build_count < max_builds: | |
165 depth += 1 | |
166 build_count += 1 | |
167 sourcestamp = build.getSourceStamp() | |
168 all_changes.extend(sourcestamp.changes[:]) | |
169 build = build.getPreviousBuild() | |
170 | |
171 debug_info["source_fetch_len"] = len(all_changes) | |
172 return all_changes | |
173 | |
174 @defer.deferredGenerator | |
175 def getAllChanges(self, request, status, debug_info): | |
176 master = request.site.buildbot_service.master | |
177 max_rev_limit = skia_vars.GetGlobalVariable('console_max_rev_limit') | |
178 default_rev_limit = skia_vars.GetGlobalVariable('console_default_rev_limit') | |
179 limit = min(max_rev_limit, | |
180 max(1, int(request.args.get('limit', [default_rev_limit])[0]))) | |
181 wfd = defer.waitForDeferred(master.db.changes.getRecentChanges(limit)) | |
182 yield wfd | |
183 chdicts = wfd.getResult() | |
184 | |
185 # convert those to Change instances | |
186 wfd = defer.waitForDeferred( | |
187 defer.gatherResults([ | |
188 changes_module.Change.fromChdict(master, chdict) | |
189 for chdict in chdicts ])) | |
190 yield wfd | |
191 all_changes = wfd.getResult() | |
192 | |
193 all_changes.sort(key=self.comparator.getSortingKey()) | |
194 | |
195 # Remove the dups | |
196 prev_change = None | |
197 new_changes = [] | |
198 for change in all_changes: | |
199 rev = change.revision | |
200 if not prev_change or rev != prev_change.revision: | |
201 new_changes.append(change) | |
202 prev_change = change | |
203 all_changes = new_changes | |
204 | |
205 debug_info["source_len"] = len(all_changes) | |
206 yield all_changes | |
207 | |
208 def getBuildDetails(self, request, builder_name, build): | |
209 """Returns an HTML list of failures for a given build.""" | |
210 details = {} | |
211 if not build.getLogs(): | |
212 return details | |
213 | |
214 for step in build.getSteps(): | |
215 (result, reason) = step.getResults() | |
216 if result == builder_status.FAILURE: | |
217 name = step.getName() | |
218 | |
219 # Remove html tags from the error text. | |
220 strip_html = re.compile(r'<.*?>') | |
221 stripped_details = strip_html.sub('', ' '.join(step.getText())) | |
222 | |
223 details['buildername'] = builder_name | |
224 details['status'] = stripped_details | |
225 details['reason'] = reason | |
226 logs = details['logs'] = [] | |
227 | |
228 if step.getLogs(): | |
229 for log in step.getLogs(): | |
230 logname = log.getName() | |
231 logurl = request.childLink( | |
232 "../builders/%s/builds/%s/steps/%s/logs/%s" % | |
233 (urllib.quote(builder_name), | |
234 build.getNumber(), | |
235 urllib.quote(name), | |
236 urllib.quote(logname))) | |
237 logs.append(dict(url=logurl, name=logname)) | |
238 return details | |
239 | |
240 def getBuildsForRevision(self, request, builder, builder_name, last_revision, | |
241 num_builds, debug_info): | |
242 """Return the list of all the builds for a given builder that we will | |
243 need to be able to display the console page. We start by the most recent | |
244 build, and we go down until we find a build that was built prior to the | |
245 last change we are interested in.""" | |
246 | |
247 revision = last_revision | |
248 | |
249 builds = [] | |
250 build = self.getHeadBuild(builder) | |
251 number = 0 | |
252 while build and number < num_builds: | |
253 debug_info["builds_scanned"] += 1 | |
254 number += 1 | |
255 | |
256 # Get the last revision in this build. | |
257 # We first try "got_revision", but if it does not work, then | |
258 # we try "revision". | |
259 got_rev = -1 | |
260 try: | |
261 got_rev = build.getProperty("got_revision") | |
262 if not self.comparator.isValidRevision(got_rev): | |
263 got_rev = -1 | |
264 except KeyError: | |
265 pass | |
266 | |
267 try: | |
268 if got_rev == -1: | |
269 got_rev = build.getProperty("revision") | |
270 if not self.comparator.isValidRevision(got_rev): | |
271 got_rev = -1 | |
272 except Exception: | |
273 pass | |
274 | |
275 # We ignore all builds that don't have last revisions. | |
276 # TODO(nsylvain): If the build is over, maybe it was a problem | |
277 # with the update source step. We need to find a way to tell the | |
278 # user that his change might have broken the source update. | |
279 if got_rev and got_rev != -1: | |
280 dev_revision = self.getChangeForBuild(build, got_rev) | |
281 details = self.getBuildDetails(request, builder_name, build) | |
282 dev_build = DevBuild(dev_revision, build, details, | |
283 getInProgressResults(build)) | |
284 builds.append(dev_build) | |
285 | |
286 # Now break if we have enough builds. | |
287 current_revision = self.getChangeForBuild(build, revision) | |
288 if self.comparator.isRevisionEarlier(dev_build, current_revision): | |
289 break | |
290 | |
291 build = build.getPreviousBuild() | |
292 | |
293 return builds | |
294 | |
295 def getChangeForBuild(self, build, revision): | |
296 if not build or not build.getChanges(): # Forced build | |
297 return DevBuild(revision, build, None) | |
298 | |
299 for change in build.getChanges(): | |
300 if change.revision == revision: | |
301 return change | |
302 | |
303 # No matching change, return the last change in build. | |
304 changes = list(build.getChanges()) | |
305 changes.sort(key=self.comparator.getSortingKey()) | |
306 return changes[-1] | |
307 | |
308 def getAllBuildsForRevision(self, status, request, last_revision, num_builds, | |
309 categories, builders, debug_info): | |
310 """Returns a dictionary of builds we need to inspect to be able to | |
311 display the console page. The key is the builder name, and the value is | |
312 an array of build we care about. We also returns a dictionary of | |
313 builders we care about. The key is it's category. | |
314 | |
315 last_revision is the last revision we want to display in the page. | |
316 categories is a list of categories to display. It is coming from the | |
317 HTTP GET parameters. | |
318 builders is a list of builders to display. It is coming from the HTTP | |
319 GET parameters. | |
320 """ | |
321 | |
322 all_builds = dict() | |
323 | |
324 # List of all builders in the dictionary. | |
325 builder_list = dict() | |
326 | |
327 debug_info["builds_scanned"] = 0 | |
328 # Get all the builders. | |
329 builder_names = status.getBuilderNames()[:] | |
330 for builder_name in builder_names: | |
331 builder = status.getBuilder(builder_name) | |
332 | |
333 # Make sure we are interested in this builder. | |
334 if categories and builder.category not in categories: | |
335 continue | |
336 if builders and builder_name not in builders: | |
337 continue | |
338 if builder_name_schema.IsTrybot(builder_name): | |
339 continue | |
340 | |
341 # We want to display this builder. | |
342 category_full = builder.category or 'default' | |
343 | |
344 category_parts = category_full.split('|') | |
345 category = category_parts[0] | |
346 if len(category_parts) > 1: | |
347 subcategory = category_parts[1] | |
348 else: | |
349 subcategory = 'default' | |
350 if not builder_list.get(category): | |
351 builder_list[category] = {} | |
352 if not builder_list[category].get(subcategory): | |
353 builder_list[category][subcategory] = {} | |
354 if not builder_list[category][subcategory].get(category_full): | |
355 builder_list[category][subcategory][category_full] = [] | |
356 | |
357 b = {} | |
358 b["color"] = "notstarted" | |
359 b["pageTitle"] = builder_name | |
360 b["url"] = "./builders/%s" % urllib.quote(builder_name, safe='() ') | |
361 b["builderName"] = builder_name | |
362 state, _ = status.getBuilder(builder_name).getState() | |
363 # Check if it's offline, if so, the box is purple. | |
364 if state == "offline": | |
365 b["color"] = "offline" | |
366 else: | |
367 # If not offline, then display the result of the last | |
368 # finished build. | |
369 build = self.getHeadBuild(status.getBuilder(builder_name)) | |
370 while build and not build.isFinished(): | |
371 build = build.getPreviousBuild() | |
372 | |
373 if build: | |
374 b["color"] = getResultsClass(build.getResults(), None, False) | |
375 | |
376 # Append this builder to the dictionary of builders. | |
377 builder_list[category][subcategory][category_full].append(b) | |
378 # Set the list of builds for this builder. | |
379 all_builds[builder_name] = self.getBuildsForRevision(request, | |
380 builder, | |
381 builder_name, | |
382 last_revision, | |
383 num_builds, | |
384 debug_info) | |
385 | |
386 return (builder_list, all_builds) | |
387 | |
388 | |
389 ## | |
390 ## Display functions | |
391 ## | |
392 | |
393 def displayStatusLine(self, builder_list, all_builds, revision, debug_info): | |
394 """Display the boxes that represent the status of each builder in the | |
395 first build "revision" was in. Returns an HTML list of errors that | |
396 happened during these builds.""" | |
397 | |
398 details = [] | |
399 builds = {} | |
400 | |
401 # Display the boxes by category group. | |
402 for category in builder_list: | |
403 for subcategory in builder_list[category]: | |
404 for category_full in builder_list[category][subcategory]: | |
405 for builder in builder_list[category][subcategory][category_full]: | |
406 builder_name = builder['builderName'] | |
407 builds[builder_name] = [] | |
408 introduced_in = None | |
409 first_not_in = None | |
410 | |
411 cached_value = self.cache.get(builder_name, revision.revision) | |
412 if cached_value: | |
413 debug_info["from_cache"] += 1 | |
414 | |
415 b = {} | |
416 b["url"] = cached_value.url | |
417 b["pageTitle"] = cached_value.pageTitle | |
418 b["color"] = cached_value.color | |
419 b["tag"] = cached_value.tag | |
420 b["builderName"] = cached_value.builderName | |
421 | |
422 builds[builder_name].append(b) | |
423 | |
424 if cached_value.details and cached_value.color == "failure": | |
425 details.append(cached_value.details) | |
426 | |
427 continue | |
428 | |
429 # Find the first build that does not include the revision. | |
430 for build in all_builds[builder_name]: | |
431 if self.comparator.isRevisionEarlier(build.revision, revision): | |
432 first_not_in = build | |
433 break | |
434 else: | |
435 introduced_in = build | |
436 | |
437 # Get the results of the first build with the revision, and the | |
438 # first build that does not include the revision. | |
439 results = None | |
440 in_progress_results = None | |
441 previous_results = None | |
442 if introduced_in: | |
443 results = introduced_in.results | |
444 in_progress_results = introduced_in.inProgressResults | |
445 if first_not_in: | |
446 previous_results = first_not_in.results | |
447 | |
448 is_running = False | |
449 if introduced_in and not introduced_in.isFinished: | |
450 is_running = True | |
451 | |
452 url = "./waterfall" | |
453 page_title = builder_name | |
454 tag = "" | |
455 current_details = {} | |
456 if introduced_in: | |
457 current_details = introduced_in.details or "" | |
458 url = "./buildstatus?builder=%s&number=%s" % ( | |
459 urllib.quote(builder_name), introduced_in.number) | |
460 page_title += " " | |
461 page_title += urllib.quote(' '.join(introduced_in.text), | |
462 ' \n\\/:') | |
463 | |
464 builder_strip = builder_name.replace(' ', '') | |
465 builder_strip = builder_strip.replace('(', '') | |
466 builder_strip = builder_strip.replace(')', '') | |
467 builder_strip = builder_strip.replace('.', '') | |
468 tag = "Tag%s%s" % (builder_strip, introduced_in.number) | |
469 | |
470 if is_running: | |
471 page_title += ' ETA: %ds' % (introduced_in.eta or 0) | |
472 | |
473 results_class = getResultsClass(results, previous_results, | |
474 is_running, in_progress_results) | |
475 | |
476 b = {} | |
477 b["url"] = url | |
478 b["pageTitle"] = page_title | |
479 b["color"] = results_class | |
480 b["tag"] = tag | |
481 b["builderName"] = builder_name | |
482 | |
483 builds[builder_name].append(b) | |
484 | |
485 # If the box is red, we add the explaination in the details | |
486 # section. | |
487 if current_details and results_class == "failure": | |
488 details.append(current_details) | |
489 | |
490 # Add this box to the cache if it's completed so we don't have | |
491 # to compute it again. | |
492 if results_class not in ("running", "running_failure", | |
493 "notstarted"): | |
494 debug_info["added_blocks"] += 1 | |
495 self.cache.insert(builder_name, revision.revision, results_class, | |
496 page_title, current_details, url, tag) | |
497 | |
498 return (builds, details) | |
499 | |
500 def filterRevisions(self, revisions, rev_filter=None, max_revs=None): | |
501 """Filter a set of revisions based on any number of filter criteria. | |
502 If specified, rev_filter should be a dict with keys corresponding to | |
503 revision attributes, and values of 1+ strings""" | |
504 if not rev_filter: | |
505 if max_revs is None: | |
506 for rev in reversed(revisions): | |
507 yield DevRevision(rev) | |
508 else: | |
509 for index, rev in enumerate(reversed(revisions)): | |
510 if index >= max_revs: | |
511 break | |
512 yield DevRevision(rev) | |
513 else: | |
514 num_revs = 0 | |
515 for rev in reversed(revisions): | |
516 if max_revs and num_revs >= max_revs: | |
517 break | |
518 try: | |
519 for field, acceptable in rev_filter.iteritems(): | |
520 if not hasattr(rev, field): | |
521 raise DoesNotPassFilter | |
522 if type(acceptable) in (str, unicode): | |
523 if getattr(rev, field) != acceptable: | |
524 raise DoesNotPassFilter | |
525 elif type(acceptable) in (list, tuple, set): | |
526 if getattr(rev, field) not in acceptable: | |
527 raise DoesNotPassFilter | |
528 num_revs += 1 | |
529 yield DevRevision(rev) | |
530 except DoesNotPassFilter: | |
531 pass | |
532 | |
533 def displayPage(self, request, status, builder_list, all_builds, revisions, | |
534 categories, repository, branch, debug_info): | |
535 """Display the console page.""" | |
536 # Build the main template directory with all the informations we have. | |
537 subs = dict() | |
538 subs["branch"] = branch or 'trunk' | |
539 subs["repository"] = repository | |
540 if categories: | |
541 subs["categories"] = ' '.join(categories) | |
542 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", | |
543 time.localtime(util.now())) | |
544 subs["debugInfo"] = debug_info | |
545 subs["ANYBRANCH"] = ANYBRANCH | |
546 | |
547 if builder_list: | |
548 builders = builder_list | |
549 else: | |
550 builders = {} | |
551 subs['builders'] = builders | |
552 subs['revisions'] = [] | |
553 | |
554 # For each revision we show one line | |
555 for revision in revisions: | |
556 r = {} | |
557 | |
558 # Fill the dictionary with this new information | |
559 r['id'] = revision.revision | |
560 r['link'] = revision.revlink | |
561 if (skia_vars.GetGlobalVariable('commit_bot_username') in revision.who | |
562 and 'Author: ' in revision.comments): | |
563 who = revision.comments.split('Author: ')[1].split('\n')[0] | |
564 who += ' (commit-bot)' | |
565 else: | |
566 who = revision.who | |
567 r['who'] = utils.FixGitSvnEmail(who) | |
568 r['date'] = revision.date | |
569 r['comments'] = revision.comments | |
570 r['repository'] = revision.repository | |
571 r['project'] = revision.project | |
572 | |
573 # Display the status for all builders. | |
574 (builds, details) = self.displayStatusLine(builder_list, | |
575 all_builds, | |
576 revision, | |
577 debug_info) | |
578 r['builds'] = builds | |
579 r['details'] = details | |
580 | |
581 # Calculate the td span for the comment and the details. | |
582 r["span"] = sum ([len(builder_list[category]) \ | |
583 for category in builder_list]) + 2 | |
584 | |
585 subs['revisions'].append(r) | |
586 | |
587 # | |
588 # Display the footer of the page. | |
589 # | |
590 debug_info["load_time"] = time.time() - debug_info["load_time"] | |
591 return subs | |
592 | |
593 | |
594 class ConsoleStatusResource(HtmlResource): | |
595 """Main console class. It displays a user-oriented status page. | |
596 Every change is a line in the page, and it shows the result of the first | |
597 build with this change for each slave.""" | |
598 | |
599 def getPageTitle(self, request): | |
600 status = self.getStatus(request) | |
601 title = status.getTitle() | |
602 if title: | |
603 return "BuildBot: %s" % title | |
604 else: | |
605 return "BuildBot" | |
606 | |
607 def getChangeManager(self, request): | |
608 return request.site.buildbot_service.parent.change_svc | |
609 | |
610 def content(self, request, cxt): | |
611 "This method builds the main console view display." | |
612 | |
613 reload_time = None | |
614 # Check if there was an arg. Don't let people reload faster than | |
615 # every 15 seconds. 0 means no reload. | |
616 if "reload" in request.args: | |
617 try: | |
618 reload_time = int(request.args["reload"][0]) | |
619 if reload_time != 0: | |
620 reload_time = max(reload_time, 15) | |
621 except ValueError: | |
622 pass | |
623 | |
624 request.setHeader('Cache-Control', 'no-cache') | |
625 | |
626 # Sets the default reload time to 60 seconds. | |
627 if not reload_time: | |
628 reload_time = skia_vars.GetGlobalVariable('default_webstatus_refresh') | |
629 | |
630 # Append the tag to refresh the page. | |
631 if reload_time is not None and reload_time != 0: | |
632 cxt['refresh'] = reload_time | |
633 | |
634 # List of categories for which we load information but hide initially. | |
635 hidden_categories_sets = request.args.get("hideCategories", []) | |
636 hide_categories = [] | |
637 for category_set in hidden_categories_sets: | |
638 hide_categories.extend(category_set.split(',')) | |
639 cxt['hide_categories'] = hide_categories | |
640 | |
641 # List of subcategories for which we load information but hide initially. | |
642 hidden_subcategories_sets = request.args.get("hideSubcategories", []) | |
643 hide_subcategories = [] | |
644 for subcategory_set in hidden_subcategories_sets: | |
645 hide_subcategories.extend(subcategory_set.split(',')) | |
646 cxt['hide_subcategories'] = hide_subcategories | |
647 | |
648 # Console event-loading limits. | |
649 cxt['default_console_limit'] = \ | |
650 skia_vars.GetGlobalVariable('console_default_rev_limit') | |
651 cxt['max_console_limit'] = \ | |
652 skia_vars.GetGlobalVariable('console_max_rev_limit') | |
653 | |
654 templates = request.site.buildbot_service.templates | |
655 template = templates.get_template("console.html") | |
656 data = template.render(cxt) | |
657 | |
658 return data | |
OLD | NEW |