Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(770)

Side by Side Diff: master/webstatus/console.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: Address comment Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « master/webstatus/buildstatus.py ('k') | master/webstatus/waterfall.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4
5 """ Skia's override of buildbot.status.web.console """
6
7
8 from buildbot import util
9 from buildbot.changes import changes as changes_module
10 from buildbot.status import builder as builder_status
11 from buildbot.status.web.base import HtmlResource
12 from buildbot.status.web.console import ANYBRANCH, \
13 CacheStatus, \
14 DevBuild, \
15 DevRevision, \
16 DoesNotPassFilter, \
17 getInProgressResults, \
18 getResultsClass, \
19 TimeRevisionComparator, \
20 IntegerRevisionComparator
21 from buildbot.status.web.status_json import JsonResource
22 from skia_master_scripts import utils
23 from twisted.internet import defer
24
25 import builder_name_schema
26 import re
27 import skia_vars
28 import time
29 import urllib
30
31
32 class ConsoleJsonStatusResource(JsonResource):
33 """JSON interface for the console page."""
34
35 def __init__(self, status, order_by_time=False):
36 JsonResource.__init__(self, status)
37
38 self.cache = CacheStatus()
39
40 if order_by_time:
41 self.comparator = TimeRevisionComparator()
42 else:
43 self.comparator = IntegerRevisionComparator()
44
45 def asDict(self, request):
46 cxt = {}
47 status = request.site.buildbot_service.getStatus()
48
49 # get url parameters
50 # Categories to show information for.
51 categories = request.args.get("category", [])
52 # List of all builders to show on the page.
53 builders = request.args.get("builder", [])
54 # Repo used to filter the changes shown.
55 repository = request.args.get("repository", [None])[0]
56 # Branch used to filter the changes shown.
57 branch = request.args.get("branch", [ANYBRANCH])[0]
58 # List of all the committers name to display on the page.
59 dev_name = request.args.get("name", [])
60
61 # Debug information to display at the end of the page.
62 debug_info = cxt['debuginfo'] = dict()
63 debug_info["load_time"] = time.time()
64
65 # Keep only the revisions we care about.
66 # By default we process the last 40 revisions.
67 # If a dev name is passed, we look for the changes by this person in the
68 # last 160 revisions.
69 num_revs = int(request.args.get("revs", [40])[0])
70 if dev_name:
71 num_revs *= 4
72 num_builds = num_revs
73
74 # Get all changes we can find. This is a DB operation, so it must use
75 # a deferred.
76 d = self.getAllChanges(request, status, debug_info)
77 def got_changes(all_changes):
78 debug_info["source_all"] = len(all_changes)
79
80 rev_filter = {}
81 if branch != ANYBRANCH:
82 rev_filter['branch'] = branch
83 if dev_name:
84 rev_filter['who'] = dev_name
85 rev_filter['repository'] = skia_vars.GetGlobalVariable('skia_git_url')
86 revisions = list(self.filterRevisions(all_changes, max_revs=num_revs,
87 rev_filter=rev_filter))
88 debug_info["revision_final"] = len(revisions)
89
90 # Fetch all the builds for all builders until we get the next build
91 # after last_revision.
92 builder_list = None
93 all_builds = None
94 if revisions:
95 last_revision = revisions[len(revisions) - 1].revision
96 debug_info["last_revision"] = last_revision
97
98 (builder_list, all_builds) = self.getAllBuildsForRevision(status,
99 request,
100 last_revision,
101 num_builds,
102 categories,
103 builders,
104 debug_info)
105
106 debug_info["added_blocks"] = 0
107 debug_info["from_cache"] = 0
108
109 if request.args.get("display_cache", None):
110 data = ""
111 data += "\nGlobal Cache\n"
112 data += self.cache.display()
113 return data
114
115 cxt.update(self.displayPage(request, status, builder_list,
116 all_builds, revisions, categories,
117 repository, branch, debug_info))
118 # Clean up the cache.
119 if debug_info["added_blocks"]:
120 self.cache.trim()
121 return {'builders': cxt['builders'],
122 'revisions': cxt['revisions']}
123 d.addCallback(got_changes)
124 return d
125
126 ##
127 ## Data gathering functions
128 ##
129
130 def getHeadBuild(self, builder):
131 """Get the most recent build for the given builder.
132 """
133 build = builder.getBuild(-1)
134
135 # HACK: Work around #601, the head build may be None if it is
136 # locked.
137 if build is None:
138 build = builder.getBuild(-2)
139
140 return build
141
142 def fetchChangesFromHistory(self, status, max_depth, max_builds, debug_info):
143 """Look at the history of the builders and try to fetch as many changes
144 as possible. We need this when the main source does not contain enough
145 sourcestamps.
146
147 max_depth defines how many builds we will parse for a given builder.
148 max_builds defines how many builds total we want to parse. This is to
149 limit the amount of time we spend in this function.
150
151 This function is sub-optimal, but the information returned by this
152 function is cached, so this function won't be called more than once.
153 """
154
155 all_changes = list()
156 build_count = 0
157 for builder_name in status.getBuilderNames()[:]:
158 if build_count > max_builds:
159 break
160
161 builder = status.getBuilder(builder_name)
162 build = self.getHeadBuild(builder)
163 depth = 0
164 while build and depth < max_depth and build_count < max_builds:
165 depth += 1
166 build_count += 1
167 sourcestamp = build.getSourceStamp()
168 all_changes.extend(sourcestamp.changes[:])
169 build = build.getPreviousBuild()
170
171 debug_info["source_fetch_len"] = len(all_changes)
172 return all_changes
173
174 @defer.deferredGenerator
175 def getAllChanges(self, request, status, debug_info):
176 master = request.site.buildbot_service.master
177 max_rev_limit = skia_vars.GetGlobalVariable('console_max_rev_limit')
178 default_rev_limit = skia_vars.GetGlobalVariable('console_default_rev_limit')
179 limit = min(max_rev_limit,
180 max(1, int(request.args.get('limit', [default_rev_limit])[0])))
181 wfd = defer.waitForDeferred(master.db.changes.getRecentChanges(limit))
182 yield wfd
183 chdicts = wfd.getResult()
184
185 # convert those to Change instances
186 wfd = defer.waitForDeferred(
187 defer.gatherResults([
188 changes_module.Change.fromChdict(master, chdict)
189 for chdict in chdicts ]))
190 yield wfd
191 all_changes = wfd.getResult()
192
193 all_changes.sort(key=self.comparator.getSortingKey())
194
195 # Remove the dups
196 prev_change = None
197 new_changes = []
198 for change in all_changes:
199 rev = change.revision
200 if not prev_change or rev != prev_change.revision:
201 new_changes.append(change)
202 prev_change = change
203 all_changes = new_changes
204
205 debug_info["source_len"] = len(all_changes)
206 yield all_changes
207
208 def getBuildDetails(self, request, builder_name, build):
209 """Returns an HTML list of failures for a given build."""
210 details = {}
211 if not build.getLogs():
212 return details
213
214 for step in build.getSteps():
215 (result, reason) = step.getResults()
216 if result == builder_status.FAILURE:
217 name = step.getName()
218
219 # Remove html tags from the error text.
220 strip_html = re.compile(r'<.*?>')
221 stripped_details = strip_html.sub('', ' '.join(step.getText()))
222
223 details['buildername'] = builder_name
224 details['status'] = stripped_details
225 details['reason'] = reason
226 logs = details['logs'] = []
227
228 if step.getLogs():
229 for log in step.getLogs():
230 logname = log.getName()
231 logurl = request.childLink(
232 "../builders/%s/builds/%s/steps/%s/logs/%s" %
233 (urllib.quote(builder_name),
234 build.getNumber(),
235 urllib.quote(name),
236 urllib.quote(logname)))
237 logs.append(dict(url=logurl, name=logname))
238 return details
239
240 def getBuildsForRevision(self, request, builder, builder_name, last_revision,
241 num_builds, debug_info):
242 """Return the list of all the builds for a given builder that we will
243 need to be able to display the console page. We start by the most recent
244 build, and we go down until we find a build that was built prior to the
245 last change we are interested in."""
246
247 revision = last_revision
248
249 builds = []
250 build = self.getHeadBuild(builder)
251 number = 0
252 while build and number < num_builds:
253 debug_info["builds_scanned"] += 1
254 number += 1
255
256 # Get the last revision in this build.
257 # We first try "got_revision", but if it does not work, then
258 # we try "revision".
259 got_rev = -1
260 try:
261 got_rev = build.getProperty("got_revision")
262 if not self.comparator.isValidRevision(got_rev):
263 got_rev = -1
264 except KeyError:
265 pass
266
267 try:
268 if got_rev == -1:
269 got_rev = build.getProperty("revision")
270 if not self.comparator.isValidRevision(got_rev):
271 got_rev = -1
272 except Exception:
273 pass
274
275 # We ignore all builds that don't have last revisions.
276 # TODO(nsylvain): If the build is over, maybe it was a problem
277 # with the update source step. We need to find a way to tell the
278 # user that his change might have broken the source update.
279 if got_rev and got_rev != -1:
280 dev_revision = self.getChangeForBuild(build, got_rev)
281 details = self.getBuildDetails(request, builder_name, build)
282 dev_build = DevBuild(dev_revision, build, details,
283 getInProgressResults(build))
284 builds.append(dev_build)
285
286 # Now break if we have enough builds.
287 current_revision = self.getChangeForBuild(build, revision)
288 if self.comparator.isRevisionEarlier(dev_build, current_revision):
289 break
290
291 build = build.getPreviousBuild()
292
293 return builds
294
295 def getChangeForBuild(self, build, revision):
296 if not build or not build.getChanges(): # Forced build
297 return DevBuild(revision, build, None)
298
299 for change in build.getChanges():
300 if change.revision == revision:
301 return change
302
303 # No matching change, return the last change in build.
304 changes = list(build.getChanges())
305 changes.sort(key=self.comparator.getSortingKey())
306 return changes[-1]
307
308 def getAllBuildsForRevision(self, status, request, last_revision, num_builds,
309 categories, builders, debug_info):
310 """Returns a dictionary of builds we need to inspect to be able to
311 display the console page. The key is the builder name, and the value is
312 an array of build we care about. We also returns a dictionary of
313 builders we care about. The key is it's category.
314
315 last_revision is the last revision we want to display in the page.
316 categories is a list of categories to display. It is coming from the
317 HTTP GET parameters.
318 builders is a list of builders to display. It is coming from the HTTP
319 GET parameters.
320 """
321
322 all_builds = dict()
323
324 # List of all builders in the dictionary.
325 builder_list = dict()
326
327 debug_info["builds_scanned"] = 0
328 # Get all the builders.
329 builder_names = status.getBuilderNames()[:]
330 for builder_name in builder_names:
331 builder = status.getBuilder(builder_name)
332
333 # Make sure we are interested in this builder.
334 if categories and builder.category not in categories:
335 continue
336 if builders and builder_name not in builders:
337 continue
338 if builder_name_schema.IsTrybot(builder_name):
339 continue
340
341 # We want to display this builder.
342 category_full = builder.category or 'default'
343
344 category_parts = category_full.split('|')
345 category = category_parts[0]
346 if len(category_parts) > 1:
347 subcategory = category_parts[1]
348 else:
349 subcategory = 'default'
350 if not builder_list.get(category):
351 builder_list[category] = {}
352 if not builder_list[category].get(subcategory):
353 builder_list[category][subcategory] = {}
354 if not builder_list[category][subcategory].get(category_full):
355 builder_list[category][subcategory][category_full] = []
356
357 b = {}
358 b["color"] = "notstarted"
359 b["pageTitle"] = builder_name
360 b["url"] = "./builders/%s" % urllib.quote(builder_name, safe='() ')
361 b["builderName"] = builder_name
362 state, _ = status.getBuilder(builder_name).getState()
363 # Check if it's offline, if so, the box is purple.
364 if state == "offline":
365 b["color"] = "offline"
366 else:
367 # If not offline, then display the result of the last
368 # finished build.
369 build = self.getHeadBuild(status.getBuilder(builder_name))
370 while build and not build.isFinished():
371 build = build.getPreviousBuild()
372
373 if build:
374 b["color"] = getResultsClass(build.getResults(), None, False)
375
376 # Append this builder to the dictionary of builders.
377 builder_list[category][subcategory][category_full].append(b)
378 # Set the list of builds for this builder.
379 all_builds[builder_name] = self.getBuildsForRevision(request,
380 builder,
381 builder_name,
382 last_revision,
383 num_builds,
384 debug_info)
385
386 return (builder_list, all_builds)
387
388
389 ##
390 ## Display functions
391 ##
392
393 def displayStatusLine(self, builder_list, all_builds, revision, debug_info):
394 """Display the boxes that represent the status of each builder in the
395 first build "revision" was in. Returns an HTML list of errors that
396 happened during these builds."""
397
398 details = []
399 builds = {}
400
401 # Display the boxes by category group.
402 for category in builder_list:
403 for subcategory in builder_list[category]:
404 for category_full in builder_list[category][subcategory]:
405 for builder in builder_list[category][subcategory][category_full]:
406 builder_name = builder['builderName']
407 builds[builder_name] = []
408 introduced_in = None
409 first_not_in = None
410
411 cached_value = self.cache.get(builder_name, revision.revision)
412 if cached_value:
413 debug_info["from_cache"] += 1
414
415 b = {}
416 b["url"] = cached_value.url
417 b["pageTitle"] = cached_value.pageTitle
418 b["color"] = cached_value.color
419 b["tag"] = cached_value.tag
420 b["builderName"] = cached_value.builderName
421
422 builds[builder_name].append(b)
423
424 if cached_value.details and cached_value.color == "failure":
425 details.append(cached_value.details)
426
427 continue
428
429 # Find the first build that does not include the revision.
430 for build in all_builds[builder_name]:
431 if self.comparator.isRevisionEarlier(build.revision, revision):
432 first_not_in = build
433 break
434 else:
435 introduced_in = build
436
437 # Get the results of the first build with the revision, and the
438 # first build that does not include the revision.
439 results = None
440 in_progress_results = None
441 previous_results = None
442 if introduced_in:
443 results = introduced_in.results
444 in_progress_results = introduced_in.inProgressResults
445 if first_not_in:
446 previous_results = first_not_in.results
447
448 is_running = False
449 if introduced_in and not introduced_in.isFinished:
450 is_running = True
451
452 url = "./waterfall"
453 page_title = builder_name
454 tag = ""
455 current_details = {}
456 if introduced_in:
457 current_details = introduced_in.details or ""
458 url = "./buildstatus?builder=%s&number=%s" % (
459 urllib.quote(builder_name), introduced_in.number)
460 page_title += " "
461 page_title += urllib.quote(' '.join(introduced_in.text),
462 ' \n\\/:')
463
464 builder_strip = builder_name.replace(' ', '')
465 builder_strip = builder_strip.replace('(', '')
466 builder_strip = builder_strip.replace(')', '')
467 builder_strip = builder_strip.replace('.', '')
468 tag = "Tag%s%s" % (builder_strip, introduced_in.number)
469
470 if is_running:
471 page_title += ' ETA: %ds' % (introduced_in.eta or 0)
472
473 results_class = getResultsClass(results, previous_results,
474 is_running, in_progress_results)
475
476 b = {}
477 b["url"] = url
478 b["pageTitle"] = page_title
479 b["color"] = results_class
480 b["tag"] = tag
481 b["builderName"] = builder_name
482
483 builds[builder_name].append(b)
484
485 # If the box is red, we add the explaination in the details
486 # section.
487 if current_details and results_class == "failure":
488 details.append(current_details)
489
490 # Add this box to the cache if it's completed so we don't have
491 # to compute it again.
492 if results_class not in ("running", "running_failure",
493 "notstarted"):
494 debug_info["added_blocks"] += 1
495 self.cache.insert(builder_name, revision.revision, results_class,
496 page_title, current_details, url, tag)
497
498 return (builds, details)
499
500 def filterRevisions(self, revisions, rev_filter=None, max_revs=None):
501 """Filter a set of revisions based on any number of filter criteria.
502 If specified, rev_filter should be a dict with keys corresponding to
503 revision attributes, and values of 1+ strings"""
504 if not rev_filter:
505 if max_revs is None:
506 for rev in reversed(revisions):
507 yield DevRevision(rev)
508 else:
509 for index, rev in enumerate(reversed(revisions)):
510 if index >= max_revs:
511 break
512 yield DevRevision(rev)
513 else:
514 num_revs = 0
515 for rev in reversed(revisions):
516 if max_revs and num_revs >= max_revs:
517 break
518 try:
519 for field, acceptable in rev_filter.iteritems():
520 if not hasattr(rev, field):
521 raise DoesNotPassFilter
522 if type(acceptable) in (str, unicode):
523 if getattr(rev, field) != acceptable:
524 raise DoesNotPassFilter
525 elif type(acceptable) in (list, tuple, set):
526 if getattr(rev, field) not in acceptable:
527 raise DoesNotPassFilter
528 num_revs += 1
529 yield DevRevision(rev)
530 except DoesNotPassFilter:
531 pass
532
533 def displayPage(self, request, status, builder_list, all_builds, revisions,
534 categories, repository, branch, debug_info):
535 """Display the console page."""
536 # Build the main template directory with all the informations we have.
537 subs = dict()
538 subs["branch"] = branch or 'trunk'
539 subs["repository"] = repository
540 if categories:
541 subs["categories"] = ' '.join(categories)
542 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S",
543 time.localtime(util.now()))
544 subs["debugInfo"] = debug_info
545 subs["ANYBRANCH"] = ANYBRANCH
546
547 if builder_list:
548 builders = builder_list
549 else:
550 builders = {}
551 subs['builders'] = builders
552 subs['revisions'] = []
553
554 # For each revision we show one line
555 for revision in revisions:
556 r = {}
557
558 # Fill the dictionary with this new information
559 r['id'] = revision.revision
560 r['link'] = revision.revlink
561 if (skia_vars.GetGlobalVariable('commit_bot_username') in revision.who
562 and 'Author: ' in revision.comments):
563 who = revision.comments.split('Author: ')[1].split('\n')[0]
564 who += ' (commit-bot)'
565 else:
566 who = revision.who
567 r['who'] = utils.FixGitSvnEmail(who)
568 r['date'] = revision.date
569 r['comments'] = revision.comments
570 r['repository'] = revision.repository
571 r['project'] = revision.project
572
573 # Display the status for all builders.
574 (builds, details) = self.displayStatusLine(builder_list,
575 all_builds,
576 revision,
577 debug_info)
578 r['builds'] = builds
579 r['details'] = details
580
581 # Calculate the td span for the comment and the details.
582 r["span"] = sum ([len(builder_list[category]) \
583 for category in builder_list]) + 2
584
585 subs['revisions'].append(r)
586
587 #
588 # Display the footer of the page.
589 #
590 debug_info["load_time"] = time.time() - debug_info["load_time"]
591 return subs
592
593
594 class ConsoleStatusResource(HtmlResource):
595 """Main console class. It displays a user-oriented status page.
596 Every change is a line in the page, and it shows the result of the first
597 build with this change for each slave."""
598
599 def getPageTitle(self, request):
600 status = self.getStatus(request)
601 title = status.getTitle()
602 if title:
603 return "BuildBot: %s" % title
604 else:
605 return "BuildBot"
606
607 def getChangeManager(self, request):
608 return request.site.buildbot_service.parent.change_svc
609
610 def content(self, request, cxt):
611 "This method builds the main console view display."
612
613 reload_time = None
614 # Check if there was an arg. Don't let people reload faster than
615 # every 15 seconds. 0 means no reload.
616 if "reload" in request.args:
617 try:
618 reload_time = int(request.args["reload"][0])
619 if reload_time != 0:
620 reload_time = max(reload_time, 15)
621 except ValueError:
622 pass
623
624 request.setHeader('Cache-Control', 'no-cache')
625
626 # Sets the default reload time to 60 seconds.
627 if not reload_time:
628 reload_time = skia_vars.GetGlobalVariable('default_webstatus_refresh')
629
630 # Append the tag to refresh the page.
631 if reload_time is not None and reload_time != 0:
632 cxt['refresh'] = reload_time
633
634 # List of categories for which we load information but hide initially.
635 hidden_categories_sets = request.args.get("hideCategories", [])
636 hide_categories = []
637 for category_set in hidden_categories_sets:
638 hide_categories.extend(category_set.split(','))
639 cxt['hide_categories'] = hide_categories
640
641 # List of subcategories for which we load information but hide initially.
642 hidden_subcategories_sets = request.args.get("hideSubcategories", [])
643 hide_subcategories = []
644 for subcategory_set in hidden_subcategories_sets:
645 hide_subcategories.extend(subcategory_set.split(','))
646 cxt['hide_subcategories'] = hide_subcategories
647
648 # Console event-loading limits.
649 cxt['default_console_limit'] = \
650 skia_vars.GetGlobalVariable('console_default_rev_limit')
651 cxt['max_console_limit'] = \
652 skia_vars.GetGlobalVariable('console_max_rev_limit')
653
654 templates = request.site.buildbot_service.templates
655 template = templates.get_template("console.html")
656 data = template.render(cxt)
657
658 return data
OLDNEW
« no previous file with comments | « master/webstatus/buildstatus.py ('k') | master/webstatus/waterfall.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698