| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 """ Skia's override of buildbot.status.web.waterfall """ | |
| 6 | |
| 7 | |
| 8 from buildbot import interfaces, util | |
| 9 from buildbot.status import builder as builder_status_module | |
| 10 from buildbot.status import buildstep | |
| 11 from buildbot.changes import changes as changes_module | |
| 12 | |
| 13 from buildbot.status.web.base import Box, HtmlResource, IBox, ICurrentBox, \ | |
| 14 ITopBox, path_to_root, \ | |
| 15 map_branches | |
| 16 from buildbot.status.web.waterfall import earlier, \ | |
| 17 later, \ | |
| 18 insertGaps, \ | |
| 19 WaterfallHelp, \ | |
| 20 ChangeEventSource | |
| 21 from twisted.python import log | |
| 22 from twisted.internet import defer | |
| 23 | |
| 24 import builder_name_schema | |
| 25 import locale | |
| 26 import time | |
| 27 import urllib | |
| 28 | |
| 29 | |
| 30 class WaterfallStatusResource(HtmlResource): | |
| 31 """This builds the main status page, with the waterfall display, and | |
| 32 all child pages.""" | |
| 33 | |
| 34 def __init__(self, categories=None, num_events=200, num_events_max=None, | |
| 35 title='Waterfall', only_show_failures=False, | |
| 36 builder_filter=lambda x: not builder_name_schema.IsTrybot(x)): | |
| 37 HtmlResource.__init__(self) | |
| 38 self.categories = categories | |
| 39 self.num_events = num_events | |
| 40 self.num_events_max = num_events_max | |
| 41 self.putChild("help", WaterfallHelp(categories)) | |
| 42 self.BuilderFilter = builder_filter | |
| 43 self.title = title | |
| 44 self.only_show_failures = only_show_failures | |
| 45 | |
| 46 def getPageTitle(self, request): | |
| 47 status = self.getStatus(request) | |
| 48 p = status.getTitle() | |
| 49 if p: | |
| 50 return "BuildBot: %s" % p | |
| 51 else: | |
| 52 return "BuildBot" | |
| 53 | |
| 54 def getChangeManager(self, request): | |
| 55 return request.site.buildbot_service.getChangeSvc() | |
| 56 | |
| 57 def get_reload_time(self, request): | |
| 58 if "reload" in request.args: | |
| 59 try: | |
| 60 reload_time = int(request.args["reload"][0]) | |
| 61 return max(reload_time, 15) | |
| 62 except ValueError: | |
| 63 pass | |
| 64 return None | |
| 65 | |
| 66 def isSuccess(self, builderStatus): | |
| 67 # Helper function to return True if the builder is not failing. | |
| 68 # The function will return false if the current state is "offline", | |
| 69 # the last build was not successful, or if a step from the current | |
| 70 # build(s) failed. | |
| 71 | |
| 72 # Make sure the builder is online. | |
| 73 if builderStatus.getState()[0] == 'offline': | |
| 74 return False | |
| 75 | |
| 76 # Look at the last finished build to see if it was success or not. | |
| 77 last_build = builderStatus.getLastFinishedBuild() | |
| 78 if last_build and last_build.getResults() != builder_status_module.SUCCESS: | |
| 79 return False | |
| 80 | |
| 81 # Check all the current builds to see if one step is already | |
| 82 # failing. | |
| 83 current_builds = builderStatus.getCurrentBuilds() | |
| 84 if current_builds: | |
| 85 for build in current_builds: | |
| 86 for step in build.getSteps(): | |
| 87 if step.getResults()[0] == builder_status_module.FAILURE: | |
| 88 return False | |
| 89 | |
| 90 # The last finished build was successful, and all the current builds | |
| 91 # don't have any failed steps. | |
| 92 return True | |
| 93 | |
| 94 def content(self, request, ctx): | |
| 95 status = self.getStatus(request) | |
| 96 master = request.site.buildbot_service.master | |
| 97 | |
| 98 # before calling content_with_db_data, make a bunch of database | |
| 99 # queries. This is a sick hack, but beats rewriting the entire | |
| 100 # waterfall around asynchronous calls | |
| 101 | |
| 102 results = {} | |
| 103 | |
| 104 # recent changes | |
| 105 changes_d = master.db.changes.getRecentChanges(40) | |
| 106 def to_changes(chdicts): | |
| 107 return defer.gatherResults([ | |
| 108 changes_module.Change.fromChdict(master, chdict) | |
| 109 for chdict in chdicts ]) | |
| 110 changes_d.addCallback(to_changes) | |
| 111 def keep_changes(changes): | |
| 112 results['changes'] = changes | |
| 113 changes_d.addCallback(keep_changes) | |
| 114 | |
| 115 # build request counts for each builder | |
| 116 all_builder_names = status.getBuilderNames(categories=self.categories) | |
| 117 brstatus_ds = [] | |
| 118 brcounts = {} | |
| 119 def keep_count(statuses, builder_name): | |
| 120 brcounts[builder_name] = len(statuses) | |
| 121 for builder_name in all_builder_names: | |
| 122 builder_status = status.getBuilder(builder_name) | |
| 123 d = builder_status.getPendingBuildRequestStatuses() | |
| 124 d.addCallback(keep_count, builder_name) | |
| 125 brstatus_ds.append(d) | |
| 126 | |
| 127 # wait for it all to finish | |
| 128 d = defer.gatherResults([ changes_d ] + brstatus_ds) | |
| 129 def call_content(_): | |
| 130 return self.content_with_db_data(results['changes'], | |
| 131 brcounts, request, ctx) | |
| 132 d.addCallback(call_content) | |
| 133 return d | |
| 134 | |
| 135 def content_with_db_data(self, changes, brcounts, request, ctx): | |
| 136 ctx['title'] = self.title | |
| 137 status = self.getStatus(request) | |
| 138 ctx['refresh'] = self.get_reload_time(request) | |
| 139 | |
| 140 # we start with all Builders available to this Waterfall: this is | |
| 141 # limited by the config-file -time categories= argument, and defaults | |
| 142 # to all defined Builders. | |
| 143 all_builder_names = status.getBuilderNames(categories=self.categories) | |
| 144 builders = [status.getBuilder(name) for name in all_builder_names] | |
| 145 | |
| 146 # Apply a filter to the builders. | |
| 147 builders = [b for b in builders if self.BuilderFilter(b.name)] | |
| 148 | |
| 149 # but if the URL has one or more builder= arguments (or the old show= | |
| 150 # argument, which is still accepted for backwards compatibility), we | |
| 151 # use that set of builders instead. We still don't show anything | |
| 152 # outside the config-file time set limited by categories=. | |
| 153 show_builders = request.args.get("show", []) | |
| 154 show_builders.extend(request.args.get("builder", [])) | |
| 155 if show_builders: | |
| 156 builders = [b for b in builders if b.name in show_builders] | |
| 157 | |
| 158 # now, if the URL has one or category= arguments, use them as a | |
| 159 # filter: only show those builders which belong to one of the given | |
| 160 # categories. | |
| 161 show_categories = request.args.get("category", []) | |
| 162 if show_categories: | |
| 163 builders = [b for b in builders if b.category in show_categories] | |
| 164 | |
| 165 # If we are only showing failures, we remove all the builders that are not | |
| 166 # currently red or won't be turning red at the end of their current run. | |
| 167 if self.only_show_failures: | |
| 168 builders = [b for b in builders if not self.isSuccess(b)] | |
| 169 | |
| 170 (change_names, builder_names, timestamps, event_grid, source_events) = \ | |
| 171 self.buildGrid(request, builders, changes) | |
| 172 | |
| 173 # start the table: top-header material | |
| 174 locale_enc = locale.getdefaultlocale()[1] | |
| 175 if locale_enc is not None: | |
| 176 locale_tz = unicode(time.tzname[time.localtime()[-1]], locale_enc) | |
| 177 else: | |
| 178 locale_tz = unicode(time.tzname[time.localtime()[-1]]) | |
| 179 ctx['tz'] = locale_tz | |
| 180 ctx['changes_url'] = request.childLink("../changes") | |
| 181 | |
| 182 bn = ctx['builders'] = [] | |
| 183 | |
| 184 for name in builder_names: | |
| 185 builder = status.getBuilder(name) | |
| 186 top_box = ITopBox(builder).getBox(request) | |
| 187 current_box = ICurrentBox(builder).getBox(status, brcounts) | |
| 188 bn.append({'name': name, | |
| 189 'url': request.childLink("../builders/%s" % | |
| 190 urllib.quote(name, safe='')), | |
| 191 'top': top_box.text, | |
| 192 'top_class': top_box.class_, | |
| 193 'status': current_box.text, | |
| 194 'status_class': current_box.class_, | |
| 195 }) | |
| 196 | |
| 197 ctx.update(self.phase2(request, change_names + builder_names, timestamps, | |
| 198 event_grid, source_events)) | |
| 199 | |
| 200 def with_args(req, remove_args=None, new_args=None, new_path=None): | |
| 201 if not remove_args: | |
| 202 remove_args = [] | |
| 203 if not new_args: | |
| 204 new_args = [] | |
| 205 newargs = req.args.copy() | |
| 206 for argname in remove_args: | |
| 207 newargs[argname] = [] | |
| 208 if "branch" in newargs: | |
| 209 newargs["branch"] = [b for b in newargs["branch"] if b] | |
| 210 for k, v in new_args: | |
| 211 if k in newargs: | |
| 212 newargs[k].append(v) | |
| 213 else: | |
| 214 newargs[k] = [v] | |
| 215 newquery = "&".join(["%s=%s" % (urllib.quote(k), urllib.quote(v)) | |
| 216 for k in newargs | |
| 217 for v in newargs[k] | |
| 218 ]) | |
| 219 if new_path: | |
| 220 new_url = new_path | |
| 221 elif req.prepath: | |
| 222 new_url = req.prepath[-1] | |
| 223 else: | |
| 224 new_url = '' | |
| 225 if newquery: | |
| 226 new_url += "?" + newquery | |
| 227 return new_url | |
| 228 | |
| 229 if timestamps: | |
| 230 bottom = timestamps[-1] | |
| 231 ctx['nextpage'] = with_args(request, ["last_time"], | |
| 232 [("last_time", str(int(bottom)))]) | |
| 233 | |
| 234 | |
| 235 helpurl = path_to_root(request) + "waterfall/help" | |
| 236 ctx['help_url'] = with_args(request, new_path=helpurl) | |
| 237 | |
| 238 if self.get_reload_time(request) is not None: | |
| 239 ctx['no_reload_page'] = with_args(request, remove_args=["reload"]) | |
| 240 | |
| 241 template = request.site.buildbot_service.templates.get_template( | |
| 242 "waterfall.html") | |
| 243 data = template.render(**ctx) | |
| 244 return data | |
| 245 | |
| 246 def buildGrid(self, request, builders, changes): | |
| 247 debug = False | |
| 248 show_events = False | |
| 249 if request.args.get("show_events", ["false"])[0].lower() == "true": | |
| 250 show_events = True | |
| 251 filter_categories = request.args.get('category', []) | |
| 252 filter_branches = [b for b in request.args.get("branch", []) if b] | |
| 253 filter_branches = map_branches(filter_branches) | |
| 254 filter_committers = [c for c in request.args.get("committer", []) if c] | |
| 255 max_time = int(request.args.get("last_time", [util.now()])[0]) | |
| 256 if "show_time" in request.args: | |
| 257 min_time = max_time - int(request.args["show_time"][0]) | |
| 258 elif "first_time" in request.args: | |
| 259 min_time = int(request.args["first_time"][0]) | |
| 260 elif filter_branches or filter_committers: | |
| 261 min_time = util.now() - 24 * 60 * 60 | |
| 262 else: | |
| 263 min_time = 0 | |
| 264 span_length = 10 # ten-second chunks | |
| 265 req_events = int(request.args.get("num_events", [self.num_events])[0]) | |
| 266 if self.num_events_max and req_events > self.num_events_max: | |
| 267 max_page_len = self.num_events_max | |
| 268 else: | |
| 269 max_page_len = req_events | |
| 270 | |
| 271 # first step is to walk backwards in time, asking each column | |
| 272 # (commit, all builders) if they have any events there. Build up the | |
| 273 # array of events, and stop when we have a reasonable number. | |
| 274 | |
| 275 commit_source = ChangeEventSource(changes) | |
| 276 | |
| 277 last_event_time = util.now() | |
| 278 sources = [commit_source] + builders | |
| 279 change_names = ["changes"] | |
| 280 builder_names = map(lambda builder: builder.getName(), builders) | |
| 281 source_names = change_names + builder_names | |
| 282 source_events = [] | |
| 283 source_generators = [] | |
| 284 | |
| 285 def get_event_from(g): | |
| 286 try: | |
| 287 while True: | |
| 288 e = g.next() | |
| 289 # e might be buildstep.BuildStepStatus, | |
| 290 # builder.BuildStatus, builder.Event, | |
| 291 # waterfall.Spacer(builder.Event), or changes.Change . | |
| 292 # The show_events=False flag means we should hide | |
| 293 # builder.Event . | |
| 294 if not show_events and isinstance(e, builder_status_module.Event): | |
| 295 continue | |
| 296 | |
| 297 if isinstance(e, buildstep.BuildStepStatus): | |
| 298 # unfinished steps are always shown | |
| 299 if e.isFinished() and e.isHidden(): | |
| 300 continue | |
| 301 | |
| 302 break | |
| 303 event = interfaces.IStatusEvent(e) | |
| 304 if debug: | |
| 305 log.msg("gen %s gave1 %s" % (g, event.getText())) | |
| 306 except StopIteration: | |
| 307 event = None | |
| 308 return event | |
| 309 | |
| 310 for s in sources: | |
| 311 gen = insertGaps(s.eventGenerator(filter_branches, | |
| 312 filter_categories, | |
| 313 filter_committers, | |
| 314 min_time), | |
| 315 show_events, | |
| 316 last_event_time) | |
| 317 source_generators.append(gen) | |
| 318 # get the first event | |
| 319 source_events.append(get_event_from(gen)) | |
| 320 event_grid = [] | |
| 321 timestamps = [] | |
| 322 | |
| 323 last_event_time = 0 | |
| 324 for e in source_events: | |
| 325 if e and e.getTimes()[0] > last_event_time: | |
| 326 last_event_time = e.getTimes()[0] | |
| 327 if last_event_time == 0: | |
| 328 last_event_time = util.now() | |
| 329 | |
| 330 span_start = last_event_time - span_length | |
| 331 debug_gather = 0 | |
| 332 | |
| 333 while 1: | |
| 334 if debug_gather: | |
| 335 log.msg("checking (%s,]" % span_start) | |
| 336 # the tableau of potential events is in source_events[]. The | |
| 337 # window crawls backwards, and we examine one source at a time. | |
| 338 # If the source's top-most event is in the window, is it pushed | |
| 339 # onto the events[] array and the tableau is refilled. This | |
| 340 # continues until the tableau event is not in the window (or is | |
| 341 # missing). | |
| 342 | |
| 343 span_events = [] # for all sources, in this span. row of event_grid | |
| 344 first_timestamp = None # timestamp of first event in the span | |
| 345 last_timestamp = None # last pre-span event, for next span | |
| 346 | |
| 347 for c in range(len(source_generators)): | |
| 348 events = [] # for this source, in this span. cell of event_grid | |
| 349 event = source_events[c] | |
| 350 while event and span_start < event.getTimes()[0]: | |
| 351 # to look at windows that don't end with the present, | |
| 352 # condition the .append on event.time <= spanFinish | |
| 353 if not IBox(event, None): | |
| 354 log.msg("BAD EVENT", event, event.getText()) | |
| 355 assert 0 | |
| 356 if debug: | |
| 357 log.msg("pushing", event.getText(), event) | |
| 358 events.append(event) | |
| 359 starts, _ = event.getTimes() | |
| 360 first_timestamp = earlier(first_timestamp, starts) | |
| 361 event = get_event_from(source_generators[c]) | |
| 362 if debug: | |
| 363 log.msg("finished span") | |
| 364 | |
| 365 if event: | |
| 366 # this is the last pre-span event for this source | |
| 367 last_timestamp = later(last_timestamp, event.getTimes()[0]) | |
| 368 if debug_gather: | |
| 369 log.msg(" got %s from %s" % (events, source_names[c])) | |
| 370 source_events[c] = event # refill the tableau | |
| 371 span_events.append(events) | |
| 372 | |
| 373 # only show events older than max_time. This makes it possible to | |
| 374 # visit a page that shows what it would be like to scroll off the | |
| 375 # bottom of this one. | |
| 376 if first_timestamp is not None and first_timestamp <= max_time: | |
| 377 event_grid.append(span_events) | |
| 378 timestamps.append(first_timestamp) | |
| 379 | |
| 380 if last_timestamp: | |
| 381 span_start = last_timestamp - span_length | |
| 382 else: | |
| 383 # no more events | |
| 384 break | |
| 385 if min_time is not None and last_timestamp < min_time: | |
| 386 break | |
| 387 | |
| 388 if len(timestamps) > max_page_len: | |
| 389 break | |
| 390 | |
| 391 # now loop | |
| 392 | |
| 393 # loop is finished. now we have event_grid[] and timestamps[] | |
| 394 if debug_gather: | |
| 395 log.msg("finished loop") | |
| 396 assert(len(timestamps) == len(event_grid)) | |
| 397 return (change_names, builder_names, timestamps, event_grid, source_events) | |
| 398 | |
| 399 def phase2(self, request, source_names, timestamps, event_grid, | |
| 400 source_events): | |
| 401 | |
| 402 if not timestamps: | |
| 403 return dict(grid=[], gridlen=0) | |
| 404 | |
| 405 # first pass: figure out the height of the chunks, populate grid | |
| 406 grid = [] | |
| 407 for i in range(1+len(source_names)): | |
| 408 grid.append([]) | |
| 409 # grid is a list of columns, one for the timestamps, and one per | |
| 410 # event source. Each column is exactly the same height. Each element | |
| 411 # of the list is a single <td> box. | |
| 412 last_date = time.strftime("%d %b %Y", time.localtime(util.now())) | |
| 413 for r in range(0, len(timestamps)): | |
| 414 chunkstrip = event_grid[r] | |
| 415 # chunkstrip is a horizontal strip of event blocks. Each block | |
| 416 # is a vertical list of events, all for the same source. | |
| 417 assert(len(chunkstrip) == len(source_names)) | |
| 418 max_rows = reduce(max, map(len, chunkstrip)) | |
| 419 for i in range(max_rows): | |
| 420 if i != max_rows - 1: | |
| 421 grid[0].append(None) | |
| 422 else: | |
| 423 # timestamp goes at the bottom of the chunk | |
| 424 stuff = [] | |
| 425 # add the date at the beginning (if it is not the same as | |
| 426 # today's date), and each time it changes | |
| 427 todayday = time.strftime("%a", time.localtime(timestamps[r])) | |
| 428 today = time.strftime("%d %b %Y", time.localtime(timestamps[r])) | |
| 429 if today != last_date: | |
| 430 stuff.append(todayday) | |
| 431 stuff.append(today) | |
| 432 last_date = today | |
| 433 stuff.append(time.strftime("%H:%M:%S", time.localtime(timestamps[r]))) | |
| 434 grid[0].append(Box(text=stuff, class_="Time", valign="bottom", | |
| 435 align="center")) | |
| 436 | |
| 437 # at this point the timestamp column has been populated with | |
| 438 # max_rows boxes, most None but the last one has the time string | |
| 439 for c in range(0, len(chunkstrip)): | |
| 440 block = chunkstrip[c] | |
| 441 assert(block != None) # should be [] instead | |
| 442 for i in range(max_rows - len(block)): | |
| 443 # fill top of chunk with blank space | |
| 444 grid[c+1].append(None) | |
| 445 for i in range(len(block)): | |
| 446 # so the events are bottom-justified | |
| 447 b = IBox(block[i]).getBox(request) | |
| 448 b.parms['valign'] = "top" | |
| 449 b.parms['align'] = "center" | |
| 450 grid[c+1].append(b) | |
| 451 # now all the other columns have max_rows new boxes too | |
| 452 # populate the last row, if empty | |
| 453 gridlen = len(grid[0]) | |
| 454 for i in range(len(grid)): | |
| 455 strip = grid[i] | |
| 456 assert(len(strip) == gridlen) | |
| 457 if strip[-1] == None: | |
| 458 if source_events[i - 1]: | |
| 459 filler = IBox(source_events[i - 1]).getBox(request) | |
| 460 else: | |
| 461 # this can happen if you delete part of the build history | |
| 462 filler = Box(text=["?"], align="center") | |
| 463 strip[-1] = filler | |
| 464 strip[-1].parms['rowspan'] = 1 | |
| 465 # second pass: bubble the events upwards to un-occupied locations | |
| 466 # Every square of the grid that has a None in it needs to have | |
| 467 # something else take its place. | |
| 468 no_bubble = request.args.get("nobubble", ['0']) | |
| 469 no_bubble = int(no_bubble[0]) | |
| 470 if not no_bubble: | |
| 471 for col in range(len(grid)): | |
| 472 strip = grid[col] | |
| 473 if col == 1: # changes are handled differently | |
| 474 for i in range(2, len(strip) + 1): | |
| 475 # only merge empty boxes. Don't bubble commit boxes. | |
| 476 if strip[-i] == None: | |
| 477 next_box = strip[-i + 1] | |
| 478 assert(next_box) | |
| 479 if next_box: | |
| 480 #if not next_box.event: | |
| 481 if next_box.spacer: | |
| 482 # bubble the empty box up | |
| 483 strip[-i] = next_box | |
| 484 strip[-i].parms['rowspan'] += 1 | |
| 485 strip[-i + 1] = None | |
| 486 else: | |
| 487 # we are above a commit box. Leave it | |
| 488 # be, and turn the current box into an | |
| 489 # empty one | |
| 490 strip[-i] = Box([], rowspan=1, | |
| 491 comment="commit bubble") | |
| 492 strip[-i].spacer = True | |
| 493 else: | |
| 494 # we are above another empty box, which | |
| 495 # somehow wasn't already converted. | |
| 496 # Shouldn't happen | |
| 497 pass | |
| 498 else: | |
| 499 for i in range(2, len(strip) + 1): | |
| 500 # strip[-i] will go from next-to-last back to first | |
| 501 if strip[-i] == None: | |
| 502 # bubble previous item up | |
| 503 assert(strip[-i + 1] != None) | |
| 504 strip[-i] = strip[-i + 1] | |
| 505 strip[-i].parms['rowspan'] += 1 | |
| 506 strip[-i + 1] = None | |
| 507 else: | |
| 508 strip[-i].parms['rowspan'] = 1 | |
| 509 | |
| 510 # convert to dicts | |
| 511 for i in range(gridlen): | |
| 512 for strip in grid: | |
| 513 if strip[i]: | |
| 514 strip[i] = strip[i].td() | |
| 515 | |
| 516 return dict(grid=grid, gridlen=gridlen, no_bubble=no_bubble, time=last_date) | |
| 517 | |
| 518 | |
| 519 class TrybotStatusResource(WaterfallStatusResource): | |
| 520 def __init__(self, **kwargs): | |
| 521 WaterfallStatusResource.__init__(self, title='Trybot Waterfall', | |
| 522 builder_filter=builder_name_schema.IsTrybot, **kwargs) | |
| 523 | |
| 524 class FailureWaterfallStatusResource(WaterfallStatusResource): | |
| 525 def __init__(self, **kwargs): | |
| 526 WaterfallStatusResource.__init__(self, title='Currently Failing', | |
| 527 only_show_failures=True, **kwargs) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 648353002:
Remove Skia's forked buildbot code (Closed)
Base URL: https://skia.googlesource.com/buildbot.git@master