| OLD | NEW |
|---|
| (Empty) |
| 1 | |
| 2 import random, weakref | |
| 3 from zope.interface import implements | |
| 4 from twisted.python import log, components | |
| 5 from twisted.python.failure import Failure | |
| 6 from twisted.spread import pb | |
| 7 from twisted.internet import reactor, defer | |
| 8 | |
| 9 from buildbot import interfaces | |
| 10 from buildbot.status.progress import Expectations | |
| 11 from buildbot.util import now | |
| 12 from buildbot.process import base | |
| 13 from buildbot.process.properties import Properties | |
| 14 | |
| 15 (ATTACHING, # slave attached, still checking hostinfo/etc | |
| 16 IDLE, # idle, available for use | |
| 17 PINGING, # build about to start, making sure it is still alive | |
| 18 BUILDING, # build is running | |
| 19 LATENT, # latent slave is not substantiated; similar to idle | |
| 20 SUBSTANTIATING, | |
| 21 ) = range(6) | |
| 22 | |
| 23 | |
| 24 class AbstractSlaveBuilder(pb.Referenceable): | |
| 25 """I am the master-side representative for one of the | |
| 26 L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote | |
| 27 buildbot. When a remote builder connects, I query it for command versions | |
| 28 and then make it available to any Builds that are ready to run. """ | |
| 29 | |
| 30 def __init__(self): | |
| 31 self.ping_watchers = [] | |
| 32 self.state = None # set in subclass | |
| 33 self.remote = None | |
| 34 self.slave = None | |
| 35 self.builder_name = None | |
| 36 | |
| 37 def __repr__(self): | |
| 38 r = ["<", self.__class__.__name__] | |
| 39 if self.builder_name: | |
| 40 r.extend([" builder=", self.builder_name]) | |
| 41 if self.slave: | |
| 42 r.extend([" slave=", self.slave.slavename]) | |
| 43 r.append(">") | |
| 44 return ''.join(r) | |
| 45 | |
| 46 def setBuilder(self, b): | |
| 47 self.builder = b | |
| 48 self.builder_name = b.name | |
| 49 | |
| 50 def getSlaveCommandVersion(self, command, oldversion=None): | |
| 51 if self.remoteCommands is None: | |
| 52 # the slave is 0.5.0 or earlier | |
| 53 return oldversion | |
| 54 return self.remoteCommands.get(command) | |
| 55 | |
| 56 def isAvailable(self): | |
| 57 # if this SlaveBuilder is busy, then it's definitely not available | |
| 58 if self.isBusy(): | |
| 59 return False | |
| 60 | |
| 61 # otherwise, check in with the BuildSlave | |
| 62 if self.slave: | |
| 63 return self.slave.canStartBuild() | |
| 64 | |
| 65 # no slave? not very available. | |
| 66 return False | |
| 67 | |
| 68 def isBusy(self): | |
| 69 return self.state not in (IDLE, LATENT) | |
| 70 | |
| 71 def buildStarted(self): | |
| 72 self.state = BUILDING | |
| 73 | |
| 74 def buildFinished(self): | |
| 75 self.state = IDLE | |
| 76 reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds) | |
| 77 | |
| 78 def attached(self, slave, remote, commands): | |
| 79 """ | |
| 80 @type slave: L{buildbot.buildslave.BuildSlave} | |
| 81 @param slave: the BuildSlave that represents the buildslave as a | |
| 82 whole | |
| 83 @type remote: L{twisted.spread.pb.RemoteReference} | |
| 84 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} | |
| 85 @type commands: dict: string -> string, or None | |
| 86 @param commands: provides the slave's version of each RemoteCommand | |
| 87 """ | |
| 88 self.state = ATTACHING | |
| 89 self.remote = remote | |
| 90 self.remoteCommands = commands # maps command name to version | |
| 91 if self.slave is None: | |
| 92 self.slave = slave | |
| 93 self.slave.addSlaveBuilder(self) | |
| 94 else: | |
| 95 assert self.slave == slave | |
| 96 log.msg("Buildslave %s attached to %s" % (slave.slavename, | |
| 97 self.builder_name)) | |
| 98 d = self.remote.callRemote("setMaster", self) | |
| 99 d.addErrback(self._attachFailure, "Builder.setMaster") | |
| 100 d.addCallback(self._attached2) | |
| 101 return d | |
| 102 | |
| 103 def _attached2(self, res): | |
| 104 d = self.remote.callRemote("print", "attached") | |
| 105 d.addErrback(self._attachFailure, "Builder.print 'attached'") | |
| 106 d.addCallback(self._attached3) | |
| 107 return d | |
| 108 | |
| 109 def _attached3(self, res): | |
| 110 # now we say they're really attached | |
| 111 self.state = IDLE | |
| 112 return self | |
| 113 | |
| 114 def _attachFailure(self, why, where): | |
| 115 assert isinstance(where, str) | |
| 116 log.msg(where) | |
| 117 log.err(why) | |
| 118 return why | |
| 119 | |
| 120 def prepare(self, builder_status): | |
| 121 return defer.succeed(None) | |
| 122 | |
| 123 def ping(self, status=None): | |
| 124 """Ping the slave to make sure it is still there. Returns a Deferred | |
| 125 that fires with True if it is. | |
| 126 | |
| 127 @param status: if you point this at a BuilderStatus, a 'pinging' | |
| 128 event will be pushed. | |
| 129 """ | |
| 130 oldstate = self.state | |
| 131 self.state = PINGING | |
| 132 newping = not self.ping_watchers | |
| 133 d = defer.Deferred() | |
| 134 self.ping_watchers.append(d) | |
| 135 if newping: | |
| 136 if status: | |
| 137 event = status.addEvent(["pinging"]) | |
| 138 d2 = defer.Deferred() | |
| 139 d2.addCallback(self._pong_status, event) | |
| 140 self.ping_watchers.insert(0, d2) | |
| 141 # I think it will make the tests run smoother if the status | |
| 142 # is updated before the ping completes | |
| 143 Ping().ping(self.remote).addCallback(self._pong) | |
| 144 | |
| 145 def reset_state(res): | |
| 146 if self.state == PINGING: | |
| 147 self.state = oldstate | |
| 148 return res | |
| 149 d.addCallback(reset_state) | |
| 150 return d | |
| 151 | |
| 152 def _pong(self, res): | |
| 153 watchers, self.ping_watchers = self.ping_watchers, [] | |
| 154 for d in watchers: | |
| 155 d.callback(res) | |
| 156 | |
| 157 def _pong_status(self, res, event): | |
| 158 if res: | |
| 159 event.text = ["ping", "success"] | |
| 160 else: | |
| 161 event.text = ["ping", "failed"] | |
| 162 event.finish() | |
| 163 | |
| 164 def detached(self): | |
| 165 log.msg("Buildslave %s detached from %s" % (self.slave.slavename, | |
| 166 self.builder_name)) | |
| 167 if self.slave: | |
| 168 self.slave.removeSlaveBuilder(self) | |
| 169 self.slave = None | |
| 170 self.remote = None | |
| 171 self.remoteCommands = None | |
| 172 | |
| 173 | |
| 174 class Ping: | |
| 175 running = False | |
| 176 | |
| 177 def ping(self, remote): | |
| 178 assert not self.running | |
| 179 self.running = True | |
| 180 log.msg("sending ping") | |
| 181 self.d = defer.Deferred() | |
| 182 # TODO: add a distinct 'ping' command on the slave.. using 'print' | |
| 183 # for this purpose is kind of silly. | |
| 184 remote.callRemote("print", "ping").addCallbacks(self._pong, | |
| 185 self._ping_failed, | |
| 186 errbackArgs=(remote,)) | |
| 187 return self.d | |
| 188 | |
| 189 def _pong(self, res): | |
| 190 log.msg("ping finished: success") | |
| 191 self.d.callback(True) | |
| 192 | |
| 193 def _ping_failed(self, res, remote): | |
| 194 log.msg("ping finished: failure") | |
| 195 # the slave has some sort of internal error, disconnect them. If we | |
| 196 # don't, we'll requeue a build and ping them again right away, | |
| 197 # creating a nasty loop. | |
| 198 remote.broker.transport.loseConnection() | |
| 199 # TODO: except, if they actually did manage to get this far, they'll | |
| 200 # probably reconnect right away, and we'll do this game again. Maybe | |
| 201 # it would be better to leave them in the PINGING state. | |
| 202 self.d.callback(False) | |
| 203 | |
| 204 | |
| 205 class SlaveBuilder(AbstractSlaveBuilder): | |
| 206 | |
| 207 def __init__(self): | |
| 208 AbstractSlaveBuilder.__init__(self) | |
| 209 self.state = ATTACHING | |
| 210 | |
| 211 def detached(self): | |
| 212 AbstractSlaveBuilder.detached(self) | |
| 213 if self.slave: | |
| 214 self.slave.removeSlaveBuilder(self) | |
| 215 self.slave = None | |
| 216 self.state = ATTACHING | |
| 217 | |
| 218 def buildFinished(self): | |
| 219 # Call the slave's buildFinished if we can; the slave may be waiting | |
| 220 # to do a graceful shutdown and needs to know when it's idle. | |
| 221 # After, we check to see if we can start other builds. | |
| 222 self.state = IDLE | |
| 223 if self.slave: | |
| 224 d = self.slave.buildFinished(self) | |
| 225 d.addCallback(lambda x: reactor.callLater(0, self.builder.botmaster.
maybeStartAllBuilds)) | |
| 226 else: | |
| 227 reactor.callLater(0, self.builder.botmaster.maybeStartAllBuilds) | |
| 228 | |
| 229 | |
| 230 class LatentSlaveBuilder(AbstractSlaveBuilder): | |
| 231 def __init__(self, slave, builder): | |
| 232 AbstractSlaveBuilder.__init__(self) | |
| 233 self.slave = slave | |
| 234 self.state = LATENT | |
| 235 self.setBuilder(builder) | |
| 236 self.slave.addSlaveBuilder(self) | |
| 237 log.msg("Latent buildslave %s attached to %s" % (slave.slavename, | |
| 238 self.builder_name)) | |
| 239 | |
| 240 def prepare(self, builder_status): | |
| 241 log.msg("substantiating slave %s" % (self,)) | |
| 242 d = self.substantiate() | |
| 243 def substantiation_failed(f): | |
| 244 builder_status.addPointEvent(['removing', 'latent', | |
| 245 self.slave.slavename]) | |
| 246 self.slave.disconnect() | |
| 247 # TODO: should failover to a new Build | |
| 248 return f | |
| 249 d.addErrback(substantiation_failed) | |
| 250 return d | |
| 251 | |
| 252 def substantiate(self): | |
| 253 self.state = SUBSTANTIATING | |
| 254 d = self.slave.substantiate(self) | |
| 255 if not self.slave.substantiated: | |
| 256 event = self.builder.builder_status.addEvent( | |
| 257 ["substantiating"]) | |
| 258 def substantiated(res): | |
| 259 msg = ["substantiate", "success"] | |
| 260 if isinstance(res, basestring): | |
| 261 msg.append(res) | |
| 262 elif isinstance(res, (tuple, list)): | |
| 263 msg.extend(res) | |
| 264 event.text = msg | |
| 265 event.finish() | |
| 266 return res | |
| 267 def substantiation_failed(res): | |
| 268 event.text = ["substantiate", "failed"] | |
| 269 # TODO add log of traceback to event | |
| 270 event.finish() | |
| 271 return res | |
| 272 d.addCallbacks(substantiated, substantiation_failed) | |
| 273 return d | |
| 274 | |
| 275 def detached(self): | |
| 276 AbstractSlaveBuilder.detached(self) | |
| 277 self.state = LATENT | |
| 278 | |
| 279 def buildStarted(self): | |
| 280 AbstractSlaveBuilder.buildStarted(self) | |
| 281 self.slave.buildStarted(self) | |
| 282 | |
| 283 def buildFinished(self): | |
| 284 AbstractSlaveBuilder.buildFinished(self) | |
| 285 self.slave.buildFinished(self) | |
| 286 | |
| 287 def _attachFailure(self, why, where): | |
| 288 self.state = LATENT | |
| 289 return AbstractSlaveBuilder._attachFailure(self, why, where) | |
| 290 | |
| 291 def ping(self, status=None): | |
| 292 if not self.slave.substantiated: | |
| 293 if status: | |
| 294 status.addEvent(["ping", "latent"]).finish() | |
| 295 return defer.succeed(True) | |
| 296 return AbstractSlaveBuilder.ping(self, status) | |
| 297 | |
| 298 | |
| 299 class Builder(pb.Referenceable): | |
| 300 """I manage all Builds of a given type. | |
| 301 | |
| 302 Each Builder is created by an entry in the config file (the c['builders'] | |
| 303 list), with a number of parameters. | |
| 304 | |
| 305 One of these parameters is the L{buildbot.process.factory.BuildFactory} | |
| 306 object that is associated with this Builder. The factory is responsible | |
| 307 for creating new L{Build<buildbot.process.base.Build>} objects. Each | |
| 308 Build object defines when and how the build is performed, so a new | |
| 309 Factory or Builder should be defined to control this behavior. | |
| 310 | |
| 311 The Builder holds on to a number of L{base.BuildRequest} objects in a | |
| 312 list named C{.buildable}. Incoming BuildRequest objects will be added to | |
| 313 this list, or (if possible) merged into an existing request. When a slave | |
| 314 becomes available, I will use my C{BuildFactory} to turn the request into | |
| 315 a new C{Build} object. The C{BuildRequest} is forgotten, the C{Build} | |
| 316 goes into C{.building} while it runs. Once the build finishes, I will | |
| 317 discard it. | |
| 318 | |
| 319 I maintain a list of available SlaveBuilders, one for each connected | |
| 320 slave that the C{slavenames} parameter says we can use. Some of these | |
| 321 will be idle, some of them will be busy running builds for me. If there | |
| 322 are multiple slaves, I can run multiple builds at once. | |
| 323 | |
| 324 I also manage forced builds, progress expectation (ETA) management, and | |
| 325 some status delivery chores. | |
| 326 | |
| 327 @type buildable: list of L{buildbot.process.base.BuildRequest} | |
| 328 @ivar buildable: BuildRequests that are ready to build, but which are | |
| 329 waiting for a buildslave to be available. | |
| 330 | |
| 331 @type building: list of L{buildbot.process.base.Build} | |
| 332 @ivar building: Builds that are actively running | |
| 333 | |
| 334 @type slaves: list of L{buildbot.buildslave.BuildSlave} objects | |
| 335 @ivar slaves: the slaves currently available for building | |
| 336 """ | |
| 337 | |
| 338 expectations = None # this is created the first time we get a good build | |
| 339 CHOOSE_SLAVES_RANDOMLY = True # disabled for determinism during tests | |
| 340 | |
| 341 def __init__(self, setup, builder_status): | |
| 342 """ | |
| 343 @type setup: dict | |
| 344 @param setup: builder setup data, as stored in | |
| 345 BuildmasterConfig['builders']. Contains name, | |
| 346 slavename(s), builddir, slavebuilddir, factory, locks. | |
| 347 @type builder_status: L{buildbot.status.builder.BuilderStatus} | |
| 348 """ | |
| 349 self.name = setup['name'] | |
| 350 self.slavenames = [] | |
| 351 if setup.has_key('slavename'): | |
| 352 self.slavenames.append(setup['slavename']) | |
| 353 if setup.has_key('slavenames'): | |
| 354 self.slavenames.extend(setup['slavenames']) | |
| 355 self.builddir = setup['builddir'] | |
| 356 self.slavebuilddir = setup['slavebuilddir'] | |
| 357 self.buildFactory = setup['factory'] | |
| 358 self.nextSlave = setup.get('nextSlave') | |
| 359 if self.nextSlave is not None and not callable(self.nextSlave): | |
| 360 raise ValueError("nextSlave must be callable") | |
| 361 self.locks = setup.get("locks", []) | |
| 362 self.env = setup.get('env', {}) | |
| 363 assert isinstance(self.env, dict) | |
| 364 if setup.has_key('periodicBuildTime'): | |
| 365 raise ValueError("periodicBuildTime can no longer be defined as" | |
| 366 " part of the Builder: use scheduler.Periodic" | |
| 367 " instead") | |
| 368 self.nextBuild = setup.get('nextBuild') | |
| 369 if self.nextBuild is not None and not callable(self.nextBuild): | |
| 370 raise ValueError("nextBuild must be callable") | |
| 371 | |
| 372 # build/wannabuild slots: Build objects move along this sequence | |
| 373 self.buildable = [] | |
| 374 self.building = [] | |
| 375 # old_building holds active builds that were stolen from a predecessor | |
| 376 self.old_building = weakref.WeakKeyDictionary() | |
| 377 | |
| 378 # buildslaves which have connected but which are not yet available. | |
| 379 # These are always in the ATTACHING state. | |
| 380 self.attaching_slaves = [] | |
| 381 | |
| 382 # buildslaves at our disposal. Each SlaveBuilder instance has a | |
| 383 # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a | |
| 384 # Build is about to start, to make sure that they're still alive. | |
| 385 self.slaves = [] | |
| 386 | |
| 387 self.builder_status = builder_status | |
| 388 self.builder_status.setSlavenames(self.slavenames) | |
| 389 | |
| 390 # for testing, to help synchronize tests | |
| 391 self.watchers = {'attach': [], 'detach': [], 'detach_all': [], | |
| 392 'idle': []} | |
| 393 | |
| 394 def setBotmaster(self, botmaster): | |
| 395 self.botmaster = botmaster | |
| 396 | |
| 397 def compareToSetup(self, setup): | |
| 398 diffs = [] | |
| 399 setup_slavenames = [] | |
| 400 if setup.has_key('slavename'): | |
| 401 setup_slavenames.append(setup['slavename']) | |
| 402 setup_slavenames.extend(setup.get('slavenames', [])) | |
| 403 if setup_slavenames != self.slavenames: | |
| 404 diffs.append('slavenames changed from %s to %s' \ | |
| 405 % (self.slavenames, setup_slavenames)) | |
| 406 if setup['builddir'] != self.builddir: | |
| 407 diffs.append('builddir changed from %s to %s' \ | |
| 408 % (self.builddir, setup['builddir'])) | |
| 409 if setup['slavebuilddir'] != self.slavebuilddir: | |
| 410 diffs.append('slavebuilddir changed from %s to %s' \ | |
| 411 % (self.slavebuilddir, setup['slavebuilddir'])) | |
| 412 if setup['factory'] != self.buildFactory: # compare objects | |
| 413 diffs.append('factory changed') | |
| 414 if setup.get('locks', []) != self.locks: | |
| 415 diffs.append('locks changed from %s to %s' % (self.locks, setup.get(
'locks'))) | |
| 416 if setup.get('nextSlave') != self.nextSlave: | |
| 417 diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, se
tup['nextSlave'])) | |
| 418 if setup.get('nextBuild') != self.nextBuild: | |
| 419 diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, se
tup['nextBuild'])) | |
| 420 return diffs | |
| 421 | |
| 422 def __repr__(self): | |
| 423 return "<Builder '%s' at %d>" % (self.name, id(self)) | |
| 424 | |
| 425 def getOldestRequestTime(self): | |
| 426 """Returns the timestamp of the oldest build request for this builder. | |
| 427 | |
| 428 If there are no build requests, None is returned.""" | |
| 429 if self.buildable: | |
| 430 return self.buildable[0].getSubmitTime() | |
| 431 else: | |
| 432 return None | |
| 433 | |
| 434 def submitBuildRequest(self, req): | |
| 435 req.setSubmitTime(now()) | |
| 436 self.buildable.append(req) | |
| 437 req.requestSubmitted(self) | |
| 438 self.builder_status.addBuildRequest(req.status) | |
| 439 self.botmaster.maybeStartAllBuilds() | |
| 440 | |
| 441 def cancelBuildRequest(self, req): | |
| 442 if req in self.buildable: | |
| 443 self.buildable.remove(req) | |
| 444 self.builder_status.removeBuildRequest(req.status, cancelled=True) | |
| 445 return True | |
| 446 return False | |
| 447 | |
| 448 def consumeTheSoulOfYourPredecessor(self, old): | |
| 449 """Suck the brain out of an old Builder. | |
| 450 | |
| 451 This takes all the runtime state from an existing Builder and moves | |
| 452 it into ourselves. This is used when a Builder is changed in the | |
| 453 master.cfg file: the new Builder has a different factory, but we want | |
| 454 all the builds that were queued for the old one to get processed by | |
| 455 the new one. Any builds which are already running will keep running. | |
| 456 The new Builder will get as many of the old SlaveBuilder objects as | |
| 457 it wants.""" | |
| 458 | |
| 459 log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" % | |
| 460 (self, old)) | |
| 461 # we claim all the pending builds, removing them from the old | |
| 462 # Builder's queue. This insures that the old Builder will not start | |
| 463 # any new work. | |
| 464 log.msg(" stealing %s buildrequests" % len(old.buildable)) | |
| 465 self.buildable.extend(old.buildable) | |
| 466 old.buildable = [] | |
| 467 | |
| 468 # old.building (i.e. builds which are still running) is not migrated | |
| 469 # directly: it keeps track of builds which were in progress in the | |
| 470 # old Builder. When those builds finish, the old Builder will be | |
| 471 # notified, not us. However, since the old SlaveBuilder will point to | |
| 472 # us, it is our maybeStartBuild() that will be triggered. | |
| 473 if old.building: | |
| 474 self.builder_status.setBigState("building") | |
| 475 # however, we do grab a weakref to the active builds, so that our | |
| 476 # BuilderControl can see them and stop them. We use a weakref because | |
| 477 # we aren't the one to get notified, so there isn't a convenient | |
| 478 # place to remove it from self.building . | |
| 479 for b in old.building: | |
| 480 self.old_building[b] = None | |
| 481 for b in old.old_building: | |
| 482 self.old_building[b] = None | |
| 483 | |
| 484 # Our set of slavenames may be different. Steal any of the old | |
| 485 # buildslaves that we want to keep using. | |
| 486 for sb in old.slaves[:]: | |
| 487 if sb.slave.slavename in self.slavenames: | |
| 488 log.msg(" stealing buildslave %s" % sb) | |
| 489 self.slaves.append(sb) | |
| 490 old.slaves.remove(sb) | |
| 491 sb.setBuilder(self) | |
| 492 | |
| 493 # old.attaching_slaves: | |
| 494 # these SlaveBuilders are waiting on a sequence of calls: | |
| 495 # remote.setMaster and remote.print . When these two complete, | |
| 496 # old._attached will be fired, which will add a 'connect' event to | |
| 497 # the builder_status and try to start a build. However, we've pulled | |
| 498 # everything out of the old builder's queue, so it will have no work | |
| 499 # to do. The outstanding remote.setMaster/print call will be holding | |
| 500 # the last reference to the old builder, so it will disappear just | |
| 501 # after that response comes back. | |
| 502 # | |
| 503 # The BotMaster will ask the slave to re-set their list of Builders | |
| 504 # shortly after this function returns, which will cause our | |
| 505 # attached() method to be fired with a bunch of references to remote | |
| 506 # SlaveBuilders, some of which we already have (by stealing them | |
| 507 # from the old Builder), some of which will be new. The new ones | |
| 508 # will be re-attached. | |
| 509 | |
| 510 # Therefore, we don't need to do anything about old.attaching_slaves | |
| 511 | |
| 512 return # all done | |
| 513 | |
| 514 def getBuild(self, number): | |
| 515 for b in self.building: | |
| 516 if b.build_status and b.build_status.number == number: | |
| 517 return b | |
| 518 for b in self.old_building.keys(): | |
| 519 if b.build_status and b.build_status.number == number: | |
| 520 return b | |
| 521 return None | |
| 522 | |
| 523 def fireTestEvent(self, name, fire_with=None): | |
| 524 if fire_with is None: | |
| 525 fire_with = self | |
| 526 watchers = self.watchers[name] | |
| 527 self.watchers[name] = [] | |
| 528 for w in watchers: | |
| 529 reactor.callLater(0, w.callback, fire_with) | |
| 530 | |
| 531 def addLatentSlave(self, slave): | |
| 532 assert interfaces.ILatentBuildSlave.providedBy(slave) | |
| 533 for s in self.slaves: | |
| 534 if s == slave: | |
| 535 break | |
| 536 else: | |
| 537 sb = LatentSlaveBuilder(slave, self) | |
| 538 self.builder_status.addPointEvent( | |
| 539 ['added', 'latent', slave.slavename]) | |
| 540 self.slaves.append(sb) | |
| 541 reactor.callLater(0, self.botmaster.maybeStartAllBuilds) | |
| 542 | |
| 543 def attached(self, slave, remote, commands): | |
| 544 """This is invoked by the BuildSlave when the self.slavename bot | |
| 545 registers their builder. | |
| 546 | |
| 547 @type slave: L{buildbot.buildslave.BuildSlave} | |
| 548 @param slave: the BuildSlave that represents the buildslave as a whole | |
| 549 @type remote: L{twisted.spread.pb.RemoteReference} | |
| 550 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} | |
| 551 @type commands: dict: string -> string, or None | |
| 552 @param commands: provides the slave's version of each RemoteCommand | |
| 553 | |
| 554 @rtype: L{twisted.internet.defer.Deferred} | |
| 555 @return: a Deferred that fires (with 'self') when the slave-side | |
| 556 builder is fully attached and ready to accept commands. | |
| 557 """ | |
| 558 for s in self.attaching_slaves + self.slaves: | |
| 559 if s.slave == slave: | |
| 560 # already attached to them. This is fairly common, since | |
| 561 # attached() gets called each time we receive the builder | |
| 562 # list from the slave, and we ask for it each time we add or | |
| 563 # remove a builder. So if the slave is hosting builders | |
| 564 # A,B,C, and the config file changes A, we'll remove A and | |
| 565 # re-add it, triggering two builder-list requests, getting | |
| 566 # two redundant calls to attached() for B, and another two | |
| 567 # for C. | |
| 568 # | |
| 569 # Therefore, when we see that we're already attached, we can | |
| 570 # just ignore it. TODO: build a diagram of the state | |
| 571 # transitions here, I'm concerned about sb.attached() failing | |
| 572 # and leaving sb.state stuck at 'ATTACHING', and about | |
| 573 # the detached() message arriving while there's some | |
| 574 # transition pending such that the response to the transition | |
| 575 # re-vivifies sb | |
| 576 return defer.succeed(self) | |
| 577 | |
| 578 sb = SlaveBuilder() | |
| 579 sb.setBuilder(self) | |
| 580 self.attaching_slaves.append(sb) | |
| 581 d = sb.attached(slave, remote, commands) | |
| 582 d.addCallback(self._attached) | |
| 583 d.addErrback(self._not_attached, slave) | |
| 584 return d | |
| 585 | |
| 586 def _attached(self, sb): | |
| 587 # TODO: make this .addSlaveEvent(slave.slavename, ['connect']) ? | |
| 588 self.builder_status.addPointEvent(['connect', sb.slave.slavename]) | |
| 589 self.attaching_slaves.remove(sb) | |
| 590 self.slaves.append(sb) | |
| 591 | |
| 592 self.fireTestEvent('attach') | |
| 593 return self | |
| 594 | |
| 595 def _not_attached(self, why, slave): | |
| 596 # already log.err'ed by SlaveBuilder._attachFailure | |
| 597 # TODO: make this .addSlaveEvent? | |
| 598 # TODO: remove from self.slaves (except that detached() should get | |
| 599 # run first, right?) | |
| 600 self.builder_status.addPointEvent(['failed', 'connect', | |
| 601 slave.slave.slavename]) | |
| 602 # TODO: add an HTMLLogFile of the exception | |
| 603 self.fireTestEvent('attach', why) | |
| 604 | |
| 605 def detached(self, slave): | |
| 606 """This is called when the connection to the bot is lost.""" | |
| 607 for sb in self.attaching_slaves + self.slaves: | |
| 608 if sb.slave == slave: | |
| 609 break | |
| 610 else: | |
| 611 log.msg("WEIRD: Builder.detached(%s) (%s)" | |
| 612 " not in attaching_slaves(%s)" | |
| 613 " or slaves(%s)" % (slave, slave.slavename, | |
| 614 self.attaching_slaves, | |
| 615 self.slaves)) | |
| 616 return | |
| 617 if sb.state == BUILDING: | |
| 618 # the Build's .lostRemote method (invoked by a notifyOnDisconnect | |
| 619 # handler) will cause the Build to be stopped, probably right | |
| 620 # after the notifyOnDisconnect that invoked us finishes running. | |
| 621 | |
| 622 # TODO: should failover to a new Build | |
| 623 #self.retryBuild(sb.build) | |
| 624 pass | |
| 625 | |
| 626 if sb in self.attaching_slaves: | |
| 627 self.attaching_slaves.remove(sb) | |
| 628 if sb in self.slaves: | |
| 629 self.slaves.remove(sb) | |
| 630 | |
| 631 # TODO: make this .addSlaveEvent? | |
| 632 self.builder_status.addPointEvent(['disconnect', slave.slavename]) | |
| 633 sb.detached() # inform the SlaveBuilder that their slave went away | |
| 634 self.updateBigStatus() | |
| 635 self.fireTestEvent('detach') | |
| 636 if not self.slaves: | |
| 637 self.fireTestEvent('detach_all') | |
| 638 | |
| 639 def updateBigStatus(self): | |
| 640 if not self.slaves: | |
| 641 self.builder_status.setBigState("offline") | |
| 642 elif self.building: | |
| 643 self.builder_status.setBigState("building") | |
| 644 else: | |
| 645 self.builder_status.setBigState("idle") | |
| 646 self.fireTestEvent('idle') | |
| 647 | |
| 648 def maybeStartBuild(self): | |
| 649 log.msg("maybeStartBuild %s: %i request(s), %i slave(s)" % | |
| 650 (self, len(self.buildable), len(self.slaves))) | |
| 651 if not self.buildable: | |
| 652 self.updateBigStatus() | |
| 653 return # nothing to do | |
| 654 | |
| 655 # pick an idle slave | |
| 656 available_slaves = [sb for sb in self.slaves if sb.isAvailable()] | |
| 657 if not available_slaves: | |
| 658 self.updateBigStatus() | |
| 659 return | |
| 660 if self.nextSlave: | |
| 661 sb = None | |
| 662 try: | |
| 663 sb = self.nextSlave(self, available_slaves) | |
| 664 except: | |
| 665 log.msg("Exception choosing next slave") | |
| 666 log.err(Failure()) | |
| 667 | |
| 668 if not sb: | |
| 669 self.updateBigStatus() | |
| 670 return | |
| 671 elif self.CHOOSE_SLAVES_RANDOMLY: | |
| 672 sb = random.choice(available_slaves) | |
| 673 else: | |
| 674 sb = available_slaves[0] | |
| 675 | |
| 676 # there is something to build, and there is a slave on which to build | |
| 677 # it. Grab the oldest request, see if we can merge it with anything | |
| 678 # else. | |
| 679 if not self.nextBuild: | |
| 680 req = self.buildable.pop(0) | |
| 681 else: | |
| 682 try: | |
| 683 req = self.nextBuild(self, self.buildable) | |
| 684 if not req: | |
| 685 # Nothing to do | |
| 686 self.updateBigStatus() | |
| 687 return | |
| 688 self.buildable.remove(req) | |
| 689 except: | |
| 690 log.msg("Exception choosing next build") | |
| 691 log.err(Failure()) | |
| 692 self.updateBigStatus() | |
| 693 return | |
| 694 self.builder_status.removeBuildRequest(req.status) | |
| 695 mergers = [] | |
| 696 botmaster = self.botmaster | |
| 697 for br in self.buildable[:]: | |
| 698 if botmaster.shouldMergeRequests(self, req, br): | |
| 699 self.buildable.remove(br) | |
| 700 self.builder_status.removeBuildRequest(br.status) | |
| 701 mergers.append(br) | |
| 702 requests = [req] + mergers | |
| 703 enforced_sb = [request.properties.getProperty('slavename') | |
| 704 for request in requests | |
| 705 if request.properties.getProperty('slavename')] | |
| 706 enforced_sb = list(set(enforced_sb)) | |
| 707 if len(enforced_sb) == 1: | |
| 708 if enforced_sb[0] not in self.slaves: | |
| 709 # It's better to not use slaves at all than use a random one. | |
| 710 log.msg("%s: %s is not a valid slave" % (self, enforced_sb[0])) | |
| 711 return | |
| 712 sb = enforced_sb[0] | |
| 713 | |
| 714 # Create a new build from our build factory and set ourself as the | |
| 715 # builder. | |
| 716 build = self.buildFactory.newBuild(requests) | |
| 717 build.setBuilder(self) | |
| 718 build.setLocks(self.locks) | |
| 719 if len(self.env) > 0: | |
| 720 build.setSlaveEnvironment(self.env) | |
| 721 | |
| 722 # start it | |
| 723 self.startBuild(build, sb) | |
| 724 | |
| 725 def startBuild(self, build, sb): | |
| 726 """Start a build on the given slave. | |
| 727 @param build: the L{base.Build} to start | |
| 728 @param sb: the L{SlaveBuilder} which will host this build | |
| 729 | |
| 730 @return: a Deferred which fires with a | |
| 731 L{buildbot.interfaces.IBuildControl} that can be used to stop the | |
| 732 Build, or to access a L{buildbot.interfaces.IBuildStatus} which will | |
| 733 watch the Build as it runs. """ | |
| 734 | |
| 735 self.building.append(build) | |
| 736 self.updateBigStatus() | |
| 737 log.msg("starting build %s using slave %s" % (build, sb)) | |
| 738 d = sb.prepare(self.builder_status) | |
| 739 def _ping(ign): | |
| 740 # ping the slave to make sure they're still there. If they've | |
| 741 # fallen off the map (due to a NAT timeout or something), this | |
| 742 # will fail in a couple of minutes, depending upon the TCP | |
| 743 # timeout. | |
| 744 # | |
| 745 # TODO: This can unnecessarily suspend the starting of a build, in | |
| 746 # situations where the slave is live but is pushing lots of data to | |
| 747 # us in a build. | |
| 748 log.msg("starting build %s.. pinging the slave %s" % (build, sb)) | |
| 749 return sb.ping() | |
| 750 d.addCallback(_ping) | |
| 751 d.addCallback(self._startBuild_1, build, sb) | |
| 752 return d | |
| 753 | |
| 754 def _startBuild_1(self, res, build, sb): | |
| 755 if not res: | |
| 756 return self._startBuildFailed("slave ping failed", build, sb) | |
| 757 # The buildslave is ready to go. sb.buildStarted() sets its state to | |
| 758 # BUILDING (so we won't try to use it for any other builds). This | |
| 759 # gets set back to IDLE by the Build itself when it finishes. | |
| 760 sb.buildStarted() | |
| 761 d = sb.remote.callRemote("startBuild") | |
| 762 d.addCallbacks(self._startBuild_2, self._startBuildFailed, | |
| 763 callbackArgs=(build,sb), errbackArgs=(build,sb)) | |
| 764 return d | |
| 765 | |
| 766 def _startBuild_2(self, res, build, sb): | |
| 767 # create the BuildStatus object that goes with the Build | |
| 768 bs = self.builder_status.newBuild() | |
| 769 | |
| 770 # start the build. This will first set up the steps, then tell the | |
| 771 # BuildStatus that it has started, which will announce it to the | |
| 772 # world (through our BuilderStatus object, which is its parent). | |
| 773 # Finally it will start the actual build process. | |
| 774 d = build.startBuild(bs, self.expectations, sb) | |
| 775 d.addCallback(self.buildFinished, sb) | |
| 776 d.addErrback(log.err) # this shouldn't happen. if it does, the slave | |
| 777 # will be wedged | |
| 778 for req in build.requests: | |
| 779 req.buildStarted(build, bs) | |
| 780 return build # this is the IBuildControl | |
| 781 | |
| 782 def _startBuildFailed(self, why, build, sb): | |
| 783 # put the build back on the buildable list | |
| 784 log.msg("I tried to tell the slave that the build %s started, but " | |
| 785 "remote_startBuild failed: %s" % (build, why)) | |
| 786 # release the slave. This will queue a call to maybeStartBuild, which | |
| 787 # will fire after other notifyOnDisconnect handlers have marked the | |
| 788 # slave as disconnected (so we don't try to use it again). | |
| 789 sb.buildFinished() | |
| 790 | |
| 791 log.msg("re-queueing the BuildRequest") | |
| 792 self.building.remove(build) | |
| 793 for req in build.requests: | |
| 794 self.buildable.insert(0, req) # the interrupted build gets first | |
| 795 # priority | |
| 796 self.builder_status.addBuildRequest(req.status) | |
| 797 | |
| 798 | |
| 799 def buildFinished(self, build, sb): | |
| 800 """This is called when the Build has finished (either success or | |
| 801 failure). Any exceptions during the build are reported with | |
| 802 results=FAILURE, not with an errback.""" | |
| 803 | |
| 804 # by the time we get here, the Build has already released the slave | |
| 805 # (which queues a call to maybeStartBuild) | |
| 806 | |
| 807 self.building.remove(build) | |
| 808 for req in build.requests: | |
| 809 req.finished(build.build_status) | |
| 810 | |
| 811 def setExpectations(self, progress): | |
| 812 """Mark the build as successful and update expectations for the next | |
| 813 build. Only call this when the build did not fail in any way that | |
| 814 would invalidate the time expectations generated by it. (if the | |
| 815 compile failed and thus terminated early, we can't use the last | |
| 816 build to predict how long the next one will take). | |
| 817 """ | |
| 818 if self.expectations: | |
| 819 self.expectations.update(progress) | |
| 820 else: | |
| 821 # the first time we get a good build, create our Expectations | |
| 822 # based upon its results | |
| 823 self.expectations = Expectations(progress) | |
| 824 log.msg("new expectations: %s seconds" % \ | |
| 825 self.expectations.expectedBuildTime()) | |
| 826 | |
| 827 def shutdownSlave(self): | |
| 828 if self.remote: | |
| 829 self.remote.callRemote("shutdown") | |
| 830 | |
| 831 | |
| 832 class BuilderControl(components.Adapter): | |
| 833 implements(interfaces.IBuilderControl) | |
| 834 | |
| 835 def requestBuild(self, req): | |
| 836 """Submit a BuildRequest to this Builder.""" | |
| 837 self.original.submitBuildRequest(req) | |
| 838 | |
| 839 def requestBuildSoon(self, req): | |
| 840 """Submit a BuildRequest like requestBuild, but raise a | |
| 841 L{buildbot.interfaces.NoSlaveError} if no slaves are currently | |
| 842 available, so it cannot be used to queue a BuildRequest in the hopes | |
| 843 that a slave will eventually connect. This method is appropriate for | |
| 844 use by things like the web-page 'Force Build' button.""" | |
| 845 if not self.original.slaves: | |
| 846 raise interfaces.NoSlaveError | |
| 847 self.requestBuild(req) | |
| 848 | |
| 849 def resubmitBuild(self, bs, reason="<rebuild, no reason given>", extraProper
ties=None): | |
| 850 if not bs.isFinished(): | |
| 851 return | |
| 852 | |
| 853 ss = bs.getSourceStamp(absolute=True) | |
| 854 if extraProperties is None: | |
| 855 properties = bs.getProperties() | |
| 856 else: | |
| 857 # Make a copy so as not to modify the original build. | |
| 858 properties = Properties() | |
| 859 properties.updateFromProperties(bs.getProperties()) | |
| 860 properties.updateFromProperties(extraProperties) | |
| 861 req = base.BuildRequest(reason, ss, self.original.name, | |
| 862 properties=properties) | |
| 863 self.requestBuild(req) | |
| 864 | |
| 865 def getPendingBuilds(self): | |
| 866 # return IBuildRequestControl objects | |
| 867 retval = [] | |
| 868 for r in self.original.buildable: | |
| 869 retval.append(BuildRequestControl(self.original, r)) | |
| 870 | |
| 871 return retval | |
| 872 | |
| 873 def getBuild(self, number): | |
| 874 return self.original.getBuild(number) | |
| 875 | |
| 876 def ping(self): | |
| 877 if not self.original.slaves: | |
| 878 self.original.builder_status.addPointEvent(["ping", "no slave"]) | |
| 879 return defer.succeed(False) # interfaces.NoSlaveError | |
| 880 dl = [] | |
| 881 for s in self.original.slaves: | |
| 882 dl.append(s.ping(self.original.builder_status)) | |
| 883 d = defer.DeferredList(dl) | |
| 884 d.addCallback(self._gatherPingResults) | |
| 885 return d | |
| 886 | |
| 887 def _gatherPingResults(self, res): | |
| 888 for ignored,success in res: | |
| 889 if not success: | |
| 890 return False | |
| 891 return True | |
| 892 | |
| 893 components.registerAdapter(BuilderControl, Builder, interfaces.IBuilderControl) | |
| 894 | |
| 895 class BuildRequestControl: | |
| 896 implements(interfaces.IBuildRequestControl) | |
| 897 | |
| 898 def __init__(self, builder, request): | |
| 899 self.original_builder = builder | |
| 900 self.original_request = request | |
| 901 | |
| 902 def subscribe(self, observer): | |
| 903 raise NotImplementedError | |
| 904 | |
| 905 def unsubscribe(self, observer): | |
| 906 raise NotImplementedError | |
| 907 | |
| 908 def cancel(self): | |
| 909 self.original_builder.cancelBuildRequest(self.original_request) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build