| OLD | NEW |
|---|
| (Empty) |
| 1 # Portions copyright Canonical Ltd. 2009 | |
| 2 | |
| 3 import time | |
| 4 from email.Message import Message | |
| 5 from email.Utils import formatdate | |
| 6 from zope.interface import implements | |
| 7 from twisted.python import log | |
| 8 from twisted.internet import defer, reactor | |
| 9 from twisted.application import service | |
| 10 import twisted.spread.pb | |
| 11 | |
| 12 from buildbot.pbutil import NewCredPerspective | |
| 13 from buildbot.status.builder import SlaveStatus | |
| 14 from buildbot.status.mail import MailNotifier | |
| 15 from buildbot.interfaces import IBuildSlave, ILatentBuildSlave | |
| 16 from buildbot.process.properties import Properties | |
| 17 | |
| 18 import sys | |
| 19 if sys.version_info[:3] < (2,4,0): | |
| 20 from sets import Set as set | |
| 21 | |
| 22 class AbstractBuildSlave(NewCredPerspective, service.MultiService): | |
| 23 """This is the master-side representative for a remote buildbot slave. | |
| 24 There is exactly one for each slave described in the config file (the | |
| 25 c['slaves'] list). When buildbots connect in (.attach), they get a | |
| 26 reference to this instance. The BotMaster object is stashed as the | |
| 27 .botmaster attribute. The BotMaster is also our '.parent' Service. | |
| 28 | |
| 29 I represent a build slave -- a remote machine capable of | |
| 30 running builds. I am instantiated by the configuration file, and can be | |
| 31 subclassed to add extra functionality.""" | |
| 32 | |
| 33 implements(IBuildSlave) | |
| 34 | |
| 35 def __init__(self, name, password, max_builds=None, | |
| 36 notify_on_missing=[], missing_timeout=3600, | |
| 37 properties={}): | |
| 38 """ | |
| 39 @param name: botname this machine will supply when it connects | |
| 40 @param password: password this machine will supply when | |
| 41 it connects | |
| 42 @param max_builds: maximum number of simultaneous builds that will | |
| 43 be run concurrently on this buildslave (the | |
| 44 default is None for no limit) | |
| 45 @param properties: properties that will be applied to builds run on | |
| 46 this slave | |
| 47 @type properties: dictionary | |
| 48 """ | |
| 49 service.MultiService.__init__(self) | |
| 50 self.slavename = name | |
| 51 self.password = password | |
| 52 self.botmaster = None # no buildmaster yet | |
| 53 self.slave_status = SlaveStatus(name) | |
| 54 self.slave = None # a RemoteReference to the Bot, when connected | |
| 55 self.slave_commands = None | |
| 56 self.slavebuilders = {} | |
| 57 self.max_builds = max_builds | |
| 58 | |
| 59 self.properties = Properties() | |
| 60 self.properties.update(properties, "BuildSlave") | |
| 61 self.properties.setProperty("slavename", name, "BuildSlave") | |
| 62 | |
| 63 self.lastMessageReceived = 0 | |
| 64 if isinstance(notify_on_missing, str): | |
| 65 notify_on_missing = [notify_on_missing] | |
| 66 self.notify_on_missing = notify_on_missing | |
| 67 for i in notify_on_missing: | |
| 68 assert isinstance(i, str) | |
| 69 self.missing_timeout = missing_timeout | |
| 70 self.missing_timer = None | |
| 71 | |
| 72 def update(self, new): | |
| 73 """ | |
| 74 Given a new BuildSlave, configure this one identically. Because | |
| 75 BuildSlave objects are remotely referenced, we can't replace them | |
| 76 without disconnecting the slave, yet there's no reason to do that. | |
| 77 """ | |
| 78 # the reconfiguration logic should guarantee this: | |
| 79 assert self.slavename == new.slavename | |
| 80 assert self.password == new.password | |
| 81 assert self.__class__ == new.__class__ | |
| 82 self.max_builds = new.max_builds | |
| 83 | |
| 84 def __repr__(self): | |
| 85 if self.botmaster: | |
| 86 builders = self.botmaster.getBuildersForSlave(self.slavename) | |
| 87 return "<%s '%s', current builders: %s>" % \ | |
| 88 (self.__class__.__name__, self.slavename, | |
| 89 ','.join(map(lambda b: b.name, builders))) | |
| 90 else: | |
| 91 return "<%s '%s', (no builders yet)>" % \ | |
| 92 (self.__class__.__name__, self.slavename) | |
| 93 | |
| 94 def setBotmaster(self, botmaster): | |
| 95 assert not self.botmaster, "BuildSlave already has a botmaster" | |
| 96 self.botmaster = botmaster | |
| 97 self.startMissingTimer() | |
| 98 | |
| 99 def stopMissingTimer(self): | |
| 100 if self.missing_timer: | |
| 101 self.missing_timer.cancel() | |
| 102 self.missing_timer = None | |
| 103 | |
| 104 def startMissingTimer(self): | |
| 105 if self.notify_on_missing and self.missing_timeout and self.parent: | |
| 106 self.stopMissingTimer() # in case it's already running | |
| 107 self.missing_timer = reactor.callLater(self.missing_timeout, | |
| 108 self._missing_timer_fired) | |
| 109 | |
| 110 def _missing_timer_fired(self): | |
| 111 self.missing_timer = None | |
| 112 # notify people, but only if we're still in the config | |
| 113 if not self.parent: | |
| 114 return | |
| 115 | |
| 116 buildmaster = self.botmaster.parent | |
| 117 status = buildmaster.getStatus() | |
| 118 text = "The Buildbot working for '%s'\n" % status.getProjectName() | |
| 119 text += ("has noticed that the buildslave named %s went away\n" % | |
| 120 self.slavename) | |
| 121 text += "\n" | |
| 122 text += ("It last disconnected at %s (buildmaster-local time)\n" % | |
| 123 time.ctime(time.time() - self.missing_timeout)) # approx | |
| 124 text += "\n" | |
| 125 text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" | |
| 126 text += "was '%s'.\n" % self.slave_status.getAdmin() | |
| 127 text += "\n" | |
| 128 text += "Sincerely,\n" | |
| 129 text += " The Buildbot\n" | |
| 130 text += " %s\n" % status.getProjectURL() | |
| 131 subject = "Buildbot: buildslave %s was lost" % self.slavename | |
| 132 return self._mail_missing_message(subject, text) | |
| 133 | |
| 134 | |
| 135 def updateSlave(self): | |
| 136 """Called to add or remove builders after the slave has connected. | |
| 137 | |
| 138 @return: a Deferred that indicates when an attached slave has | |
| 139 accepted the new builders and/or released the old ones.""" | |
| 140 if self.slave: | |
| 141 return self.sendBuilderList() | |
| 142 else: | |
| 143 return defer.succeed(None) | |
| 144 | |
| 145 def updateSlaveStatus(self, buildStarted=None, buildFinished=None): | |
| 146 if buildStarted: | |
| 147 self.slave_status.buildStarted(buildStarted) | |
| 148 if buildFinished: | |
| 149 self.slave_status.buildFinished(buildFinished) | |
| 150 | |
| 151 def attached(self, bot): | |
| 152 """This is called when the slave connects. | |
| 153 | |
| 154 @return: a Deferred that fires with a suitable pb.IPerspective to | |
| 155 give to the slave (i.e. 'self')""" | |
| 156 | |
| 157 if self.slave: | |
| 158 # uh-oh, we've got a duplicate slave. The most likely | |
| 159 # explanation is that the slave is behind a slow link, thinks we | |
| 160 # went away, and has attempted to reconnect, so we've got two | |
| 161 # "connections" from the same slave, but the previous one is | |
| 162 # stale. Give the new one precedence. | |
| 163 log.msg("duplicate slave %s replacing old one" % self.slavename) | |
| 164 | |
| 165 # just in case we've got two identically-configured slaves, | |
| 166 # report the IP addresses of both so someone can resolve the | |
| 167 # squabble | |
| 168 tport = self.slave.broker.transport | |
| 169 log.msg("old slave was connected from", tport.getPeer()) | |
| 170 log.msg("new slave is from", bot.broker.transport.getPeer()) | |
| 171 d = self.disconnect() | |
| 172 else: | |
| 173 d = defer.succeed(None) | |
| 174 # now we go through a sequence of calls, gathering information, then | |
| 175 # tell the Botmaster that it can finally give this slave to all the | |
| 176 # Builders that care about it. | |
| 177 | |
| 178 # we accumulate slave information in this 'state' dictionary, then | |
| 179 # set it atomically if we make it far enough through the process | |
| 180 state = {} | |
| 181 | |
| 182 # Reset graceful shutdown status | |
| 183 self.slave_status.setGraceful(False) | |
| 184 # We want to know when the graceful shutdown flag changes | |
| 185 self.slave_status.addGracefulWatcher(self._gracefulChanged) | |
| 186 | |
| 187 def _log_attachment_on_slave(res): | |
| 188 d1 = bot.callRemote("print", "attached") | |
| 189 d1.addErrback(lambda why: None) | |
| 190 return d1 | |
| 191 d.addCallback(_log_attachment_on_slave) | |
| 192 | |
| 193 def _get_info(res): | |
| 194 d1 = bot.callRemote("getSlaveInfo") | |
| 195 def _got_info(info): | |
| 196 log.msg("Got slaveinfo from '%s'" % self.slavename) | |
| 197 # TODO: info{} might have other keys | |
| 198 state["admin"] = info.get("admin") | |
| 199 state["host"] = info.get("host") | |
| 200 state["access_uri"] = info.get("access_uri", None) | |
| 201 def _info_unavailable(why): | |
| 202 # maybe an old slave, doesn't implement remote_getSlaveInfo | |
| 203 log.msg("BuildSlave.info_unavailable") | |
| 204 log.err(why) | |
| 205 d1.addCallbacks(_got_info, _info_unavailable) | |
| 206 return d1 | |
| 207 d.addCallback(_get_info) | |
| 208 | |
| 209 def _get_version(res): | |
| 210 d1 = bot.callRemote("getVersion") | |
| 211 def _got_version(version): | |
| 212 state["version"] = version | |
| 213 def _version_unavailable(why): | |
| 214 # probably an old slave | |
| 215 log.msg("BuildSlave.version_unavailable") | |
| 216 log.err(why) | |
| 217 d1.addCallbacks(_got_version, _version_unavailable) | |
| 218 d.addCallback(_get_version) | |
| 219 | |
| 220 def _get_commands(res): | |
| 221 d1 = bot.callRemote("getCommands") | |
| 222 def _got_commands(commands): | |
| 223 state["slave_commands"] = commands | |
| 224 def _commands_unavailable(why): | |
| 225 # probably an old slave | |
| 226 log.msg("BuildSlave._commands_unavailable") | |
| 227 if why.check(AttributeError): | |
| 228 return | |
| 229 log.err(why) | |
| 230 d1.addCallbacks(_got_commands, _commands_unavailable) | |
| 231 return d1 | |
| 232 d.addCallback(_get_commands) | |
| 233 | |
| 234 def _accept_slave(res): | |
| 235 self.slave_status.setAdmin(state.get("admin")) | |
| 236 self.slave_status.setHost(state.get("host")) | |
| 237 self.slave_status.setAccessURI(state.get("access_uri")) | |
| 238 self.slave_status.setVersion(state.get("version")) | |
| 239 self.slave_status.setConnected(True) | |
| 240 self.slave_commands = state.get("slave_commands") | |
| 241 self.slave = bot | |
| 242 log.msg("bot attached") | |
| 243 self.messageReceivedFromSlave() | |
| 244 self.stopMissingTimer() | |
| 245 self.botmaster.parent.status.slaveConnected(self.slavename) | |
| 246 | |
| 247 return self.updateSlave() | |
| 248 d.addCallback(_accept_slave) | |
| 249 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) | |
| 250 | |
| 251 # Finally, the slave gets a reference to this BuildSlave. They | |
| 252 # receive this later, after we've started using them. | |
| 253 d.addCallback(lambda res: self) | |
| 254 return d | |
| 255 | |
| 256 def messageReceivedFromSlave(self): | |
| 257 now = time.time() | |
| 258 self.lastMessageReceived = now | |
| 259 self.slave_status.setLastMessageReceived(now) | |
| 260 | |
| 261 def detached(self, mind): | |
| 262 self.slave = None | |
| 263 self.slave_status.removeGracefulWatcher(self._gracefulChanged) | |
| 264 self.slave_status.setConnected(False) | |
| 265 log.msg("BuildSlave.detached(%s)" % self.slavename) | |
| 266 self.botmaster.parent.status.slaveDisconnected(self.slavename) | |
| 267 | |
| 268 def disconnect(self): | |
| 269 """Forcibly disconnect the slave. | |
| 270 | |
| 271 This severs the TCP connection and returns a Deferred that will fire | |
| 272 (with None) when the connection is probably gone. | |
| 273 | |
| 274 If the slave is still alive, they will probably try to reconnect | |
| 275 again in a moment. | |
| 276 | |
| 277 This is called in two circumstances. The first is when a slave is | |
| 278 removed from the config file. In this case, when they try to | |
| 279 reconnect, they will be rejected as an unknown slave. The second is | |
| 280 when we wind up with two connections for the same slave, in which | |
| 281 case we disconnect the older connection. | |
| 282 """ | |
| 283 | |
| 284 if not self.slave: | |
| 285 return defer.succeed(None) | |
| 286 log.msg("disconnecting old slave %s now" % self.slavename) | |
| 287 # When this Deferred fires, we'll be ready to accept the new slave | |
| 288 return self._disconnect(self.slave) | |
| 289 | |
| 290 def _disconnect(self, slave): | |
| 291 # all kinds of teardown will happen as a result of | |
| 292 # loseConnection(), but it happens after a reactor iteration or | |
| 293 # two. Hook the actual disconnect so we can know when it is safe | |
| 294 # to connect the new slave. We have to wait one additional | |
| 295 # iteration (with callLater(0)) to make sure the *other* | |
| 296 # notifyOnDisconnect handlers have had a chance to run. | |
| 297 d = defer.Deferred() | |
| 298 | |
| 299 # notifyOnDisconnect runs the callback with one argument, the | |
| 300 # RemoteReference being disconnected. | |
| 301 def _disconnected(rref): | |
| 302 reactor.callLater(0, d.callback, None) | |
| 303 slave.notifyOnDisconnect(_disconnected) | |
| 304 tport = slave.broker.transport | |
| 305 # this is the polite way to request that a socket be closed | |
| 306 tport.loseConnection() | |
| 307 try: | |
| 308 # but really we don't want to wait for the transmit queue to | |
| 309 # drain. The remote end is unlikely to ACK the data, so we'd | |
| 310 # probably have to wait for a (20-minute) TCP timeout. | |
| 311 #tport._closeSocket() | |
| 312 # however, doing _closeSocket (whether before or after | |
| 313 # loseConnection) somehow prevents the notifyOnDisconnect | |
| 314 # handlers from being run. Bummer. | |
| 315 tport.offset = 0 | |
| 316 tport.dataBuffer = "" | |
| 317 except: | |
| 318 # however, these hacks are pretty internal, so don't blow up if | |
| 319 # they fail or are unavailable | |
| 320 log.msg("failed to accelerate the shutdown process") | |
| 321 pass | |
| 322 log.msg("waiting for slave to finish disconnecting") | |
| 323 | |
| 324 return d | |
| 325 | |
| 326 def sendBuilderList(self): | |
| 327 our_builders = self.botmaster.getBuildersForSlave(self.slavename) | |
| 328 blist = [(b.name, b.slavebuilddir) for b in our_builders] | |
| 329 d = self.slave.callRemote("setBuilderList", blist) | |
| 330 return d | |
| 331 | |
| 332 def perspective_keepalive(self): | |
| 333 pass | |
| 334 | |
| 335 def addSlaveBuilder(self, sb): | |
| 336 self.slavebuilders[sb.builder_name] = sb | |
| 337 | |
| 338 def removeSlaveBuilder(self, sb): | |
| 339 try: | |
| 340 del self.slavebuilders[sb.builder_name] | |
| 341 except KeyError: | |
| 342 pass | |
| 343 | |
| 344 def canStartBuild(self): | |
| 345 """ | |
| 346 I am called when a build is requested to see if this buildslave | |
| 347 can start a build. This function can be used to limit overall | |
| 348 concurrency on the buildslave. | |
| 349 """ | |
| 350 # If we're waiting to shutdown gracefully, then we shouldn't | |
| 351 # accept any new jobs. | |
| 352 if self.slave_status.getGraceful(): | |
| 353 return False | |
| 354 | |
| 355 if self.max_builds: | |
| 356 active_builders = [sb for sb in self.slavebuilders.values() | |
| 357 if sb.isBusy()] | |
| 358 if len(active_builders) >= self.max_builds: | |
| 359 return False | |
| 360 return True | |
| 361 | |
| 362 def _mail_missing_message(self, subject, text): | |
| 363 # first, see if we have a MailNotifier we can use. This gives us a | |
| 364 # fromaddr and a relayhost. | |
| 365 buildmaster = self.botmaster.parent | |
| 366 for st in buildmaster.statusTargets: | |
| 367 if isinstance(st, MailNotifier): | |
| 368 break | |
| 369 else: | |
| 370 # if not, they get a default MailNotifier, which always uses SMTP | |
| 371 # to localhost and uses a dummy fromaddr of "buildbot". | |
| 372 log.msg("buildslave-missing msg using default MailNotifier") | |
| 373 st = MailNotifier("buildbot") | |
| 374 # now construct the mail | |
| 375 | |
| 376 m = Message() | |
| 377 m.set_payload(text) | |
| 378 m['Date'] = formatdate(localtime=True) | |
| 379 m['Subject'] = subject | |
| 380 m['From'] = st.fromaddr | |
| 381 recipients = self.notify_on_missing | |
| 382 m['To'] = ", ".join(recipients) | |
| 383 d = st.sendMessage(m, recipients) | |
| 384 # return the Deferred for testing purposes | |
| 385 return d | |
| 386 | |
| 387 def _gracefulChanged(self, graceful): | |
| 388 """This is called when our graceful shutdown setting changes""" | |
| 389 if graceful: | |
| 390 active_builders = [sb for sb in self.slavebuilders.values() | |
| 391 if sb.isBusy()] | |
| 392 if len(active_builders) == 0: | |
| 393 # Shut down! | |
| 394 self.shutdown() | |
| 395 | |
| 396 def shutdown(self): | |
| 397 """Shutdown the slave""" | |
| 398 # Look for a builder with a remote reference to the client side | |
| 399 # slave. If we can find one, then call "shutdown" on the remote | |
| 400 # builder, which will cause the slave buildbot process to exit. | |
| 401 d = None | |
| 402 for b in self.slavebuilders.values(): | |
| 403 if b.remote: | |
| 404 d = b.remote.callRemote("shutdown") | |
| 405 break | |
| 406 | |
| 407 if d: | |
| 408 log.msg("Shutting down slave: %s" % self.slavename) | |
| 409 # The remote shutdown call will not complete successfully since the | |
| 410 # buildbot process exits almost immediately after getting the | |
| 411 # shutdown request. | |
| 412 # Here we look at the reason why the remote call failed, and if | |
| 413 # it's because the connection was lost, that means the slave | |
| 414 # shutdown as expected. | |
| 415 def _errback(why): | |
| 416 if why.check(twisted.spread.pb.PBConnectionLost): | |
| 417 log.msg("Lost connection to %s" % self.slavename) | |
| 418 else: | |
| 419 log.err("Unexpected error when trying to shutdown %s" % self
.slavename) | |
| 420 d.addErrback(_errback) | |
| 421 return d | |
| 422 log.err("Couldn't find remote builder to shut down slave") | |
| 423 return defer.succeed(None) | |
| 424 | |
| 425 class BuildSlave(AbstractBuildSlave): | |
| 426 | |
| 427 def sendBuilderList(self): | |
| 428 d = AbstractBuildSlave.sendBuilderList(self) | |
| 429 def _sent(slist): | |
| 430 dl = [] | |
| 431 for name, remote in slist.items(): | |
| 432 # use get() since we might have changed our mind since then | |
| 433 b = self.botmaster.builders.get(name) | |
| 434 if b: | |
| 435 d1 = b.attached(self, remote, self.slave_commands) | |
| 436 dl.append(d1) | |
| 437 return defer.DeferredList(dl) | |
| 438 def _set_failed(why): | |
| 439 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) | |
| 440 log.err(why) | |
| 441 # TODO: hang up on them?, without setBuilderList we can't use | |
| 442 # them | |
| 443 d.addCallbacks(_sent, _set_failed) | |
| 444 return d | |
| 445 | |
| 446 def detached(self, mind): | |
| 447 AbstractBuildSlave.detached(self, mind) | |
| 448 self.botmaster.slaveLost(self) | |
| 449 self.startMissingTimer() | |
| 450 | |
| 451 def buildFinished(self, sb): | |
| 452 """This is called when a build on this slave is finished.""" | |
| 453 # If we're gracefully shutting down, and we have no more active | |
| 454 # builders, then it's safe to disconnect | |
| 455 if self.slave_status.getGraceful(): | |
| 456 active_builders = [sb for sb in self.slavebuilders.values() | |
| 457 if sb.isBusy()] | |
| 458 if len(active_builders) == 0: | |
| 459 # Shut down! | |
| 460 return self.shutdown() | |
| 461 return defer.succeed(None) | |
| 462 | |
| 463 class AbstractLatentBuildSlave(AbstractBuildSlave): | |
| 464 """A build slave that will start up a slave instance when needed. | |
| 465 | |
| 466 To use, subclass and implement start_instance and stop_instance. | |
| 467 | |
| 468 See ec2buildslave.py for a concrete example. Also see the stub example in | |
| 469 test/test_slaves.py. | |
| 470 """ | |
| 471 | |
| 472 implements(ILatentBuildSlave) | |
| 473 | |
| 474 substantiated = False | |
| 475 substantiation_deferred = None | |
| 476 build_wait_timer = None | |
| 477 _start_result = _shutdown_callback_handle = None | |
| 478 | |
| 479 def __init__(self, name, password, max_builds=None, | |
| 480 notify_on_missing=[], missing_timeout=60*20, | |
| 481 build_wait_timeout=60*10, | |
| 482 properties={}): | |
| 483 AbstractBuildSlave.__init__( | |
| 484 self, name, password, max_builds, notify_on_missing, | |
| 485 missing_timeout, properties) | |
| 486 self.building = set() | |
| 487 self.build_wait_timeout = build_wait_timeout | |
| 488 | |
| 489 def start_instance(self): | |
| 490 # responsible for starting instance that will try to connect with | |
| 491 # this master. Should return deferred. Problems should use an | |
| 492 # errback. | |
| 493 raise NotImplementedError | |
| 494 | |
| 495 def stop_instance(self, fast=False): | |
| 496 # responsible for shutting down instance. | |
| 497 raise NotImplementedError | |
| 498 | |
| 499 def substantiate(self, sb): | |
| 500 if self.substantiated: | |
| 501 self._clearBuildWaitTimer() | |
| 502 self._setBuildWaitTimer() | |
| 503 return defer.succeed(self) | |
| 504 if self.substantiation_deferred is None: | |
| 505 if self.parent and not self.missing_timer: | |
| 506 # start timer. if timer times out, fail deferred | |
| 507 self.missing_timer = reactor.callLater( | |
| 508 self.missing_timeout, | |
| 509 self._substantiation_failed, defer.TimeoutError()) | |
| 510 self.substantiation_deferred = defer.Deferred() | |
| 511 if self.slave is None: | |
| 512 self._substantiate() # start up instance | |
| 513 # else: we're waiting for an old one to detach. the _substantiate | |
| 514 # will be done in ``detached`` below. | |
| 515 return self.substantiation_deferred | |
| 516 | |
| 517 def _substantiate(self): | |
| 518 # register event trigger | |
| 519 d = self.start_instance() | |
| 520 self._shutdown_callback_handle = reactor.addSystemEventTrigger( | |
| 521 'before', 'shutdown', self._soft_disconnect, fast=True) | |
| 522 def stash_reply(result): | |
| 523 self._start_result = result | |
| 524 def clean_up(failure): | |
| 525 if self.missing_timer is not None: | |
| 526 self.missing_timer.cancel() | |
| 527 self._substantiation_failed(failure) | |
| 528 if self._shutdown_callback_handle is not None: | |
| 529 handle = self._shutdown_callback_handle | |
| 530 del self._shutdown_callback_handle | |
| 531 reactor.removeSystemEventTrigger(handle) | |
| 532 return failure | |
| 533 d.addCallbacks(stash_reply, clean_up) | |
| 534 return d | |
| 535 | |
| 536 def attached(self, bot): | |
| 537 if self.substantiation_deferred is None: | |
| 538 msg = 'Slave %s received connection while not trying to ' \ | |
| 539 'substantiate. Disconnecting.' % (self.slavename,) | |
| 540 log.msg(msg) | |
| 541 self._disconnect(bot) | |
| 542 return defer.fail(RuntimeError(msg)) | |
| 543 return AbstractBuildSlave.attached(self, bot) | |
| 544 | |
| 545 def detached(self, mind): | |
| 546 AbstractBuildSlave.detached(self, mind) | |
| 547 if self.substantiation_deferred is not None: | |
| 548 self._substantiate() | |
| 549 | |
| 550 def _substantiation_failed(self, failure): | |
| 551 d = self.substantiation_deferred | |
| 552 self.substantiation_deferred = None | |
| 553 self.missing_timer = None | |
| 554 d.errback(failure) | |
| 555 self.insubstantiate() | |
| 556 # notify people, but only if we're still in the config | |
| 557 if not self.parent or not self.notify_on_missing: | |
| 558 return | |
| 559 | |
| 560 buildmaster = self.botmaster.parent | |
| 561 status = buildmaster.getStatus() | |
| 562 text = "The Buildbot working for '%s'\n" % status.getProjectName() | |
| 563 text += ("has noticed that the latent buildslave named %s \n" % | |
| 564 self.slavename) | |
| 565 text += "never substantiated after a request\n" | |
| 566 text += "\n" | |
| 567 text += ("The request was made at %s (buildmaster-local time)\n" % | |
| 568 time.ctime(time.time() - self.missing_timeout)) # approx | |
| 569 text += "\n" | |
| 570 text += "Sincerely,\n" | |
| 571 text += " The Buildbot\n" | |
| 572 text += " %s\n" % status.getProjectURL() | |
| 573 subject = "Buildbot: buildslave %s never substantiated" % self.slavename | |
| 574 return self._mail_missing_message(subject, text) | |
| 575 | |
| 576 def buildStarted(self, sb): | |
| 577 assert self.substantiated | |
| 578 self._clearBuildWaitTimer() | |
| 579 self.building.add(sb.builder_name) | |
| 580 | |
| 581 def buildFinished(self, sb): | |
| 582 self.building.remove(sb.builder_name) | |
| 583 if not self.building: | |
| 584 self._setBuildWaitTimer() | |
| 585 | |
| 586 def _clearBuildWaitTimer(self): | |
| 587 if self.build_wait_timer is not None: | |
| 588 if self.build_wait_timer.active(): | |
| 589 self.build_wait_timer.cancel() | |
| 590 self.build_wait_timer = None | |
| 591 | |
| 592 def _setBuildWaitTimer(self): | |
| 593 self._clearBuildWaitTimer() | |
| 594 self.build_wait_timer = reactor.callLater( | |
| 595 self.build_wait_timeout, self._soft_disconnect) | |
| 596 | |
| 597 def insubstantiate(self, fast=False): | |
| 598 self._clearBuildWaitTimer() | |
| 599 d = self.stop_instance(fast) | |
| 600 if self._shutdown_callback_handle is not None: | |
| 601 handle = self._shutdown_callback_handle | |
| 602 del self._shutdown_callback_handle | |
| 603 reactor.removeSystemEventTrigger(handle) | |
| 604 self.substantiated = False | |
| 605 self.building.clear() # just to be sure | |
| 606 return d | |
| 607 | |
| 608 def _soft_disconnect(self, fast=False): | |
| 609 d = AbstractBuildSlave.disconnect(self) | |
| 610 if self.slave is not None: | |
| 611 # this could be called when the slave needs to shut down, such as | |
| 612 # in BotMaster.removeSlave, *or* when a new slave requests a | |
| 613 # connection when we already have a slave. It's not clear what to | |
| 614 # do in the second case: this shouldn't happen, and if it | |
| 615 # does...if it's a latent slave, shutting down will probably kill | |
| 616 # something we want...but we can't know what the status is. So, | |
| 617 # here, we just do what should be appropriate for the first case, | |
| 618 # and put our heads in the sand for the second, at least for now. | |
| 619 # The best solution to the odd situation is removing it as a | |
| 620 # possibilty: make the master in charge of connecting to the | |
| 621 # slave, rather than vice versa. TODO. | |
| 622 d = defer.DeferredList([d, self.insubstantiate(fast)]) | |
| 623 else: | |
| 624 if self.substantiation_deferred is not None: | |
| 625 # unlike the previous block, we don't expect this situation when | |
| 626 # ``attached`` calls ``disconnect``, only when we get a simple | |
| 627 # request to "go away". | |
| 628 self.substantiation_deferred.errback() | |
| 629 self.substantiation_deferred = None | |
| 630 if self.missing_timer: | |
| 631 self.missing_timer.cancel() | |
| 632 self.missing_timer = None | |
| 633 self.stop_instance() | |
| 634 return d | |
| 635 | |
| 636 def disconnect(self): | |
| 637 d = self._soft_disconnect() | |
| 638 # this removes the slave from all builders. It won't come back | |
| 639 # without a restart (or maybe a sighup) | |
| 640 self.botmaster.slaveLost(self) | |
| 641 | |
| 642 def stopService(self): | |
| 643 res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) | |
| 644 if self.slave is not None: | |
| 645 d = self._soft_disconnect() | |
| 646 res = defer.DeferredList([res, d]) | |
| 647 return res | |
| 648 | |
| 649 def updateSlave(self): | |
| 650 """Called to add or remove builders after the slave has connected. | |
| 651 | |
| 652 Also called after botmaster's builders are initially set. | |
| 653 | |
| 654 @return: a Deferred that indicates when an attached slave has | |
| 655 accepted the new builders and/or released the old ones.""" | |
| 656 for b in self.botmaster.getBuildersForSlave(self.slavename): | |
| 657 if b.name not in self.slavebuilders: | |
| 658 b.addLatentSlave(self) | |
| 659 return AbstractBuildSlave.updateSlave(self) | |
| 660 | |
| 661 def sendBuilderList(self): | |
| 662 d = AbstractBuildSlave.sendBuilderList(self) | |
| 663 def _sent(slist): | |
| 664 dl = [] | |
| 665 for name, remote in slist.items(): | |
| 666 # use get() since we might have changed our mind since then. | |
| 667 # we're checking on the builder in addition to the | |
| 668 # slavebuilders out of a bit of paranoia. | |
| 669 b = self.botmaster.builders.get(name) | |
| 670 sb = self.slavebuilders.get(name) | |
| 671 if b and sb: | |
| 672 d1 = sb.attached(self, remote, self.slave_commands) | |
| 673 dl.append(d1) | |
| 674 return defer.DeferredList(dl) | |
| 675 def _set_failed(why): | |
| 676 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) | |
| 677 log.err(why) | |
| 678 # TODO: hang up on them?, without setBuilderList we can't use | |
| 679 # them | |
| 680 if self.substantiation_deferred: | |
| 681 self.substantiation_deferred.errback() | |
| 682 self.substantiation_deferred = None | |
| 683 if self.missing_timer: | |
| 684 self.missing_timer.cancel() | |
| 685 self.missing_timer = None | |
| 686 # TODO: maybe log? send an email? | |
| 687 return why | |
| 688 d.addCallbacks(_sent, _set_failed) | |
| 689 def _substantiated(res): | |
| 690 self.substantiated = True | |
| 691 if self.substantiation_deferred: | |
| 692 d = self.substantiation_deferred | |
| 693 del self.substantiation_deferred | |
| 694 res = self._start_result | |
| 695 del self._start_result | |
| 696 d.callback(res) | |
| 697 # note that the missing_timer is already handled within | |
| 698 # ``attached`` | |
| 699 if not self.building: | |
| 700 self._setBuildWaitTimer() | |
| 701 d.addCallback(_substantiated) | |
| 702 return d | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build