| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- test-case-name: buildbot.test.test_run -*- | |
| 2 | |
| 3 import os | |
| 4 signal = None | |
| 5 try: | |
| 6 import signal | |
| 7 except ImportError: | |
| 8 pass | |
| 9 from cPickle import load | |
| 10 import warnings | |
| 11 | |
| 12 from zope.interface import implements | |
| 13 from twisted.python import log, components | |
| 14 from twisted.python.failure import Failure | |
| 15 from twisted.internet import defer, reactor | |
| 16 from twisted.spread import pb | |
| 17 from twisted.cred import portal, checkers | |
| 18 from twisted.application import service, strports | |
| 19 from twisted.persisted import styles | |
| 20 | |
| 21 import buildbot | |
| 22 # sibling imports | |
| 23 from buildbot.util import now, safeTranslate | |
| 24 from buildbot.pbutil import NewCredPerspective | |
| 25 from buildbot.process.builder import Builder, IDLE | |
| 26 from buildbot.process.base import BuildRequest | |
| 27 from buildbot.status.builder import Status | |
| 28 from buildbot.changes.changes import Change, ChangeMaster, TestChangeMaster | |
| 29 from buildbot.sourcestamp import SourceStamp | |
| 30 from buildbot.buildslave import BuildSlave | |
| 31 from buildbot import interfaces, locks | |
| 32 from buildbot.process.properties import Properties | |
| 33 from buildbot.config import BuilderConfig | |
| 34 | |
| 35 ######################################## | |
| 36 | |
| 37 class BotMaster(service.MultiService): | |
| 38 | |
| 39 """This is the master-side service which manages remote buildbot slaves. | |
| 40 It provides them with BuildSlaves, and distributes file change | |
| 41 notification messages to them. | |
| 42 """ | |
| 43 | |
| 44 debug = 0 | |
| 45 | |
| 46 def __init__(self): | |
| 47 service.MultiService.__init__(self) | |
| 48 self.builders = {} | |
| 49 self.builderNames = [] | |
| 50 # builders maps Builder names to instances of bb.p.builder.Builder, | |
| 51 # which is the master-side object that defines and controls a build. | |
| 52 # They are added by calling botmaster.addBuilder() from the startup | |
| 53 # code. | |
| 54 | |
| 55 # self.slaves contains a ready BuildSlave instance for each | |
| 56 # potential buildslave, i.e. all the ones listed in the config file. | |
| 57 # If the slave is connected, self.slaves[slavename].slave will | |
| 58 # contain a RemoteReference to their Bot instance. If it is not | |
| 59 # connected, that attribute will hold None. | |
| 60 self.slaves = {} # maps slavename to BuildSlave | |
| 61 self.statusClientService = None | |
| 62 self.watchers = {} | |
| 63 | |
| 64 # self.locks holds the real Lock instances | |
| 65 self.locks = {} | |
| 66 | |
| 67 # self.mergeRequests is the callable override for merging build | |
| 68 # requests | |
| 69 self.mergeRequests = None | |
| 70 | |
| 71 # self.prioritizeBuilders is the callable override for builder order | |
| 72 # traversal | |
| 73 self.prioritizeBuilders = None | |
| 74 | |
| 75 # these four are convenience functions for testing | |
| 76 | |
| 77 def waitUntilBuilderAttached(self, name): | |
| 78 b = self.builders[name] | |
| 79 #if b.slaves: | |
| 80 # return defer.succeed(None) | |
| 81 d = defer.Deferred() | |
| 82 b.watchers['attach'].append(d) | |
| 83 return d | |
| 84 | |
| 85 def waitUntilBuilderDetached(self, name): | |
| 86 b = self.builders.get(name) | |
| 87 if not b or not b.slaves: | |
| 88 return defer.succeed(None) | |
| 89 d = defer.Deferred() | |
| 90 b.watchers['detach'].append(d) | |
| 91 return d | |
| 92 | |
| 93 def waitUntilBuilderFullyDetached(self, name): | |
| 94 b = self.builders.get(name) | |
| 95 # TODO: this looks too deeply inside the Builder object | |
| 96 if not b or not b.slaves: | |
| 97 return defer.succeed(None) | |
| 98 d = defer.Deferred() | |
| 99 b.watchers['detach_all'].append(d) | |
| 100 return d | |
| 101 | |
| 102 def waitUntilBuilderIdle(self, name): | |
| 103 b = self.builders[name] | |
| 104 # TODO: this looks way too deeply inside the Builder object | |
| 105 for sb in b.slaves: | |
| 106 if sb.state != IDLE: | |
| 107 d = defer.Deferred() | |
| 108 b.watchers['idle'].append(d) | |
| 109 return d | |
| 110 return defer.succeed(None) | |
| 111 | |
| 112 def loadConfig_Slaves(self, new_slaves): | |
| 113 old_slaves = [c for c in list(self) | |
| 114 if interfaces.IBuildSlave.providedBy(c)] | |
| 115 | |
| 116 # identify added/removed slaves. For each slave we construct a tuple | |
| 117 # of (name, password, class), and we consider the slave to be already | |
| 118 # present if the tuples match. (we include the class to make sure | |
| 119 # that BuildSlave(name,pw) is different than | |
| 120 # SubclassOfBuildSlave(name,pw) ). If the password or class has | |
| 121 # changed, we will remove the old version of the slave and replace it | |
| 122 # with a new one. If anything else has changed, we just update the | |
| 123 # old BuildSlave instance in place. If the name has changed, of | |
| 124 # course, it looks exactly the same as deleting one slave and adding | |
| 125 # an unrelated one. | |
| 126 old_t = {} | |
| 127 for s in old_slaves: | |
| 128 old_t[(s.slavename, s.password, s.__class__)] = s | |
| 129 new_t = {} | |
| 130 for s in new_slaves: | |
| 131 new_t[(s.slavename, s.password, s.__class__)] = s | |
| 132 removed = [old_t[t] | |
| 133 for t in old_t | |
| 134 if t not in new_t] | |
| 135 added = [new_t[t] | |
| 136 for t in new_t | |
| 137 if t not in old_t] | |
| 138 remaining_t = [t | |
| 139 for t in new_t | |
| 140 if t in old_t] | |
| 141 # removeSlave will hang up on the old bot | |
| 142 dl = [] | |
| 143 for s in removed: | |
| 144 dl.append(self.removeSlave(s)) | |
| 145 d = defer.DeferredList(dl, fireOnOneErrback=True) | |
| 146 def _add(res): | |
| 147 for s in added: | |
| 148 self.addSlave(s) | |
| 149 for t in remaining_t: | |
| 150 old_t[t].update(new_t[t]) | |
| 151 d.addCallback(_add) | |
| 152 return d | |
| 153 | |
| 154 def addSlave(self, s): | |
| 155 s.setServiceParent(self) | |
| 156 s.setBotmaster(self) | |
| 157 self.slaves[s.slavename] = s | |
| 158 | |
| 159 def removeSlave(self, s): | |
| 160 # TODO: technically, disownServiceParent could return a Deferred | |
| 161 s.disownServiceParent() | |
| 162 d = self.slaves[s.slavename].disconnect() | |
| 163 del self.slaves[s.slavename] | |
| 164 return d | |
| 165 | |
| 166 def slaveLost(self, bot): | |
| 167 for name, b in self.builders.items(): | |
| 168 if bot.slavename in b.slavenames: | |
| 169 b.detached(bot) | |
| 170 | |
| 171 def getBuildersForSlave(self, slavename): | |
| 172 return [b | |
| 173 for b in self.builders.values() | |
| 174 if slavename in b.slavenames] | |
| 175 | |
| 176 def getBuildernames(self): | |
| 177 return self.builderNames | |
| 178 | |
| 179 def getBuilders(self): | |
| 180 allBuilders = [self.builders[name] for name in self.builderNames] | |
| 181 return allBuilders | |
| 182 | |
| 183 def setBuilders(self, builders): | |
| 184 self.builders = {} | |
| 185 self.builderNames = [] | |
| 186 for b in builders: | |
| 187 for slavename in b.slavenames: | |
| 188 # this is actually validated earlier | |
| 189 assert slavename in self.slaves | |
| 190 self.builders[b.name] = b | |
| 191 self.builderNames.append(b.name) | |
| 192 b.setBotmaster(self) | |
| 193 d = self._updateAllSlaves() | |
| 194 return d | |
| 195 | |
| 196 def _updateAllSlaves(self): | |
| 197 """Notify all buildslaves about changes in their Builders.""" | |
| 198 dl = [s.updateSlave() for s in self.slaves.values()] | |
| 199 return defer.DeferredList(dl) | |
| 200 | |
| 201 def maybeStartAllBuilds(self): | |
| 202 builders = self.builders.values() | |
| 203 if self.prioritizeBuilders is not None: | |
| 204 try: | |
| 205 builders = self.prioritizeBuilders(self.parent, builders) | |
| 206 except: | |
| 207 log.msg("Exception prioritizing builders") | |
| 208 log.err(Failure()) | |
| 209 return | |
| 210 else: | |
| 211 def _sortfunc(b1, b2): | |
| 212 t1 = b1.getOldestRequestTime() | |
| 213 t2 = b2.getOldestRequestTime() | |
| 214 # If t1 or t2 is None, then there are no build requests, | |
| 215 # so sort it at the end | |
| 216 if t1 is None: | |
| 217 return 1 | |
| 218 if t2 is None: | |
| 219 return -1 | |
| 220 return cmp(t1, t2) | |
| 221 builders.sort(_sortfunc) | |
| 222 try: | |
| 223 for b in builders: | |
| 224 b.maybeStartBuild() | |
| 225 except: | |
| 226 log.msg("Exception starting builds") | |
| 227 log.err(Failure()) | |
| 228 | |
| 229 def shouldMergeRequests(self, builder, req1, req2): | |
| 230 """Determine whether two BuildRequests should be merged for | |
| 231 the given builder. | |
| 232 | |
| 233 """ | |
| 234 if self.mergeRequests is not None: | |
| 235 return self.mergeRequests(builder, req1, req2) | |
| 236 return req1.canBeMergedWith(req2) | |
| 237 | |
| 238 def getPerspective(self, slavename): | |
| 239 return self.slaves[slavename] | |
| 240 | |
| 241 def shutdownSlaves(self): | |
| 242 # TODO: make this into a bot method rather than a builder method | |
| 243 for b in self.slaves.values(): | |
| 244 b.shutdownSlave() | |
| 245 | |
| 246 def stopService(self): | |
| 247 for b in self.builders.values(): | |
| 248 b.builder_status.addPointEvent(["master", "shutdown"]) | |
| 249 b.builder_status.saveYourself() | |
| 250 return service.Service.stopService(self) | |
| 251 | |
| 252 def getLockByID(self, lockid): | |
| 253 """Convert a Lock identifier into an actual Lock instance. | |
| 254 @param lockid: a locks.MasterLock or locks.SlaveLock instance | |
| 255 @return: a locks.RealMasterLock or locks.RealSlaveLock instance | |
| 256 """ | |
| 257 assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) | |
| 258 if not lockid in self.locks: | |
| 259 self.locks[lockid] = lockid.lockClass(lockid) | |
| 260 # if the master.cfg file has changed maxCount= on the lock, the next | |
| 261 # time a build is started, they'll get a new RealLock instance. Note | |
| 262 # that this requires that MasterLock and SlaveLock (marker) instances | |
| 263 # be hashable and that they should compare properly. | |
| 264 return self.locks[lockid] | |
| 265 | |
| 266 ######################################## | |
| 267 | |
| 268 | |
| 269 | |
| 270 class DebugPerspective(NewCredPerspective): | |
| 271 def attached(self, mind): | |
| 272 return self | |
| 273 def detached(self, mind): | |
| 274 pass | |
| 275 | |
| 276 def perspective_requestBuild(self, buildername, reason, branch, revision, pr
operties={}): | |
| 277 c = interfaces.IControl(self.master) | |
| 278 bc = c.getBuilder(buildername) | |
| 279 ss = SourceStamp(branch, revision) | |
| 280 bpr = Properties() | |
| 281 bpr.update(properties, "remote requestBuild") | |
| 282 br = BuildRequest(reason, ss, builderName=buildername, properties=bpr) | |
| 283 bc.requestBuild(br) | |
| 284 | |
| 285 def perspective_pingBuilder(self, buildername): | |
| 286 c = interfaces.IControl(self.master) | |
| 287 bc = c.getBuilder(buildername) | |
| 288 bc.ping() | |
| 289 | |
| 290 def perspective_fakeChange(self, file, revision=None, who="fakeUser", | |
| 291 branch=None): | |
| 292 change = Change(who, [file], "some fake comments\n", | |
| 293 branch=branch, revision=revision) | |
| 294 c = interfaces.IControl(self.master) | |
| 295 c.addChange(change) | |
| 296 | |
| 297 def perspective_setCurrentState(self, buildername, state): | |
| 298 builder = self.botmaster.builders.get(buildername) | |
| 299 if not builder: return | |
| 300 if state == "offline": | |
| 301 builder.statusbag.currentlyOffline() | |
| 302 if state == "idle": | |
| 303 builder.statusbag.currentlyIdle() | |
| 304 if state == "waiting": | |
| 305 builder.statusbag.currentlyWaiting(now()+10) | |
| 306 if state == "building": | |
| 307 builder.statusbag.currentlyBuilding(None) | |
| 308 def perspective_reload(self): | |
| 309 print "doing reload of the config file" | |
| 310 self.master.loadTheConfigFile() | |
| 311 def perspective_pokeIRC(self): | |
| 312 print "saying something on IRC" | |
| 313 from buildbot.status import words | |
| 314 for s in self.master: | |
| 315 if isinstance(s, words.IRC): | |
| 316 bot = s.f | |
| 317 for channel in bot.channels: | |
| 318 print " channel", channel | |
| 319 bot.p.msg(channel, "Ow, quit it") | |
| 320 | |
| 321 def perspective_print(self, msg): | |
| 322 print "debug", msg | |
| 323 | |
| 324 class Dispatcher: | |
| 325 implements(portal.IRealm) | |
| 326 | |
| 327 def __init__(self): | |
| 328 self.names = {} | |
| 329 | |
| 330 def register(self, name, afactory): | |
| 331 self.names[name] = afactory | |
| 332 def unregister(self, name): | |
| 333 del self.names[name] | |
| 334 | |
| 335 def requestAvatar(self, avatarID, mind, interface): | |
| 336 assert interface == pb.IPerspective | |
| 337 afactory = self.names.get(avatarID) | |
| 338 if afactory: | |
| 339 p = afactory.getPerspective() | |
| 340 elif avatarID == "debug": | |
| 341 p = DebugPerspective() | |
| 342 p.master = self.master | |
| 343 p.botmaster = self.botmaster | |
| 344 elif avatarID == "statusClient": | |
| 345 p = self.statusClientService.getPerspective() | |
| 346 else: | |
| 347 # it must be one of the buildslaves: no other names will make it | |
| 348 # past the checker | |
| 349 p = self.botmaster.getPerspective(avatarID) | |
| 350 | |
| 351 if not p: | |
| 352 raise ValueError("no perspective for '%s'" % avatarID) | |
| 353 | |
| 354 d = defer.maybeDeferred(p.attached, mind) | |
| 355 d.addCallback(self._avatarAttached, mind) | |
| 356 return d | |
| 357 | |
| 358 def _avatarAttached(self, p, mind): | |
| 359 return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind)) | |
| 360 | |
| 361 ######################################## | |
| 362 | |
| 363 # service hierarchy: | |
| 364 # BuildMaster | |
| 365 # BotMaster | |
| 366 # ChangeMaster | |
| 367 # all IChangeSource objects | |
| 368 # StatusClientService | |
| 369 # TCPClient(self.ircFactory) | |
| 370 # TCPServer(self.slaveFactory) -> dispatcher.requestAvatar | |
| 371 # TCPServer(self.site) | |
| 372 # UNIXServer(ResourcePublisher(self.site)) | |
| 373 | |
| 374 | |
| 375 class BuildMaster(service.MultiService): | |
| 376 debug = 0 | |
| 377 manhole = None | |
| 378 debugPassword = None | |
| 379 projectName = "(unspecified)" | |
| 380 projectURL = None | |
| 381 buildbotURL = None | |
| 382 change_svc = None | |
| 383 properties = Properties() | |
| 384 | |
| 385 def __init__(self, basedir, configFileName="master.cfg"): | |
| 386 service.MultiService.__init__(self) | |
| 387 self.setName("buildmaster") | |
| 388 self.basedir = basedir | |
| 389 self.configFileName = configFileName | |
| 390 | |
| 391 # the dispatcher is the realm in which all inbound connections are | |
| 392 # looked up: slave builders, change notifications, status clients, and | |
| 393 # the debug port | |
| 394 dispatcher = Dispatcher() | |
| 395 dispatcher.master = self | |
| 396 self.dispatcher = dispatcher | |
| 397 self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() | |
| 398 # the checker starts with no user/passwd pairs: they are added later | |
| 399 p = portal.Portal(dispatcher) | |
| 400 p.registerChecker(self.checker) | |
| 401 self.slaveFactory = pb.PBServerFactory(p) | |
| 402 self.slaveFactory.unsafeTracebacks = True # let them see exceptions | |
| 403 | |
| 404 self.slavePortnum = None | |
| 405 self.slavePort = None | |
| 406 | |
| 407 self.botmaster = BotMaster() | |
| 408 self.botmaster.setName("botmaster") | |
| 409 self.botmaster.setServiceParent(self) | |
| 410 dispatcher.botmaster = self.botmaster | |
| 411 | |
| 412 self.status = Status(self.botmaster, self.basedir) | |
| 413 | |
| 414 self.statusTargets = [] | |
| 415 | |
| 416 # this ChangeMaster is a dummy, only used by tests. In the real | |
| 417 # buildmaster, where the BuildMaster instance is activated | |
| 418 # (startService is called) by twistd, this attribute is overwritten. | |
| 419 self.useChanges(TestChangeMaster()) | |
| 420 | |
| 421 self.readConfig = False | |
| 422 | |
| 423 def startService(self): | |
| 424 service.MultiService.startService(self) | |
| 425 self.loadChanges() # must be done before loading the config file | |
| 426 if not self.readConfig: | |
| 427 # TODO: consider catching exceptions during this call to | |
| 428 # loadTheConfigFile and bailing (reactor.stop) if it fails, | |
| 429 # since without a config file we can't do anything except reload | |
| 430 # the config file, and it would be nice for the user to discover | |
| 431 # this quickly. | |
| 432 self.loadTheConfigFile() | |
| 433 if signal and hasattr(signal, "SIGHUP"): | |
| 434 signal.signal(signal.SIGHUP, self._handleSIGHUP) | |
| 435 for b in self.botmaster.builders.values(): | |
| 436 b.builder_status.addPointEvent(["master", "started"]) | |
| 437 b.builder_status.saveYourself() | |
| 438 | |
| 439 def useChanges(self, changes): | |
| 440 if self.change_svc: | |
| 441 # TODO: can return a Deferred | |
| 442 self.change_svc.disownServiceParent() | |
| 443 self.change_svc = changes | |
| 444 self.change_svc.basedir = self.basedir | |
| 445 self.change_svc.setName("changemaster") | |
| 446 self.dispatcher.changemaster = self.change_svc | |
| 447 self.change_svc.setServiceParent(self) | |
| 448 | |
| 449 def loadChanges(self): | |
| 450 filename = os.path.join(self.basedir, "changes.pck") | |
| 451 try: | |
| 452 changes = load(open(filename, "rb")) | |
| 453 styles.doUpgrade() | |
| 454 except IOError: | |
| 455 log.msg("changes.pck missing, using new one") | |
| 456 changes = ChangeMaster() | |
| 457 except EOFError: | |
| 458 log.msg("corrupted changes.pck, using new one") | |
| 459 changes = ChangeMaster() | |
| 460 self.useChanges(changes) | |
| 461 | |
| 462 def _handleSIGHUP(self, *args): | |
| 463 reactor.callLater(0, self.loadTheConfigFile) | |
| 464 | |
| 465 def getStatus(self): | |
| 466 """ | |
| 467 @rtype: L{buildbot.status.builder.Status} | |
| 468 """ | |
| 469 return self.status | |
| 470 | |
| 471 def loadTheConfigFile(self, configFile=None): | |
| 472 if not configFile: | |
| 473 configFile = os.path.join(self.basedir, self.configFileName) | |
| 474 | |
| 475 log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.versio
n) | |
| 476 log.msg("loading configuration from %s" % configFile) | |
| 477 configFile = os.path.expanduser(configFile) | |
| 478 | |
| 479 try: | |
| 480 f = open(configFile, "r") | |
| 481 except IOError, e: | |
| 482 log.msg("unable to open config file '%s'" % configFile) | |
| 483 log.msg("leaving old configuration in place") | |
| 484 log.err(e) | |
| 485 return | |
| 486 | |
| 487 try: | |
| 488 self.loadConfig(f) | |
| 489 except: | |
| 490 log.msg("error during loadConfig") | |
| 491 log.err() | |
| 492 log.msg("The new config file is unusable, so I'll ignore it.") | |
| 493 log.msg("I will keep using the previous config file instead.") | |
| 494 f.close() | |
| 495 | |
| 496 def loadConfig(self, f): | |
| 497 """Internal function to load a specific configuration file. Any | |
| 498 errors in the file will be signalled by raising an exception. | |
| 499 | |
| 500 @return: a Deferred that will fire (with None) when the configuration | |
| 501 changes have been completed. This may involve a round-trip to each | |
| 502 buildslave that was involved.""" | |
| 503 | |
| 504 localDict = {'basedir': os.path.expanduser(self.basedir)} | |
| 505 try: | |
| 506 exec f in localDict | |
| 507 except: | |
| 508 log.msg("error while parsing config file") | |
| 509 raise | |
| 510 | |
| 511 try: | |
| 512 config = localDict['BuildmasterConfig'] | |
| 513 except KeyError: | |
| 514 log.err("missing config dictionary") | |
| 515 log.err("config file must define BuildmasterConfig") | |
| 516 raise | |
| 517 | |
| 518 known_keys = ("bots", "slaves", | |
| 519 "sources", "change_source", | |
| 520 "schedulers", "builders", "mergeRequests", | |
| 521 "slavePortnum", "debugPassword", "logCompressionLimit", | |
| 522 "manhole", "status", "projectName", "projectURL", | |
| 523 "buildbotURL", "properties", "prioritizeBuilders", | |
| 524 "eventHorizon", "buildCacheSize", "logHorizon", "buildHori
zon", | |
| 525 "changeHorizon", "logMaxSize", "logMaxTailSize", | |
| 526 "logCompressionMethod", | |
| 527 ) | |
| 528 for k in config.keys(): | |
| 529 if k not in known_keys: | |
| 530 log.msg("unknown key '%s' defined in config dictionary" % k) | |
| 531 | |
| 532 try: | |
| 533 # required | |
| 534 schedulers = config['schedulers'] | |
| 535 builders = config['builders'] | |
| 536 slavePortnum = config['slavePortnum'] | |
| 537 #slaves = config['slaves'] | |
| 538 #change_source = config['change_source'] | |
| 539 | |
| 540 # optional | |
| 541 debugPassword = config.get('debugPassword') | |
| 542 manhole = config.get('manhole') | |
| 543 status = config.get('status', []) | |
| 544 projectName = config.get('projectName') | |
| 545 projectURL = config.get('projectURL') | |
| 546 buildbotURL = config.get('buildbotURL') | |
| 547 properties = config.get('properties', {}) | |
| 548 buildCacheSize = config.get('buildCacheSize', None) | |
| 549 eventHorizon = config.get('eventHorizon', None) | |
| 550 logHorizon = config.get('logHorizon', None) | |
| 551 buildHorizon = config.get('buildHorizon', None) | |
| 552 logCompressionLimit = config.get('logCompressionLimit', 4*1024) | |
| 553 if logCompressionLimit is not None and not \ | |
| 554 isinstance(logCompressionLimit, int): | |
| 555 raise ValueError("logCompressionLimit needs to be bool or int") | |
| 556 logCompressionMethod = config.get('logCompressionMethod', "bz2") | |
| 557 if logCompressionMethod not in ('bz2', 'gz'): | |
| 558 raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz
'") | |
| 559 logMaxSize = config.get('logMaxSize') | |
| 560 if logMaxSize is not None and not \ | |
| 561 isinstance(logMaxSize, int): | |
| 562 raise ValueError("logMaxSize needs to be None or int") | |
| 563 logMaxTailSize = config.get('logMaxTailSize') | |
| 564 if logMaxTailSize is not None and not \ | |
| 565 isinstance(logMaxTailSize, int): | |
| 566 raise ValueError("logMaxTailSize needs to be None or int") | |
| 567 mergeRequests = config.get('mergeRequests') | |
| 568 if mergeRequests is not None and not callable(mergeRequests): | |
| 569 raise ValueError("mergeRequests must be a callable") | |
| 570 prioritizeBuilders = config.get('prioritizeBuilders') | |
| 571 if prioritizeBuilders is not None and not callable(prioritizeBuilder
s): | |
| 572 raise ValueError("prioritizeBuilders must be callable") | |
| 573 changeHorizon = config.get("changeHorizon") | |
| 574 if changeHorizon is not None and not isinstance(changeHorizon, int): | |
| 575 raise ValueError("changeHorizon needs to be an int") | |
| 576 | |
| 577 except KeyError, e: | |
| 578 log.msg("config dictionary is missing a required parameter") | |
| 579 log.msg("leaving old configuration in place") | |
| 580 raise | |
| 581 | |
| 582 #if "bots" in config: | |
| 583 # raise KeyError("c['bots'] is no longer accepted") | |
| 584 | |
| 585 slaves = config.get('slaves', []) | |
| 586 if "bots" in config: | |
| 587 m = ("c['bots'] is deprecated as of 0.7.6 and will be " | |
| 588 "removed by 0.8.0 . Please use c['slaves'] instead.") | |
| 589 log.msg(m) | |
| 590 warnings.warn(m, DeprecationWarning) | |
| 591 for name, passwd in config['bots']: | |
| 592 slaves.append(BuildSlave(name, passwd)) | |
| 593 | |
| 594 if "bots" not in config and "slaves" not in config: | |
| 595 log.msg("config dictionary must have either 'bots' or 'slaves'") | |
| 596 log.msg("leaving old configuration in place") | |
| 597 raise KeyError("must have either 'bots' or 'slaves'") | |
| 598 | |
| 599 #if "sources" in config: | |
| 600 # raise KeyError("c['sources'] is no longer accepted") | |
| 601 | |
| 602 if changeHorizon is not None: | |
| 603 self.change_svc.changeHorizon = changeHorizon | |
| 604 | |
| 605 change_source = config.get('change_source', []) | |
| 606 if isinstance(change_source, (list, tuple)): | |
| 607 change_sources = change_source | |
| 608 else: | |
| 609 change_sources = [change_source] | |
| 610 if "sources" in config: | |
| 611 m = ("c['sources'] is deprecated as of 0.7.6 and will be " | |
| 612 "removed by 0.8.0 . Please use c['change_source'] instead.") | |
| 613 log.msg(m) | |
| 614 warnings.warn(m, DeprecationWarning) | |
| 615 for s in config['sources']: | |
| 616 change_sources.append(s) | |
| 617 | |
| 618 # do some validation first | |
| 619 for s in slaves: | |
| 620 assert interfaces.IBuildSlave.providedBy(s) | |
| 621 if s.slavename in ("debug", "change", "status"): | |
| 622 raise KeyError( | |
| 623 "reserved name '%s' used for a bot" % s.slavename) | |
| 624 if config.has_key('interlocks'): | |
| 625 raise KeyError("c['interlocks'] is no longer accepted") | |
| 626 | |
| 627 assert isinstance(change_sources, (list, tuple)) | |
| 628 for s in change_sources: | |
| 629 assert interfaces.IChangeSource(s, None) | |
| 630 # this assertion catches c['schedulers'] = Scheduler(), since | |
| 631 # Schedulers are service.MultiServices and thus iterable. | |
| 632 errmsg = "c['schedulers'] must be a list of Scheduler instances" | |
| 633 assert isinstance(schedulers, (list, tuple)), errmsg | |
| 634 for s in schedulers: | |
| 635 assert interfaces.IScheduler(s, None), errmsg | |
| 636 assert isinstance(status, (list, tuple)) | |
| 637 for s in status: | |
| 638 assert interfaces.IStatusReceiver(s, None) | |
| 639 | |
| 640 slavenames = [s.slavename for s in slaves] | |
| 641 buildernames = [] | |
| 642 dirnames = [] | |
| 643 | |
| 644 # convert builders from objects to config dictionaries | |
| 645 builders_dicts = [] | |
| 646 for b in builders: | |
| 647 if isinstance(b, BuilderConfig): | |
| 648 builders_dicts.append(b.getConfigDict()) | |
| 649 elif type(b) is dict: | |
| 650 builders_dicts.append(b) | |
| 651 else: | |
| 652 raise ValueError("builder %s is not a BuilderConfig object (or a
dict)" % b) | |
| 653 builders = builders_dicts | |
| 654 | |
| 655 for b in builders: | |
| 656 if b.has_key('slavename') and b['slavename'] not in slavenames: | |
| 657 raise ValueError("builder %s uses undefined slave %s" \ | |
| 658 % (b['name'], b['slavename'])) | |
| 659 for n in b.get('slavenames', []): | |
| 660 if n not in slavenames: | |
| 661 raise ValueError("builder %s uses undefined slave %s" \ | |
| 662 % (b['name'], n)) | |
| 663 if b['name'] in buildernames: | |
| 664 raise ValueError("duplicate builder name %s" | |
| 665 % b['name']) | |
| 666 buildernames.append(b['name']) | |
| 667 | |
| 668 # sanity check name (BuilderConfig does this too) | |
| 669 if b['name'].startswith("_"): | |
| 670 errmsg = ("builder names must not start with an " | |
| 671 "underscore: " + b['name']) | |
| 672 log.err(errmsg) | |
| 673 raise ValueError(errmsg) | |
| 674 | |
| 675 # Fix the dictionnary with default values, in case this wasn't | |
| 676 # specified with a BuilderConfig object (which sets the same default
s) | |
| 677 b.setdefault('builddir', safeTranslate(b['name'])) | |
| 678 b.setdefault('slavebuilddir', b['builddir']) | |
| 679 | |
| 680 if b['builddir'] in dirnames: | |
| 681 raise ValueError("builder %s reuses builddir %s" | |
| 682 % (b['name'], b['builddir'])) | |
| 683 dirnames.append(b['builddir']) | |
| 684 | |
| 685 unscheduled_buildernames = buildernames[:] | |
| 686 schedulernames = [] | |
| 687 for s in schedulers: | |
| 688 for b in s.listBuilderNames(): | |
| 689 assert b in buildernames, \ | |
| 690 "%s uses unknown builder %s" % (s, b) | |
| 691 if b in unscheduled_buildernames: | |
| 692 unscheduled_buildernames.remove(b) | |
| 693 | |
| 694 if s.name in schedulernames: | |
| 695 # TODO: schedulers share a namespace with other Service | |
| 696 # children of the BuildMaster node, like status plugins, the | |
| 697 # Manhole, the ChangeMaster, and the BotMaster (although most | |
| 698 # of these don't have names) | |
| 699 msg = ("Schedulers must have unique names, but " | |
| 700 "'%s' was a duplicate" % (s.name,)) | |
| 701 raise ValueError(msg) | |
| 702 schedulernames.append(s.name) | |
| 703 | |
| 704 if unscheduled_buildernames: | |
| 705 log.msg("Warning: some Builders have no Schedulers to drive them:" | |
| 706 " %s" % (unscheduled_buildernames,)) | |
| 707 | |
| 708 # assert that all locks used by the Builds and their Steps are | |
| 709 # uniquely named. | |
| 710 lock_dict = {} | |
| 711 for b in builders: | |
| 712 for l in b.get('locks', []): | |
| 713 if isinstance(l, locks.LockAccess): # User specified access to t
he lock | |
| 714 l = l.lockid | |
| 715 if lock_dict.has_key(l.name): | |
| 716 if lock_dict[l.name] is not l: | |
| 717 raise ValueError("Two different locks (%s and %s) " | |
| 718 "share the name %s" | |
| 719 % (l, lock_dict[l.name], l.name)) | |
| 720 else: | |
| 721 lock_dict[l.name] = l | |
| 722 # TODO: this will break with any BuildFactory that doesn't use a | |
| 723 # .steps list, but I think the verification step is more | |
| 724 # important. | |
| 725 for s in b['factory'].steps: | |
| 726 for l in s[1].get('locks', []): | |
| 727 if isinstance(l, locks.LockAccess): # User specified access
to the lock | |
| 728 l = l.lockid | |
| 729 if lock_dict.has_key(l.name): | |
| 730 if lock_dict[l.name] is not l: | |
| 731 raise ValueError("Two different locks (%s and %s)" | |
| 732 " share the name %s" | |
| 733 % (l, lock_dict[l.name], l.name)) | |
| 734 else: | |
| 735 lock_dict[l.name] = l | |
| 736 | |
| 737 if not isinstance(properties, dict): | |
| 738 raise ValueError("c['properties'] must be a dictionary") | |
| 739 | |
| 740 # slavePortnum supposed to be a strports specification | |
| 741 if type(slavePortnum) is int: | |
| 742 slavePortnum = "tcp:%d" % slavePortnum | |
| 743 | |
| 744 # now we're committed to implementing the new configuration, so do | |
| 745 # it atomically | |
| 746 # TODO: actually, this is spread across a couple of Deferreds, so it | |
| 747 # really isn't atomic. | |
| 748 | |
| 749 d = defer.succeed(None) | |
| 750 | |
| 751 self.projectName = projectName | |
| 752 self.projectURL = projectURL | |
| 753 self.buildbotURL = buildbotURL | |
| 754 | |
| 755 self.properties = Properties() | |
| 756 self.properties.update(properties, self.configFileName) | |
| 757 | |
| 758 self.status.logCompressionLimit = logCompressionLimit | |
| 759 self.status.logCompressionMethod = logCompressionMethod | |
| 760 self.status.logMaxSize = logMaxSize | |
| 761 self.status.logMaxTailSize = logMaxTailSize | |
| 762 # Update any of our existing builders with the current log parameters. | |
| 763 # This is required so that the new value is picked up after a | |
| 764 # reconfig. | |
| 765 for builder in self.botmaster.builders.values(): | |
| 766 builder.builder_status.setLogCompressionLimit(logCompressionLimit) | |
| 767 builder.builder_status.setLogCompressionMethod(logCompressionMethod) | |
| 768 builder.builder_status.setLogMaxSize(logMaxSize) | |
| 769 builder.builder_status.setLogMaxTailSize(logMaxTailSize) | |
| 770 | |
| 771 if mergeRequests is not None: | |
| 772 self.botmaster.mergeRequests = mergeRequests | |
| 773 if prioritizeBuilders is not None: | |
| 774 self.botmaster.prioritizeBuilders = prioritizeBuilders | |
| 775 | |
| 776 self.buildCacheSize = buildCacheSize | |
| 777 self.eventHorizon = eventHorizon | |
| 778 self.logHorizon = logHorizon | |
| 779 self.buildHorizon = buildHorizon | |
| 780 | |
| 781 # self.slaves: Disconnect any that were attached and removed from the | |
| 782 # list. Update self.checker with the new list of passwords, including | |
| 783 # debug/change/status. | |
| 784 d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) | |
| 785 | |
| 786 # self.debugPassword | |
| 787 if debugPassword: | |
| 788 self.checker.addUser("debug", debugPassword) | |
| 789 self.debugPassword = debugPassword | |
| 790 | |
| 791 # self.manhole | |
| 792 if manhole != self.manhole: | |
| 793 # changing | |
| 794 if self.manhole: | |
| 795 # disownServiceParent may return a Deferred | |
| 796 d.addCallback(lambda res: self.manhole.disownServiceParent()) | |
| 797 def _remove(res): | |
| 798 self.manhole = None | |
| 799 return res | |
| 800 d.addCallback(_remove) | |
| 801 if manhole: | |
| 802 def _add(res): | |
| 803 self.manhole = manhole | |
| 804 manhole.setServiceParent(self) | |
| 805 d.addCallback(_add) | |
| 806 | |
| 807 # add/remove self.botmaster.builders to match builders. The | |
| 808 # botmaster will handle startup/shutdown issues. | |
| 809 d.addCallback(lambda res: self.loadConfig_Builders(builders)) | |
| 810 | |
| 811 d.addCallback(lambda res: self.loadConfig_status(status)) | |
| 812 | |
| 813 # Schedulers are added after Builders in case they start right away | |
| 814 d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers)) | |
| 815 # and Sources go after Schedulers for the same reason | |
| 816 d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) | |
| 817 | |
| 818 # self.slavePort | |
| 819 if self.slavePortnum != slavePortnum: | |
| 820 if self.slavePort: | |
| 821 def closeSlavePort(res): | |
| 822 d1 = self.slavePort.disownServiceParent() | |
| 823 self.slavePort = None | |
| 824 return d1 | |
| 825 d.addCallback(closeSlavePort) | |
| 826 if slavePortnum is not None: | |
| 827 def openSlavePort(res): | |
| 828 self.slavePort = strports.service(slavePortnum, | |
| 829 self.slaveFactory) | |
| 830 self.slavePort.setServiceParent(self) | |
| 831 d.addCallback(openSlavePort) | |
| 832 log.msg("BuildMaster listening on port %s" % slavePortnum) | |
| 833 self.slavePortnum = slavePortnum | |
| 834 | |
| 835 log.msg("configuration update started") | |
| 836 def _done(res): | |
| 837 self.readConfig = True | |
| 838 log.msg("configuration update complete") | |
| 839 d.addCallback(_done) | |
| 840 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) | |
| 841 return d | |
| 842 | |
| 843 def loadConfig_Slaves(self, new_slaves): | |
| 844 # set up the Checker with the names and passwords of all valid bots | |
| 845 self.checker.users = {} # violates abstraction, oh well | |
| 846 for s in new_slaves: | |
| 847 self.checker.addUser(s.slavename, s.password) | |
| 848 self.checker.addUser("change", "changepw") | |
| 849 # let the BotMaster take care of the rest | |
| 850 return self.botmaster.loadConfig_Slaves(new_slaves) | |
| 851 | |
| 852 def loadConfig_Sources(self, sources): | |
| 853 if not sources: | |
| 854 log.msg("warning: no ChangeSources specified in c['change_source']") | |
| 855 # shut down any that were removed, start any that were added | |
| 856 deleted_sources = [s for s in self.change_svc if s not in sources] | |
| 857 added_sources = [s for s in sources if s not in self.change_svc] | |
| 858 log.msg("adding %d new changesources, removing %d" % | |
| 859 (len(added_sources), len(deleted_sources))) | |
| 860 dl = [self.change_svc.removeSource(s) for s in deleted_sources] | |
| 861 def addNewOnes(res): | |
| 862 [self.change_svc.addSource(s) for s in added_sources] | |
| 863 d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0) | |
| 864 d.addCallback(addNewOnes) | |
| 865 return d | |
| 866 | |
| 867 def allSchedulers(self): | |
| 868 return [child for child in self | |
| 869 if interfaces.IScheduler.providedBy(child)] | |
| 870 | |
| 871 | |
| 872 def loadConfig_Schedulers(self, newschedulers): | |
| 873 oldschedulers = self.allSchedulers() | |
| 874 removed = [s for s in oldschedulers if s not in newschedulers] | |
| 875 added = [s for s in newschedulers if s not in oldschedulers] | |
| 876 dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed] | |
| 877 def addNewOnes(res): | |
| 878 log.msg("adding %d new schedulers, removed %d" % | |
| 879 (len(added), len(dl))) | |
| 880 for s in added: | |
| 881 s.setServiceParent(self) | |
| 882 d = defer.DeferredList(dl, fireOnOneErrback=1) | |
| 883 d.addCallback(addNewOnes) | |
| 884 if removed or added: | |
| 885 # notify Downstream schedulers to potentially pick up | |
| 886 # new schedulers now that we have removed and added some | |
| 887 def updateDownstreams(res): | |
| 888 log.msg("notifying downstream schedulers of changes") | |
| 889 for s in newschedulers: | |
| 890 if interfaces.IDownstreamScheduler.providedBy(s): | |
| 891 s.checkUpstreamScheduler() | |
| 892 d.addCallback(updateDownstreams) | |
| 893 return d | |
| 894 | |
| 895 def loadConfig_Builders(self, newBuilderData): | |
| 896 somethingChanged = False | |
| 897 newList = {} | |
| 898 newBuilderNames = [] | |
| 899 allBuilders = self.botmaster.builders.copy() | |
| 900 for data in newBuilderData: | |
| 901 name = data['name'] | |
| 902 newList[name] = data | |
| 903 newBuilderNames.append(name) | |
| 904 | |
| 905 # identify all that were removed | |
| 906 for oldname in self.botmaster.getBuildernames(): | |
| 907 if oldname not in newList: | |
| 908 log.msg("removing old builder %s" % oldname) | |
| 909 del allBuilders[oldname] | |
| 910 somethingChanged = True | |
| 911 # announce the change | |
| 912 self.status.builderRemoved(oldname) | |
| 913 | |
| 914 # everything in newList is either unchanged, changed, or new | |
| 915 for name, data in newList.items(): | |
| 916 old = self.botmaster.builders.get(name) | |
| 917 basedir = data['builddir'] | |
| 918 #name, slave, builddir, factory = data | |
| 919 if not old: # new | |
| 920 # category added after 0.6.2 | |
| 921 category = data.get('category', None) | |
| 922 log.msg("adding new builder %s for category %s" % | |
| 923 (name, category)) | |
| 924 statusbag = self.status.builderAdded(name, basedir, category) | |
| 925 builder = Builder(data, statusbag) | |
| 926 allBuilders[name] = builder | |
| 927 somethingChanged = True | |
| 928 elif old.compareToSetup(data): | |
| 929 # changed: try to minimize the disruption and only modify the | |
| 930 # pieces that really changed | |
| 931 diffs = old.compareToSetup(data) | |
| 932 log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) | |
| 933 | |
| 934 statusbag = old.builder_status | |
| 935 statusbag.saveYourself() # seems like a good idea | |
| 936 # TODO: if the basedir was changed, we probably need to make | |
| 937 # a new statusbag | |
| 938 new_builder = Builder(data, statusbag) | |
| 939 new_builder.consumeTheSoulOfYourPredecessor(old) | |
| 940 # that migrates any retained slavebuilders too | |
| 941 | |
| 942 # point out that the builder was updated. On the Waterfall, | |
| 943 # this will appear just after any currently-running builds. | |
| 944 statusbag.addPointEvent(["config", "updated"]) | |
| 945 | |
| 946 allBuilders[name] = new_builder | |
| 947 somethingChanged = True | |
| 948 else: | |
| 949 # unchanged: leave it alone | |
| 950 log.msg("builder %s is unchanged" % name) | |
| 951 pass | |
| 952 | |
| 953 # regardless of whether anything changed, get each builder status | |
| 954 # to update its config | |
| 955 for builder in allBuilders.values(): | |
| 956 builder.builder_status.reconfigFromBuildmaster(self) | |
| 957 | |
| 958 # and then tell the botmaster if anything's changed | |
| 959 if somethingChanged: | |
| 960 sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] | |
| 961 d = self.botmaster.setBuilders(sortedAllBuilders) | |
| 962 return d | |
| 963 return None | |
| 964 | |
| 965 def loadConfig_status(self, status): | |
| 966 dl = [] | |
| 967 | |
| 968 # remove old ones | |
| 969 for s in self.statusTargets[:]: | |
| 970 if not s in status: | |
| 971 log.msg("removing IStatusReceiver", s) | |
| 972 d = defer.maybeDeferred(s.disownServiceParent) | |
| 973 dl.append(d) | |
| 974 self.statusTargets.remove(s) | |
| 975 # after those are finished going away, add new ones | |
| 976 def addNewOnes(res): | |
| 977 for s in status: | |
| 978 if not s in self.statusTargets: | |
| 979 log.msg("adding IStatusReceiver", s) | |
| 980 s.setServiceParent(self) | |
| 981 self.statusTargets.append(s) | |
| 982 d = defer.DeferredList(dl, fireOnOneErrback=1) | |
| 983 d.addCallback(addNewOnes) | |
| 984 return d | |
| 985 | |
| 986 | |
| 987 def addChange(self, change): | |
| 988 for s in self.allSchedulers(): | |
| 989 s.addChange(change) | |
| 990 self.status.changeAdded(change) | |
| 991 | |
| 992 def submitBuildSet(self, bs): | |
| 993 # determine the set of Builders to use | |
| 994 builders = [] | |
| 995 for name in bs.builderNames: | |
| 996 b = self.botmaster.builders.get(name) | |
| 997 if b: | |
| 998 if b not in builders: | |
| 999 builders.append(b) | |
| 1000 continue | |
| 1001 # TODO: add aliases like 'all' | |
| 1002 raise KeyError("no such builder named '%s'" % name) | |
| 1003 | |
| 1004 # now tell the BuildSet to create BuildRequests for all those | |
| 1005 # Builders and submit them | |
| 1006 bs.start(builders) | |
| 1007 self.status.buildsetSubmitted(bs.status) | |
| 1008 | |
| 1009 | |
| 1010 class Control: | |
| 1011 implements(interfaces.IControl) | |
| 1012 | |
| 1013 def __init__(self, master): | |
| 1014 self.master = master | |
| 1015 | |
| 1016 def addChange(self, change): | |
| 1017 self.master.change_svc.addChange(change) | |
| 1018 | |
| 1019 def submitBuildSet(self, bs): | |
| 1020 self.master.submitBuildSet(bs) | |
| 1021 | |
| 1022 def getBuilder(self, name): | |
| 1023 b = self.master.botmaster.builders[name] | |
| 1024 return interfaces.IBuilderControl(b) | |
| 1025 | |
| 1026 components.registerAdapter(Control, BuildMaster, interfaces.IControl) | |
| 1027 | |
| 1028 # so anybody who can get a handle on the BuildMaster can cause a build with: | |
| 1029 # IControl(master).getBuilder("full-2.3").requestBuild(buildrequest) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build