| OLD | NEW |
|---|
| (Empty) |
| 1 | |
| 2 import signal | |
| 3 import shutil, os, errno | |
| 4 from cStringIO import StringIO | |
| 5 from twisted.internet import defer, reactor, protocol | |
| 6 from twisted.python import log, util | |
| 7 | |
| 8 from buildbot import master, interfaces | |
| 9 from buildbot.slave import bot | |
| 10 from buildbot.buildslave import BuildSlave | |
| 11 from buildbot.process.builder import Builder | |
| 12 from buildbot.process.base import BuildRequest, Build | |
| 13 from buildbot.process.buildstep import BuildStep | |
| 14 from buildbot.sourcestamp import SourceStamp | |
| 15 from buildbot.status import builder | |
| 16 from buildbot.process.properties import Properties | |
| 17 | |
| 18 | |
| 19 | |
| 20 class _PutEverythingGetter(protocol.ProcessProtocol): | |
| 21 def __init__(self, deferred, stdin): | |
| 22 self.deferred = deferred | |
| 23 self.outBuf = StringIO() | |
| 24 self.errBuf = StringIO() | |
| 25 self.outReceived = self.outBuf.write | |
| 26 self.errReceived = self.errBuf.write | |
| 27 self.stdin = stdin | |
| 28 | |
| 29 def connectionMade(self): | |
| 30 if self.stdin is not None: | |
| 31 self.transport.write(self.stdin) | |
| 32 self.transport.closeStdin() | |
| 33 | |
| 34 def processEnded(self, reason): | |
| 35 out = self.outBuf.getvalue() | |
| 36 err = self.errBuf.getvalue() | |
| 37 e = reason.value | |
| 38 code = e.exitCode | |
| 39 if e.signal: | |
| 40 self.deferred.errback((out, err, e.signal)) | |
| 41 else: | |
| 42 self.deferred.callback((out, err, code)) | |
| 43 | |
| 44 def myGetProcessOutputAndValue(executable, args=(), env={}, path='.', | |
| 45 _reactor_ignored=None, stdin=None): | |
| 46 """Like twisted.internet.utils.getProcessOutputAndValue but takes | |
| 47 stdin, too.""" | |
| 48 d = defer.Deferred() | |
| 49 p = _PutEverythingGetter(d, stdin) | |
| 50 reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) | |
| 51 return d | |
| 52 | |
| 53 | |
| 54 class MyBot(bot.Bot): | |
| 55 def remote_getSlaveInfo(self): | |
| 56 return self.parent.info | |
| 57 | |
| 58 class MyBuildSlave(bot.BuildSlave): | |
| 59 botClass = MyBot | |
| 60 | |
| 61 def rmtree(d): | |
| 62 try: | |
| 63 shutil.rmtree(d, ignore_errors=1) | |
| 64 except OSError, e: | |
| 65 # stupid 2.2 appears to ignore ignore_errors | |
| 66 if e.errno != errno.ENOENT: | |
| 67 raise | |
| 68 | |
| 69 class RunMixin: | |
| 70 master = None | |
| 71 | |
| 72 def rmtree(self, d): | |
| 73 rmtree(d) | |
| 74 | |
| 75 def setUp(self): | |
| 76 self.slaves = {} | |
| 77 self.rmtree("basedir") | |
| 78 os.mkdir("basedir") | |
| 79 self.master = master.BuildMaster("basedir") | |
| 80 self.status = self.master.getStatus() | |
| 81 self.control = interfaces.IControl(self.master) | |
| 82 | |
| 83 def connectOneSlave(self, slavename, opts={}): | |
| 84 port = self.master.slavePort._port.getHost().port | |
| 85 self.rmtree("slavebase-%s" % slavename) | |
| 86 os.mkdir("slavebase-%s" % slavename) | |
| 87 slave = MyBuildSlave("localhost", port, slavename, "sekrit", | |
| 88 "slavebase-%s" % slavename, | |
| 89 keepalive=0, usePTY=False, debugOpts=opts) | |
| 90 slave.info = {"admin": "one"} | |
| 91 self.slaves[slavename] = slave | |
| 92 slave.startService() | |
| 93 | |
| 94 def connectSlave(self, builders=["dummy"], slavename="bot1", | |
| 95 opts={}): | |
| 96 # connect buildslave 'slavename' and wait for it to connect to all of | |
| 97 # the given builders | |
| 98 dl = [] | |
| 99 # initiate call for all of them, before waiting on result, | |
| 100 # otherwise we might miss some | |
| 101 for b in builders: | |
| 102 dl.append(self.master.botmaster.waitUntilBuilderAttached(b)) | |
| 103 d = defer.DeferredList(dl) | |
| 104 self.connectOneSlave(slavename, opts) | |
| 105 return d | |
| 106 | |
| 107 def connectSlave2(self): | |
| 108 # this takes over for bot1, so it has to share the slavename | |
| 109 port = self.master.slavePort._port.getHost().port | |
| 110 self.rmtree("slavebase-bot2") | |
| 111 os.mkdir("slavebase-bot2") | |
| 112 # this uses bot1, really | |
| 113 slave = MyBuildSlave("localhost", port, "bot1", "sekrit", | |
| 114 "slavebase-bot2", keepalive=0, usePTY=False) | |
| 115 slave.info = {"admin": "two"} | |
| 116 self.slaves['bot2'] = slave | |
| 117 slave.startService() | |
| 118 | |
| 119 def connectSlaveFastTimeout(self): | |
| 120 # this slave has a very fast keepalive timeout | |
| 121 port = self.master.slavePort._port.getHost().port | |
| 122 self.rmtree("slavebase-bot1") | |
| 123 os.mkdir("slavebase-bot1") | |
| 124 slave = MyBuildSlave("localhost", port, "bot1", "sekrit", | |
| 125 "slavebase-bot1", keepalive=2, usePTY=False, | |
| 126 keepaliveTimeout=1) | |
| 127 slave.info = {"admin": "one"} | |
| 128 self.slaves['bot1'] = slave | |
| 129 slave.startService() | |
| 130 d = self.master.botmaster.waitUntilBuilderAttached("dummy") | |
| 131 return d | |
| 132 | |
| 133 # things to start builds | |
| 134 def requestBuild(self, builder): | |
| 135 # returns a Deferred that fires with an IBuildStatus object when the | |
| 136 # build is finished | |
| 137 req = BuildRequest("forced build", SourceStamp(), 'test_builder') | |
| 138 self.control.getBuilder(builder).requestBuild(req) | |
| 139 return req.waitUntilFinished() | |
| 140 | |
| 141 def failUnlessBuildSucceeded(self, bs): | |
| 142 if bs.getResults() != builder.SUCCESS: | |
| 143 log.msg("failUnlessBuildSucceeded noticed that the build failed") | |
| 144 self.logBuildResults(bs) | |
| 145 self.failUnlessEqual(bs.getResults(), builder.SUCCESS) | |
| 146 return bs # useful for chaining | |
| 147 | |
| 148 def logBuildResults(self, bs): | |
| 149 # emit the build status and the contents of all logs to test.log | |
| 150 log.msg("logBuildResults starting") | |
| 151 log.msg(" bs.getResults() == %s" % builder.Results[bs.getResults()]) | |
| 152 log.msg(" bs.isFinished() == %s" % bs.isFinished()) | |
| 153 for s in bs.getSteps(): | |
| 154 for l in s.getLogs(): | |
| 155 log.msg("--- START step %s / log %s ---" % (s.getName(), | |
| 156 l.getName())) | |
| 157 if not l.getName().endswith(".html"): | |
| 158 log.msg(l.getTextWithHeaders()) | |
| 159 log.msg("--- STOP ---") | |
| 160 log.msg("logBuildResults finished") | |
| 161 | |
| 162 def tearDown(self): | |
| 163 log.msg("doing tearDown") | |
| 164 d = self.shutdownAllSlaves() | |
| 165 d.addCallback(self._tearDown_1) | |
| 166 d.addCallback(self._tearDown_2) | |
| 167 return d | |
| 168 def _tearDown_1(self, res): | |
| 169 if self.master: | |
| 170 return defer.maybeDeferred(self.master.stopService) | |
| 171 def _tearDown_2(self, res): | |
| 172 self.master = None | |
| 173 log.msg("tearDown done") | |
| 174 | |
| 175 | |
| 176 # various forms of slave death | |
| 177 | |
| 178 def shutdownAllSlaves(self): | |
| 179 # the slave has disconnected normally: they SIGINT'ed it, or it shut | |
| 180 # down willingly. This will kill child processes and give them a | |
| 181 # chance to finish up. We return a Deferred that will fire when | |
| 182 # everything is finished shutting down. | |
| 183 | |
| 184 log.msg("doing shutdownAllSlaves") | |
| 185 dl = [] | |
| 186 for slave in self.slaves.values(): | |
| 187 slave.stopService() | |
| 188 dl.append(slave.waitUntilDisconnected()) | |
| 189 d = defer.DeferredList(dl) | |
| 190 d.addCallback(self._shutdownAllSlavesDone) | |
| 191 return d | |
| 192 def _shutdownAllSlavesDone(self, res): | |
| 193 for name in self.slaves.keys(): | |
| 194 del self.slaves[name] | |
| 195 return self.master.botmaster.waitUntilBuilderFullyDetached("dummy") | |
| 196 | |
| 197 def shutdownSlave(self, slavename, buildername): | |
| 198 # this slave has disconnected normally: they SIGINT'ed it, or it shut | |
| 199 # down willingly. This will kill child processes and give them a | |
| 200 # chance to finish up. We return a Deferred that will fire when | |
| 201 # everything is finished shutting down, and the given Builder knows | |
| 202 # that the slave has gone away. | |
| 203 | |
| 204 s = self.slaves[slavename] | |
| 205 dl = [self.master.botmaster.waitUntilBuilderDetached(buildername), | |
| 206 s.waitUntilDisconnected()] | |
| 207 d = defer.DeferredList(dl) | |
| 208 d.addCallback(self._shutdownSlave_done, slavename) | |
| 209 s.stopService() | |
| 210 return d | |
| 211 def _shutdownSlave_done(self, res, slavename): | |
| 212 del self.slaves[slavename] | |
| 213 | |
| 214 def killSlave(self, slavename="bot1", buildername="dummy"): | |
| 215 # the slave has died, its host sent a FIN. The .notifyOnDisconnect | |
| 216 # callbacks will terminate the current step, so the build should be | |
| 217 # flunked (no further steps should be started). | |
| 218 self.slaves[slavename].bf.continueTrying = 0 | |
| 219 bot = self.slaves[slavename].getServiceNamed("bot") | |
| 220 broker = bot.builders[buildername].remote.broker | |
| 221 broker.transport.loseConnection() | |
| 222 del self.slaves[slavename] | |
| 223 | |
| 224 def disappearSlave(self, slavename="bot1", buildername="dummy", | |
| 225 allowReconnect=False): | |
| 226 # the slave's host has vanished off the net, leaving the connection | |
| 227 # dangling. This will be detected quickly by app-level keepalives or | |
| 228 # a ping, or slowly by TCP timeouts. | |
| 229 | |
| 230 # simulate this by replacing the slave Broker's .dataReceived method | |
| 231 # with one that just throws away all data. | |
| 232 def discard(data): | |
| 233 pass | |
| 234 bot = self.slaves[slavename].getServiceNamed("bot") | |
| 235 broker = bot.builders[buildername].remote.broker | |
| 236 broker.dataReceived = discard # seal its ears | |
| 237 broker.transport.write = discard # and take away its voice | |
| 238 if not allowReconnect: | |
| 239 # also discourage it from reconnecting once the connection goes away | |
| 240 assert self.slaves[slavename].bf.continueTrying | |
| 241 self.slaves[slavename].bf.continueTrying = False | |
| 242 | |
| 243 def ghostSlave(self): | |
| 244 # the slave thinks it has lost the connection, and initiated a | |
| 245 # reconnect. The master doesn't yet realize it has lost the previous | |
| 246 # connection, and sees two connections at once. | |
| 247 raise NotImplementedError | |
| 248 | |
| 249 | |
| 250 def setupBuildStepStatus(basedir): | |
| 251 """Return a BuildStep with a suitable BuildStepStatus object, ready to | |
| 252 use.""" | |
| 253 os.mkdir(basedir) | |
| 254 botmaster = None | |
| 255 s0 = builder.Status(botmaster, basedir) | |
| 256 s1 = s0.builderAdded("buildername", "buildername") | |
| 257 s2 = builder.BuildStatus(s1, 1) | |
| 258 s3 = builder.BuildStepStatus(s2) | |
| 259 s3.setName("foostep") | |
| 260 s3.started = True | |
| 261 s3.stepStarted() | |
| 262 return s3 | |
| 263 | |
| 264 def fake_slaveVersion(command, oldversion=None): | |
| 265 from buildbot.slave.registry import commandRegistry | |
| 266 return commandRegistry[command] | |
| 267 | |
| 268 class FakeBuildMaster: | |
| 269 properties = Properties(masterprop="master") | |
| 270 | |
| 271 class FakeBotMaster: | |
| 272 parent = FakeBuildMaster() | |
| 273 | |
| 274 def makeBuildStep(basedir, step_class=BuildStep, **kwargs): | |
| 275 bss = setupBuildStepStatus(basedir) | |
| 276 | |
| 277 ss = SourceStamp() | |
| 278 setup = {'name': "builder1", "slavename": "bot1", | |
| 279 'builddir': "builddir", 'slavebuilddir': "slavebuilddir", 'factory'
: None} | |
| 280 b0 = Builder(setup, bss.getBuild().getBuilder()) | |
| 281 b0.botmaster = FakeBotMaster() | |
| 282 br = BuildRequest("reason", ss, 'test_builder') | |
| 283 b = Build([br]) | |
| 284 b.setBuilder(b0) | |
| 285 s = step_class(**kwargs) | |
| 286 s.setBuild(b) | |
| 287 s.setStepStatus(bss) | |
| 288 b.build_status = bss.getBuild() | |
| 289 b.setupProperties() | |
| 290 s.slaveVersion = fake_slaveVersion | |
| 291 return s | |
| 292 | |
| 293 | |
| 294 def findDir(): | |
| 295 # the same directory that holds this script | |
| 296 return util.sibpath(__file__, ".") | |
| 297 | |
| 298 class SignalMixin: | |
| 299 sigchldHandler = None | |
| 300 | |
| 301 def setUpSignalHandler(self): | |
| 302 # make sure SIGCHLD handler is installed, as it should be on | |
| 303 # reactor.run(). problem is reactor may not have been run when this | |
| 304 # test runs. | |
| 305 if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): | |
| 306 self.sigchldHandler = signal.signal(signal.SIGCHLD, | |
| 307 reactor._handleSigchld) | |
| 308 | |
| 309 def tearDownSignalHandler(self): | |
| 310 if self.sigchldHandler: | |
| 311 signal.signal(signal.SIGCHLD, self.sigchldHandler) | |
| 312 | |
| 313 # these classes are used to test SlaveCommands in isolation | |
| 314 | |
| 315 class FakeSlaveBuilder: | |
| 316 debug = False | |
| 317 def __init__(self, usePTY, basedir): | |
| 318 self.updates = [] | |
| 319 self.basedir = basedir | |
| 320 self.usePTY = usePTY | |
| 321 | |
| 322 def sendUpdate(self, data): | |
| 323 if self.debug: | |
| 324 print "FakeSlaveBuilder.sendUpdate", data | |
| 325 self.updates.append(data) | |
| 326 | |
| 327 | |
| 328 class SlaveCommandTestBase(SignalMixin): | |
| 329 usePTY = False | |
| 330 | |
| 331 def setUp(self): | |
| 332 self.setUpSignalHandler() | |
| 333 | |
| 334 def tearDown(self): | |
| 335 self.tearDownSignalHandler() | |
| 336 | |
| 337 def setUpBuilder(self, basedir): | |
| 338 if not os.path.exists(basedir): | |
| 339 os.mkdir(basedir) | |
| 340 self.builder = FakeSlaveBuilder(self.usePTY, basedir) | |
| 341 | |
| 342 def startCommand(self, cmdclass, args): | |
| 343 stepId = 0 | |
| 344 self.cmd = c = cmdclass(self.builder, stepId, args) | |
| 345 c.running = True | |
| 346 d = c.doStart() | |
| 347 return d | |
| 348 | |
| 349 def collectUpdates(self, res=None): | |
| 350 logs = {} | |
| 351 for u in self.builder.updates: | |
| 352 for k in u.keys(): | |
| 353 if k == "log": | |
| 354 logname,data = u[k] | |
| 355 oldlog = logs.get(("log",logname), "") | |
| 356 logs[("log",logname)] = oldlog + data | |
| 357 elif k == "rc": | |
| 358 pass | |
| 359 else: | |
| 360 logs[k] = logs.get(k, "") + u[k] | |
| 361 return logs | |
| 362 | |
| 363 def findRC(self): | |
| 364 for u in self.builder.updates: | |
| 365 if "rc" in u: | |
| 366 return u["rc"] | |
| 367 return None | |
| 368 | |
| 369 def printStderr(self): | |
| 370 for u in self.builder.updates: | |
| 371 if "stderr" in u: | |
| 372 print u["stderr"] | |
| 373 | |
| 374 # ---------------------------------------- | |
| 375 | |
| 376 class LocalWrapper: | |
| 377 # r = pb.Referenceable() | |
| 378 # w = LocalWrapper(r) | |
| 379 # now you can do things like w.callRemote() | |
| 380 def __init__(self, target): | |
| 381 self.target = target | |
| 382 | |
| 383 def callRemote(self, name, *args, **kwargs): | |
| 384 # callRemote is not allowed to fire its Deferred in the same turn | |
| 385 d = defer.Deferred() | |
| 386 d.addCallback(self._callRemote, *args, **kwargs) | |
| 387 reactor.callLater(0, d.callback, name) | |
| 388 return d | |
| 389 | |
| 390 def _callRemote(self, name, *args, **kwargs): | |
| 391 method = getattr(self.target, "remote_"+name) | |
| 392 return method(*args, **kwargs) | |
| 393 | |
| 394 def notifyOnDisconnect(self, observer): | |
| 395 pass | |
| 396 def dontNotifyOnDisconnect(self, observer): | |
| 397 pass | |
| 398 | |
| 399 | |
| 400 class LocalSlaveBuilder(bot.SlaveBuilder): | |
| 401 """I am object that behaves like a pb.RemoteReference, but in fact I | |
| 402 invoke methods locally.""" | |
| 403 _arg_filter = None | |
| 404 | |
| 405 def setArgFilter(self, filter): | |
| 406 self._arg_filter = filter | |
| 407 | |
| 408 def remote_startCommand(self, stepref, stepId, command, args): | |
| 409 if self._arg_filter: | |
| 410 args = self._arg_filter(args) | |
| 411 # stepref should be a RemoteReference to the RemoteCommand | |
| 412 return bot.SlaveBuilder.remote_startCommand(self, | |
| 413 LocalWrapper(stepref), | |
| 414 stepId, command, args) | |
| 415 | |
| 416 class StepTester: | |
| 417 """Utility class to exercise BuildSteps and RemoteCommands, without | |
| 418 really using a Build or a Bot. No networks are used. | |
| 419 | |
| 420 Use this as follows:: | |
| 421 | |
| 422 class MyTest(StepTester, unittest.TestCase): | |
| 423 def testOne(self): | |
| 424 self.slavebase = 'testOne.slave' | |
| 425 self.masterbase = 'testOne.master' | |
| 426 sb = self.makeSlaveBuilder() | |
| 427 step = self.makeStep(stepclass, **kwargs) | |
| 428 d = self.runStep(step) | |
| 429 d.addCallback(_checkResults) | |
| 430 return d | |
| 431 """ | |
| 432 | |
| 433 #slavebase = "slavebase" | |
| 434 slavebuilderbase = "slavebuilderbase" | |
| 435 #masterbase = "masterbase" | |
| 436 | |
| 437 def makeSlaveBuilder(self): | |
| 438 os.mkdir(self.slavebase) | |
| 439 os.mkdir(os.path.join(self.slavebase, self.slavebuilderbase)) | |
| 440 b = bot.Bot(self.slavebase, False) | |
| 441 b.startService() | |
| 442 sb = LocalSlaveBuilder("slavebuildername", False) | |
| 443 sb.setArgFilter(self.filterArgs) | |
| 444 sb.usePTY = False | |
| 445 sb.setServiceParent(b) | |
| 446 sb.setBuilddir(self.slavebuilderbase) | |
| 447 self.remote = LocalWrapper(sb) | |
| 448 return sb | |
| 449 | |
| 450 workdir = "build" | |
| 451 def makeStep(self, factory, **kwargs): | |
| 452 step = makeBuildStep(self.masterbase, factory, **kwargs) | |
| 453 step.setBuildSlave(BuildSlave("name", "password")) | |
| 454 step.setDefaultWorkdir(self.workdir) | |
| 455 return step | |
| 456 | |
| 457 def runStep(self, step): | |
| 458 d = defer.maybeDeferred(step.startStep, self.remote) | |
| 459 return d | |
| 460 | |
| 461 def wrap(self, target): | |
| 462 return LocalWrapper(target) | |
| 463 | |
| 464 def filterArgs(self, args): | |
| 465 # this can be overridden | |
| 466 return args | |
| 467 | |
| 468 # ---------------------------------------- | |
| 469 | |
| 470 _flags = {} | |
| 471 | |
| 472 def setTestFlag(flagname, value): | |
| 473 _flags[flagname] = value | |
| 474 | |
| 475 class SetTestFlagStep(BuildStep): | |
| 476 """ | |
| 477 A special BuildStep to set a named flag; this can be used with the | |
| 478 TestFlagMixin to monitor what has and has not run in a particular | |
| 479 configuration. | |
| 480 """ | |
| 481 def __init__(self, flagname='flag', value=1, **kwargs): | |
| 482 BuildStep.__init__(self, **kwargs) | |
| 483 self.addFactoryArguments(flagname=flagname, value=value) | |
| 484 | |
| 485 self.flagname = flagname | |
| 486 self.value = value | |
| 487 | |
| 488 def start(self): | |
| 489 properties = self.build.getProperties() | |
| 490 _flags[self.flagname] = properties.render(self.value) | |
| 491 self.finished(builder.SUCCESS) | |
| 492 | |
| 493 class TestFlagMixin: | |
| 494 def clearFlags(self): | |
| 495 """ | |
| 496 Set up for a test by clearing all flags; call this from your test | |
| 497 function. | |
| 498 """ | |
| 499 _flags.clear() | |
| 500 | |
| 501 def failIfFlagSet(self, flagname, msg=None): | |
| 502 if not msg: msg = "flag '%s' is set" % flagname | |
| 503 self.failIf(_flags.has_key(flagname), msg=msg) | |
| 504 | |
| 505 def failIfFlagNotSet(self, flagname, msg=None): | |
| 506 if not msg: msg = "flag '%s' is not set" % flagname | |
| 507 self.failUnless(_flags.has_key(flagname), msg=msg) | |
| 508 | |
| 509 def getFlag(self, flagname): | |
| 510 self.failIfFlagNotSet(flagname, "flag '%s' not set" % flagname) | |
| 511 return _flags.get(flagname) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build