| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- test-case-name: buildbot.test.test_runner -*- | |
| 2 | |
| 3 # N.B.: don't import anything that might pull in a reactor yet. Some of our | |
| 4 # subcommands want to load modules that need the gtk reactor. | |
| 5 import os, sys, stat, re, time | |
| 6 import traceback | |
| 7 from twisted.python import usage, util, runtime | |
| 8 | |
| 9 from buildbot.interfaces import BuildbotNotRunningError | |
| 10 | |
| 11 # the create/start/stop commands should all be run as the same user, | |
| 12 # preferably a separate 'buildbot' account. | |
| 13 | |
| 14 # Note that the terms 'options' and 'config' are used intechangeably here - in | |
| 15 # fact, they are intercanged several times. Caveat legator. | |
| 16 | |
| 17 class OptionsWithOptionsFile(usage.Options): | |
| 18 # subclasses should set this to a list-of-lists in order to source the | |
| 19 # .buildbot/options file. | |
| 20 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] | |
| 21 buildbotOptions = None | |
| 22 | |
| 23 def __init__(self, *args): | |
| 24 # for options in self.buildbotOptions, optParameters, and the options | |
| 25 # file, change the default in optParameters *before* calling through | |
| 26 # to the parent constructor | |
| 27 | |
| 28 if self.buildbotOptions: | |
| 29 optfile = loadOptionsFile() | |
| 30 for optfile_name, option_name in self.buildbotOptions: | |
| 31 for i in range(len(self.optParameters)): | |
| 32 if self.optParameters[i][0] == option_name and optfile_name
in optfile: | |
| 33 self.optParameters[i][2] = optfile[optfile_name] | |
| 34 usage.Options.__init__(self, *args) | |
| 35 | |
| 36 def loadOptionsFile(filename="options", here=None, home=None): | |
| 37 """Find the .buildbot/FILENAME file. Crawl from the current directory up | |
| 38 towards the root, and also look in ~/.buildbot . The first directory | |
| 39 that's owned by the user and has the file we're looking for wins. Windows | |
| 40 skips the owned-by-user test. | |
| 41 | |
| 42 @rtype: dict | |
| 43 @return: a dictionary of names defined in the options file. If no options | |
| 44 file was found, return an empty dict. | |
| 45 """ | |
| 46 | |
| 47 if here is None: | |
| 48 here = os.getcwd() | |
| 49 here = os.path.abspath(here) | |
| 50 | |
| 51 if home is None: | |
| 52 if runtime.platformType == 'win32': | |
| 53 home = os.path.join(os.environ['APPDATA'], "buildbot") | |
| 54 else: | |
| 55 home = os.path.expanduser("~/.buildbot") | |
| 56 | |
| 57 searchpath = [] | |
| 58 toomany = 20 | |
| 59 while True: | |
| 60 searchpath.append(os.path.join(here, ".buildbot")) | |
| 61 next = os.path.dirname(here) | |
| 62 if next == here: | |
| 63 break # we've hit the root | |
| 64 here = next | |
| 65 toomany -= 1 # just in case | |
| 66 if toomany == 0: | |
| 67 raise ValueError("Hey, I seem to have wandered up into the " | |
| 68 "infinite glories of the heavens. Oops.") | |
| 69 searchpath.append(home) | |
| 70 | |
| 71 localDict = {} | |
| 72 | |
| 73 for d in searchpath: | |
| 74 if os.path.isdir(d): | |
| 75 if runtime.platformType != 'win32': | |
| 76 if os.stat(d)[stat.ST_UID] != os.getuid(): | |
| 77 print "skipping %s because you don't own it" % d | |
| 78 continue # security, skip other people's directories | |
| 79 optfile = os.path.join(d, filename) | |
| 80 if os.path.exists(optfile): | |
| 81 try: | |
| 82 f = open(optfile, "r") | |
| 83 options = f.read() | |
| 84 exec options in localDict | |
| 85 except: | |
| 86 print "error while reading %s" % optfile | |
| 87 raise | |
| 88 break | |
| 89 | |
| 90 for k in localDict.keys(): | |
| 91 if k.startswith("__"): | |
| 92 del localDict[k] | |
| 93 return localDict | |
| 94 | |
| 95 class MakerBase(OptionsWithOptionsFile): | |
| 96 optFlags = [ | |
| 97 ['help', 'h', "Display this message"], | |
| 98 ["quiet", "q", "Do not emit the commands being run"], | |
| 99 ] | |
| 100 | |
| 101 longdesc = """ | |
| 102 Operates upon the specified <basedir> (or the current directory, if not | |
| 103 specified). | |
| 104 """ | |
| 105 | |
| 106 opt_h = usage.Options.opt_help | |
| 107 | |
| 108 def parseArgs(self, *args): | |
| 109 if len(args) > 0: | |
| 110 self['basedir'] = args[0] | |
| 111 else: | |
| 112 # Use the current directory if no basedir was specified. | |
| 113 self['basedir'] = os.getcwd() | |
| 114 if len(args) > 1: | |
| 115 raise usage.UsageError("I wasn't expecting so many arguments") | |
| 116 | |
| 117 def postOptions(self): | |
| 118 self['basedir'] = os.path.abspath(self['basedir']) | |
| 119 | |
| 120 makefile_sample = """# -*- makefile -*- | |
| 121 | |
| 122 # This is a simple makefile which lives in a buildmaster/buildslave | |
| 123 # directory (next to the buildbot.tac file). It allows you to start/stop the | |
| 124 # master or slave by doing 'make start' or 'make stop'. | |
| 125 | |
| 126 # The 'reconfig' target will tell a buildmaster to reload its config file. | |
| 127 | |
| 128 start: | |
| 129 twistd --no_save -y buildbot.tac | |
| 130 | |
| 131 stop: | |
| 132 kill `cat twistd.pid` | |
| 133 | |
| 134 reconfig: | |
| 135 kill -HUP `cat twistd.pid` | |
| 136 | |
| 137 log: | |
| 138 tail -f twistd.log | |
| 139 """ | |
| 140 | |
| 141 class Maker: | |
| 142 def __init__(self, config): | |
| 143 self.config = config | |
| 144 self.basedir = config['basedir'] | |
| 145 self.force = config.get('force', False) | |
| 146 self.quiet = config['quiet'] | |
| 147 | |
| 148 def mkdir(self): | |
| 149 if os.path.exists(self.basedir): | |
| 150 if not self.quiet: | |
| 151 print "updating existing installation" | |
| 152 return | |
| 153 if not self.quiet: print "mkdir", self.basedir | |
| 154 os.mkdir(self.basedir) | |
| 155 | |
| 156 def mkinfo(self): | |
| 157 path = os.path.join(self.basedir, "info") | |
| 158 if not os.path.exists(path): | |
| 159 if not self.quiet: print "mkdir", path | |
| 160 os.mkdir(path) | |
| 161 created = False | |
| 162 admin = os.path.join(path, "admin") | |
| 163 if not os.path.exists(admin): | |
| 164 if not self.quiet: | |
| 165 print "Creating info/admin, you need to edit it appropriately" | |
| 166 f = open(admin, "wt") | |
| 167 f.write("Your Name Here <admin@youraddress.invalid>\n") | |
| 168 f.close() | |
| 169 created = True | |
| 170 host = os.path.join(path, "host") | |
| 171 if not os.path.exists(host): | |
| 172 if not self.quiet: | |
| 173 print "Creating info/host, you need to edit it appropriately" | |
| 174 f = open(host, "wt") | |
| 175 f.write("Please put a description of this build host here\n") | |
| 176 f.close() | |
| 177 created = True | |
| 178 access_uri = os.path.join(path, "access_uri") | |
| 179 if not os.path.exists(access_uri): | |
| 180 if not self.quiet: | |
| 181 print "Not creating info/access_uri - add it if you wish" | |
| 182 if created and not self.quiet: | |
| 183 print "Please edit the files in %s appropriately." % path | |
| 184 | |
| 185 def chdir(self): | |
| 186 if not self.quiet: print "chdir", self.basedir | |
| 187 os.chdir(self.basedir) | |
| 188 | |
| 189 def makeTAC(self, contents, secret=False): | |
| 190 tacfile = "buildbot.tac" | |
| 191 if os.path.exists(tacfile): | |
| 192 oldcontents = open(tacfile, "rt").read() | |
| 193 if oldcontents == contents: | |
| 194 if not self.quiet: | |
| 195 print "buildbot.tac already exists and is correct" | |
| 196 return | |
| 197 if not self.quiet: | |
| 198 print "not touching existing buildbot.tac" | |
| 199 print "creating buildbot.tac.new instead" | |
| 200 tacfile = "buildbot.tac.new" | |
| 201 f = open(tacfile, "wt") | |
| 202 f.write(contents) | |
| 203 f.close() | |
| 204 if secret: | |
| 205 os.chmod(tacfile, 0600) | |
| 206 | |
| 207 def makefile(self): | |
| 208 target = "Makefile.sample" | |
| 209 if os.path.exists(target): | |
| 210 oldcontents = open(target, "rt").read() | |
| 211 if oldcontents == makefile_sample: | |
| 212 if not self.quiet: | |
| 213 print "Makefile.sample already exists and is correct" | |
| 214 return | |
| 215 if not self.quiet: | |
| 216 print "replacing Makefile.sample" | |
| 217 else: | |
| 218 if not self.quiet: | |
| 219 print "creating Makefile.sample" | |
| 220 f = open(target, "wt") | |
| 221 f.write(makefile_sample) | |
| 222 f.close() | |
| 223 | |
| 224 def sampleconfig(self, source): | |
| 225 target = "master.cfg.sample" | |
| 226 config_sample = open(source, "rt").read() | |
| 227 if os.path.exists(target): | |
| 228 oldcontents = open(target, "rt").read() | |
| 229 if oldcontents == config_sample: | |
| 230 if not self.quiet: | |
| 231 print "master.cfg.sample already exists and is up-to-date" | |
| 232 return | |
| 233 if not self.quiet: | |
| 234 print "replacing master.cfg.sample" | |
| 235 else: | |
| 236 if not self.quiet: | |
| 237 print "creating master.cfg.sample" | |
| 238 f = open(target, "wt") | |
| 239 f.write(config_sample) | |
| 240 f.close() | |
| 241 os.chmod(target, 0600) | |
| 242 | |
| 243 def public_html(self, files): | |
| 244 webdir = os.path.join(self.basedir, "public_html") | |
| 245 if os.path.exists(webdir): | |
| 246 if not self.quiet: | |
| 247 print "public_html/ already exists: not replacing" | |
| 248 return | |
| 249 else: | |
| 250 os.mkdir(webdir) | |
| 251 if not self.quiet: | |
| 252 print "populating public_html/" | |
| 253 for target, source in files.iteritems(): | |
| 254 target = os.path.join(webdir, target) | |
| 255 f = open(target, "wt") | |
| 256 f.write(open(source, "rt").read()) | |
| 257 f.close() | |
| 258 | |
| 259 def populate_if_missing(self, target, source, overwrite=False): | |
| 260 new_contents = open(source, "rt").read() | |
| 261 if os.path.exists(target): | |
| 262 old_contents = open(target, "rt").read() | |
| 263 if old_contents != new_contents: | |
| 264 if overwrite: | |
| 265 if not self.quiet: | |
| 266 print "%s has old/modified contents" % target | |
| 267 print " overwriting it with new contents" | |
| 268 open(target, "wt").write(new_contents) | |
| 269 else: | |
| 270 if not self.quiet: | |
| 271 print "%s has old/modified contents" % target | |
| 272 print " writing new contents to %s.new" % target | |
| 273 open(target + ".new", "wt").write(new_contents) | |
| 274 # otherwise, it's up to date | |
| 275 else: | |
| 276 if not self.quiet: | |
| 277 print "populating %s" % target | |
| 278 open(target, "wt").write(new_contents) | |
| 279 | |
| 280 def upgrade_public_html(self, files): | |
| 281 webdir = os.path.join(self.basedir, "public_html") | |
| 282 if not os.path.exists(webdir): | |
| 283 if not self.quiet: | |
| 284 print "populating public_html/" | |
| 285 os.mkdir(webdir) | |
| 286 for target, source in files.iteritems(): | |
| 287 self.populate_if_missing(os.path.join(webdir, target), | |
| 288 source) | |
| 289 | |
| 290 def check_master_cfg(self): | |
| 291 from buildbot.master import BuildMaster | |
| 292 from twisted.python import log, failure | |
| 293 | |
| 294 master_cfg = os.path.join(self.basedir, "master.cfg") | |
| 295 if not os.path.exists(master_cfg): | |
| 296 if not self.quiet: | |
| 297 print "No master.cfg found" | |
| 298 return 1 | |
| 299 | |
| 300 # side-effects of loading the config file: | |
| 301 | |
| 302 # for each Builder defined in c['builders'], if the status directory | |
| 303 # didn't already exist, it will be created, and the | |
| 304 # $BUILDERNAME/builder pickle might be created (with a single | |
| 305 # "builder created" event). | |
| 306 | |
| 307 # we put basedir in front of sys.path, because that's how the | |
| 308 # buildmaster itself will run, and it is quite common to have the | |
| 309 # buildmaster import helper classes from other .py files in its | |
| 310 # basedir. | |
| 311 | |
| 312 if sys.path[0] != self.basedir: | |
| 313 sys.path.insert(0, self.basedir) | |
| 314 | |
| 315 m = BuildMaster(self.basedir) | |
| 316 # we need to route log.msg to stdout, so any problems can be seen | |
| 317 # there. But if everything goes well, I'd rather not clutter stdout | |
| 318 # with log messages. So instead we add a logObserver which gathers | |
| 319 # messages and only displays them if something goes wrong. | |
| 320 messages = [] | |
| 321 log.addObserver(messages.append) | |
| 322 try: | |
| 323 # this will raise an exception if there's something wrong with | |
| 324 # the config file. Note that this BuildMaster instance is never | |
| 325 # started, so it won't actually do anything with the | |
| 326 # configuration. | |
| 327 m.loadConfig(open(master_cfg, "r")) | |
| 328 except: | |
| 329 f = failure.Failure() | |
| 330 if not self.quiet: | |
| 331 print | |
| 332 for m in messages: | |
| 333 print "".join(m['message']) | |
| 334 print f | |
| 335 print | |
| 336 print "An error was detected in the master.cfg file." | |
| 337 print "Please correct the problem and run 'buildbot upgrade-mast
er' again." | |
| 338 print | |
| 339 return 1 | |
| 340 return 0 | |
| 341 | |
| 342 class UpgradeMasterOptions(MakerBase): | |
| 343 optFlags = [ | |
| 344 ["replace", "r", "Replace any modified files without confirmation."], | |
| 345 ] | |
| 346 | |
| 347 def getSynopsis(self): | |
| 348 return "Usage: buildbot upgrade-master [options] [<basedir>]" | |
| 349 | |
| 350 longdesc = """ | |
| 351 This command takes an existing buildmaster working directory and | |
| 352 adds/modifies the files there to work with the current version of | |
| 353 buildbot. When this command is finished, the buildmaster directory should | |
| 354 look much like a brand-new one created by the 'create-master' command. | |
| 355 | |
| 356 Use this after you've upgraded your buildbot installation and before you | |
| 357 restart the buildmaster to use the new version. | |
| 358 | |
| 359 If you have modified the files in your working directory, this command | |
| 360 will leave them untouched, but will put the new recommended contents in a | |
| 361 .new file (for example, if index.html has been modified, this command | |
| 362 will create index.html.new). You can then look at the new version and | |
| 363 decide how to merge its contents into your modified file. | |
| 364 """ | |
| 365 | |
| 366 def upgradeMaster(config): | |
| 367 basedir = config['basedir'] | |
| 368 m = Maker(config) | |
| 369 # TODO: check Makefile | |
| 370 # TODO: check TAC file | |
| 371 # check web files: index.html, default.css, robots.txt | |
| 372 webdir = os.path.join(basedir, "public_html") | |
| 373 m.upgrade_public_html({ | |
| 374 'index.html' : util.sibpath(__file__, "../status/web/index.html"), | |
| 375 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/bg_gradient.
jpg"), | |
| 376 'buildbot.css' : util.sibpath(__file__, "../status/web/default.css"), | |
| 377 'robots.txt' : util.sibpath(__file__, "../status/web/robots.txt"), | |
| 378 }) | |
| 379 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), | |
| 380 util.sibpath(__file__, "sample.cfg"), | |
| 381 overwrite=True) | |
| 382 rc = m.check_master_cfg() | |
| 383 if rc: | |
| 384 return rc | |
| 385 if not config['quiet']: | |
| 386 print "upgrade complete" | |
| 387 | |
| 388 | |
| 389 class MasterOptions(MakerBase): | |
| 390 optFlags = [ | |
| 391 ["force", "f", | |
| 392 "Re-use an existing directory (will not overwrite master.cfg file)"], | |
| 393 ] | |
| 394 optParameters = [ | |
| 395 ["config", "c", "master.cfg", "name of the buildmaster config file"], | |
| 396 ["log-size", "s", "1000000", | |
| 397 "size at which to rotate twisted log files"], | |
| 398 ["log-count", "l", "None", | |
| 399 "limit the number of kept old twisted log files"], | |
| 400 ] | |
| 401 def getSynopsis(self): | |
| 402 return "Usage: buildbot create-master [options] [<basedir>]" | |
| 403 | |
| 404 longdesc = """ | |
| 405 This command creates a buildmaster working directory and buildbot.tac | |
| 406 file. The master will live in <dir> and create various files there. | |
| 407 | |
| 408 At runtime, the master will read a configuration file (named | |
| 409 'master.cfg' by default) in its basedir. This file should contain python | |
| 410 code which eventually defines a dictionary named 'BuildmasterConfig'. | |
| 411 The elements of this dictionary are used to configure the Buildmaster. | |
| 412 See doc/config.xhtml for details about what can be controlled through | |
| 413 this interface.""" | |
| 414 | |
| 415 def postOptions(self): | |
| 416 MakerBase.postOptions(self) | |
| 417 if not re.match('^\d+$', self['log-size']): | |
| 418 raise usage.UsageError("log-size parameter needs to be an int") | |
| 419 if not re.match('^\d+$', self['log-count']) and \ | |
| 420 self['log-count'] != 'None': | |
| 421 raise usage.UsageError("log-count parameter needs to be an int "+ | |
| 422 " or None") | |
| 423 | |
| 424 | |
| 425 masterTAC = """ | |
| 426 from twisted.application import service | |
| 427 from buildbot.master import BuildMaster | |
| 428 | |
| 429 basedir = r'%(basedir)s' | |
| 430 configfile = r'%(config)s' | |
| 431 rotateLength = %(log-size)s | |
| 432 maxRotatedFiles = %(log-count)s | |
| 433 | |
| 434 application = service.Application('buildmaster') | |
| 435 try: | |
| 436 from twisted.python.logfile import LogFile | |
| 437 from twisted.python.log import ILogObserver, FileLogObserver | |
| 438 logfile = LogFile.fromFullPath("twistd.log", rotateLength=rotateLength, | |
| 439 maxRotatedFiles=maxRotatedFiles) | |
| 440 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) | |
| 441 except ImportError: | |
| 442 # probably not yet twisted 8.2.0 and beyond, can't set log yet | |
| 443 pass | |
| 444 BuildMaster(basedir, configfile).setServiceParent(application) | |
| 445 | |
| 446 """ | |
| 447 | |
| 448 def createMaster(config): | |
| 449 m = Maker(config) | |
| 450 m.mkdir() | |
| 451 m.chdir() | |
| 452 contents = masterTAC % config | |
| 453 m.makeTAC(contents) | |
| 454 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) | |
| 455 m.public_html({ | |
| 456 'index.html' : util.sibpath(__file__, "../status/web/index.html"), | |
| 457 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/bg_gradient.
jpg"), | |
| 458 'buildbot.css' : util.sibpath(__file__, "../status/web/default.css"), | |
| 459 'robots.txt' : util.sibpath(__file__, "../status/web/robots.txt"), | |
| 460 }) | |
| 461 m.makefile() | |
| 462 | |
| 463 if not m.quiet: print "buildmaster configured in %s" % m.basedir | |
| 464 | |
| 465 class SlaveOptions(MakerBase): | |
| 466 optFlags = [ | |
| 467 ["force", "f", "Re-use an existing directory"], | |
| 468 ] | |
| 469 optParameters = [ | |
| 470 # ["name", "n", None, "Name for this build slave"], | |
| 471 # ["passwd", "p", None, "Password for this build slave"], | |
| 472 # ["basedir", "d", ".", "Base directory to use"], | |
| 473 # ["master", "m", "localhost:8007", | |
| 474 # "Location of the buildmaster (host:port)"], | |
| 475 | |
| 476 ["keepalive", "k", 600, | |
| 477 "Interval at which keepalives should be sent (in seconds)"], | |
| 478 ["usepty", None, 0, | |
| 479 "(1 or 0) child processes should be run in a pty (default 0)"], | |
| 480 ["umask", None, "None", | |
| 481 "controls permissions of generated files. Use --umask=022 to be world-r
eadable"], | |
| 482 ["maxdelay", None, 300, | |
| 483 "Maximum time between connection attempts"], | |
| 484 ["log-size", "s", "1000000", | |
| 485 "size at which to rotate twisted log files"], | |
| 486 ["log-count", "l", "None", | |
| 487 "limit the number of kept old twisted log files"], | |
| 488 ] | |
| 489 | |
| 490 longdesc = """ | |
| 491 This command creates a buildslave working directory and buildbot.tac | |
| 492 file. The bot will use the <name> and <passwd> arguments to authenticate | |
| 493 itself when connecting to the master. All commands are run in a | |
| 494 build-specific subdirectory of <basedir>. <master> is a string of the | |
| 495 form 'hostname:port', and specifies where the buildmaster can be reached. | |
| 496 | |
| 497 <name>, <passwd>, and <master> will be provided by the buildmaster | |
| 498 administrator for your bot. You must choose <basedir> yourself. | |
| 499 """ | |
| 500 | |
| 501 def getSynopsis(self): | |
| 502 return "Usage: buildbot create-slave [options] <basedir> <master> <na
me> <passwd>" | |
| 503 | |
| 504 def parseArgs(self, *args): | |
| 505 if len(args) < 4: | |
| 506 raise usage.UsageError("command needs more arguments") | |
| 507 basedir, master, name, passwd = args | |
| 508 if master[:5] == "http:": | |
| 509 raise usage.UsageError("<master> is not a URL - do not use URL") | |
| 510 self['basedir'] = basedir | |
| 511 self['master'] = master | |
| 512 self['name'] = name | |
| 513 self['passwd'] = passwd | |
| 514 | |
| 515 def postOptions(self): | |
| 516 MakerBase.postOptions(self) | |
| 517 self['usepty'] = int(self['usepty']) | |
| 518 self['keepalive'] = int(self['keepalive']) | |
| 519 self['maxdelay'] = int(self['maxdelay']) | |
| 520 if self['master'].find(":") == -1: | |
| 521 raise usage.UsageError("--master must be in the form host:portnum") | |
| 522 if not re.match('^\d+$', self['log-size']): | |
| 523 raise usage.UsageError("log-size parameter needs to be an int") | |
| 524 if not re.match('^\d+$', self['log-count']) and \ | |
| 525 self['log-count'] != 'None': | |
| 526 raise usage.UsageError("log-count parameter needs to be an int "+ | |
| 527 " or None") | |
| 528 | |
| 529 slaveTAC = """ | |
| 530 from twisted.application import service | |
| 531 from buildbot.slave.bot import BuildSlave | |
| 532 | |
| 533 basedir = r'%(basedir)s' | |
| 534 buildmaster_host = '%(host)s' | |
| 535 port = %(port)d | |
| 536 slavename = '%(name)s' | |
| 537 passwd = '%(passwd)s' | |
| 538 keepalive = %(keepalive)d | |
| 539 usepty = %(usepty)d | |
| 540 umask = %(umask)s | |
| 541 maxdelay = %(maxdelay)d | |
| 542 rotateLength = %(log-size)s | |
| 543 maxRotatedFiles = %(log-count)s | |
| 544 | |
| 545 application = service.Application('buildslave') | |
| 546 try: | |
| 547 from twisted.python.logfile import LogFile | |
| 548 from twisted.python.log import ILogObserver, FileLogObserver | |
| 549 logfile = LogFile.fromFullPath("twistd.log", rotateLength=rotateLength, | |
| 550 maxRotatedFiles=maxRotatedFiles) | |
| 551 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) | |
| 552 except ImportError: | |
| 553 # probably not yet twisted 8.2.0 and beyond, can't set log yet | |
| 554 pass | |
| 555 s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, | |
| 556 keepalive, usepty, umask=umask, maxdelay=maxdelay) | |
| 557 s.setServiceParent(application) | |
| 558 | |
| 559 """ | |
| 560 | |
| 561 def createSlave(config): | |
| 562 m = Maker(config) | |
| 563 m.mkdir() | |
| 564 m.chdir() | |
| 565 try: | |
| 566 master = config['master'] | |
| 567 host, port = re.search(r'(.+):(\d+)', master).groups() | |
| 568 config['host'] = host | |
| 569 config['port'] = int(port) | |
| 570 except: | |
| 571 print "unparseable master location '%s'" % master | |
| 572 print " expecting something more like localhost:8007" | |
| 573 raise | |
| 574 contents = slaveTAC % config | |
| 575 | |
| 576 m.makeTAC(contents, secret=True) | |
| 577 | |
| 578 m.makefile() | |
| 579 m.mkinfo() | |
| 580 | |
| 581 if not m.quiet: print "buildslave configured in %s" % m.basedir | |
| 582 | |
| 583 | |
| 584 | |
| 585 def stop(config, signame="TERM", wait=False): | |
| 586 import signal | |
| 587 basedir = config['basedir'] | |
| 588 quiet = config['quiet'] | |
| 589 os.chdir(basedir) | |
| 590 try: | |
| 591 f = open("twistd.pid", "rt") | |
| 592 except: | |
| 593 raise BuildbotNotRunningError | |
| 594 pid = int(f.read().strip()) | |
| 595 signum = getattr(signal, "SIG"+signame) | |
| 596 timer = 0 | |
| 597 try: | |
| 598 os.kill(pid, signum) | |
| 599 except OSError, e: | |
| 600 if e.errno != 3: | |
| 601 raise | |
| 602 | |
| 603 if not wait: | |
| 604 if not quiet: | |
| 605 print "sent SIG%s to process" % signame | |
| 606 return | |
| 607 time.sleep(0.1) | |
| 608 while timer < 10: | |
| 609 # poll once per second until twistd.pid goes away, up to 10 seconds | |
| 610 try: | |
| 611 os.kill(pid, 0) | |
| 612 except OSError: | |
| 613 if not quiet: | |
| 614 print "buildbot process %d is dead" % pid | |
| 615 return | |
| 616 timer += 1 | |
| 617 time.sleep(1) | |
| 618 if not quiet: | |
| 619 print "never saw process go away" | |
| 620 | |
| 621 def restart(config): | |
| 622 quiet = config['quiet'] | |
| 623 from buildbot.scripts.startup import start | |
| 624 try: | |
| 625 stop(config, wait=True) | |
| 626 except BuildbotNotRunningError: | |
| 627 pass | |
| 628 if not quiet: | |
| 629 print "now restarting buildbot process.." | |
| 630 start(config) | |
| 631 | |
| 632 | |
| 633 class StartOptions(MakerBase): | |
| 634 optFlags = [ | |
| 635 ['quiet', 'q', "Don't display startup log messages"], | |
| 636 ] | |
| 637 def getSynopsis(self): | |
| 638 return "Usage: buildbot start [<basedir>]" | |
| 639 | |
| 640 class StopOptions(MakerBase): | |
| 641 def getSynopsis(self): | |
| 642 return "Usage: buildbot stop [<basedir>]" | |
| 643 | |
| 644 class ReconfigOptions(MakerBase): | |
| 645 optFlags = [ | |
| 646 ['quiet', 'q', "Don't display log messages about reconfiguration"], | |
| 647 ] | |
| 648 def getSynopsis(self): | |
| 649 return "Usage: buildbot reconfig [<basedir>]" | |
| 650 | |
| 651 | |
| 652 | |
| 653 class RestartOptions(MakerBase): | |
| 654 optFlags = [ | |
| 655 ['quiet', 'q', "Don't display startup log messages"], | |
| 656 ] | |
| 657 def getSynopsis(self): | |
| 658 return "Usage: buildbot restart [<basedir>]" | |
| 659 | |
| 660 class DebugClientOptions(OptionsWithOptionsFile): | |
| 661 optFlags = [ | |
| 662 ['help', 'h', "Display this message"], | |
| 663 ] | |
| 664 optParameters = [ | |
| 665 ["master", "m", None, | |
| 666 "Location of the buildmaster's slaveport (host:port)"], | |
| 667 ["passwd", "p", None, "Debug password to use"], | |
| 668 ["myoption", "O", "DEF", "My Option!"], | |
| 669 ] | |
| 670 buildbotOptions = [ | |
| 671 [ 'debugMaster', 'passwd' ], | |
| 672 [ 'master', 'master' ], | |
| 673 ] | |
| 674 | |
| 675 def parseArgs(self, *args): | |
| 676 if len(args) > 0: | |
| 677 self['master'] = args[0] | |
| 678 if len(args) > 1: | |
| 679 self['passwd'] = args[1] | |
| 680 if len(args) > 2: | |
| 681 raise usage.UsageError("I wasn't expecting so many arguments") | |
| 682 | |
| 683 def postOptions(self): | |
| 684 print self['myoption'] | |
| 685 sys.exit(1) | |
| 686 | |
| 687 def debugclient(config): | |
| 688 from buildbot.clients import debug | |
| 689 | |
| 690 master = config.get('master') | |
| 691 if master is None: | |
| 692 raise usage.UsageError("master must be specified: on the command " | |
| 693 "line or in ~/.buildbot/options") | |
| 694 | |
| 695 passwd = config.get('passwd') | |
| 696 if passwd is None: | |
| 697 raise usage.UsageError("passwd must be specified: on the command " | |
| 698 "line or in ~/.buildbot/options") | |
| 699 | |
| 700 d = debug.DebugWidget(master, passwd) | |
| 701 d.run() | |
| 702 | |
| 703 class StatusClientOptions(OptionsWithOptionsFile): | |
| 704 optFlags = [ | |
| 705 ['help', 'h', "Display this message"], | |
| 706 ] | |
| 707 optParameters = [ | |
| 708 ["master", "m", None, | |
| 709 "Location of the buildmaster's status port (host:port)"], | |
| 710 ] | |
| 711 buildbotOptions = [ | |
| 712 [ 'masterstatus', 'master' ], | |
| 713 ] | |
| 714 | |
| 715 def parseArgs(self, *args): | |
| 716 if len(args) > 0: | |
| 717 self['master'] = args[0] | |
| 718 if len(args) > 1: | |
| 719 raise usage.UsageError("I wasn't expecting so many arguments") | |
| 720 | |
| 721 def statuslog(config): | |
| 722 from buildbot.clients import base | |
| 723 master = config.get('master') | |
| 724 if master is None: | |
| 725 raise usage.UsageError("master must be specified: on the command " | |
| 726 "line or in ~/.buildbot/options") | |
| 727 c = base.TextClient(master) | |
| 728 c.run() | |
| 729 | |
| 730 def statusgui(config): | |
| 731 from buildbot.clients import gtkPanes | |
| 732 master = config.get('master') | |
| 733 if master is None: | |
| 734 raise usage.UsageError("master must be specified: on the command " | |
| 735 "line or in ~/.buildbot/options") | |
| 736 c = gtkPanes.GtkClient(master) | |
| 737 c.run() | |
| 738 | |
| 739 class SendChangeOptions(OptionsWithOptionsFile): | |
| 740 def __init__(self): | |
| 741 OptionsWithOptionsFile.__init__(self) | |
| 742 self['properties'] = {} | |
| 743 | |
| 744 optParameters = [ | |
| 745 ("master", "m", None, | |
| 746 "Location of the buildmaster's PBListener (host:port)"), | |
| 747 ("username", "u", None, "Username performing the commit"), | |
| 748 ("branch", "b", None, "Branch specifier"), | |
| 749 ("category", "c", None, "Category of repository"), | |
| 750 ("revision", "r", None, "Revision specifier (string)"), | |
| 751 ("revision_number", "n", None, "Revision specifier (integer)"), | |
| 752 ("revision_file", None, None, "Filename containing revision spec"), | |
| 753 ("property", "p", None, | |
| 754 "A property for the change, in the format: name:value"), | |
| 755 ("comments", "m", None, "log message"), | |
| 756 ("logfile", "F", None, | |
| 757 "Read the log messages from this file (- for stdin)"), | |
| 758 ("when", "w", None, "timestamp to use as the change time"), | |
| 759 ] | |
| 760 | |
| 761 buildbotOptions = [ | |
| 762 [ 'master', 'master' ], | |
| 763 [ 'username', 'username' ], | |
| 764 [ 'branch', 'branch' ], | |
| 765 [ 'category', 'category' ], | |
| 766 ] | |
| 767 | |
| 768 def getSynopsis(self): | |
| 769 return "Usage: buildbot sendchange [options] filenames.." | |
| 770 def parseArgs(self, *args): | |
| 771 self['files'] = args | |
| 772 def opt_property(self, property): | |
| 773 name,value = property.split(':') | |
| 774 self['properties'][name] = value | |
| 775 | |
| 776 | |
| 777 def sendchange(config, runReactor=False): | |
| 778 """Send a single change to the buildmaster's PBChangeSource. The | |
| 779 connection will be drpoped as soon as the Change has been sent.""" | |
| 780 from buildbot.clients.sendchange import Sender | |
| 781 | |
| 782 user = config.get('username') | |
| 783 master = config.get('master') | |
| 784 branch = config.get('branch') | |
| 785 category = config.get('category') | |
| 786 revision = config.get('revision') | |
| 787 properties = config.get('properties', {}) | |
| 788 if config.get('when'): | |
| 789 when = float(config.get('when')) | |
| 790 else: | |
| 791 when = None | |
| 792 # SVN and P4 use numeric revisions | |
| 793 if config.get("revision_number"): | |
| 794 revision = int(config['revision_number']) | |
| 795 if config.get("revision_file"): | |
| 796 revision = open(config["revision_file"],"r").read() | |
| 797 | |
| 798 comments = config.get('comments') | |
| 799 if not comments and config.get('logfile'): | |
| 800 if config['logfile'] == "-": | |
| 801 f = sys.stdin | |
| 802 else: | |
| 803 f = open(config['logfile'], "rt") | |
| 804 comments = f.read() | |
| 805 if comments is None: | |
| 806 comments = "" | |
| 807 | |
| 808 files = config.get('files', []) | |
| 809 | |
| 810 assert user, "you must provide a username" | |
| 811 assert master, "you must provide the master location" | |
| 812 | |
| 813 s = Sender(master, user) | |
| 814 d = s.send(branch, revision, comments, files, category=category, when=when, | |
| 815 properties=properties) | |
| 816 if runReactor: | |
| 817 d.addCallbacks(s.printSuccess, s.printFailure) | |
| 818 d.addBoth(s.stop) | |
| 819 s.run() | |
| 820 return d | |
| 821 | |
| 822 | |
| 823 class ForceOptions(OptionsWithOptionsFile): | |
| 824 optParameters = [ | |
| 825 ["builder", None, None, "which Builder to start"], | |
| 826 ["branch", None, None, "which branch to build"], | |
| 827 ["revision", None, None, "which revision to build"], | |
| 828 ["reason", None, None, "the reason for starting the build"], | |
| 829 ] | |
| 830 | |
| 831 def parseArgs(self, *args): | |
| 832 args = list(args) | |
| 833 if len(args) > 0: | |
| 834 if self['builder'] is not None: | |
| 835 raise usage.UsageError("--builder provided in two ways") | |
| 836 self['builder'] = args.pop(0) | |
| 837 if len(args) > 0: | |
| 838 if self['reason'] is not None: | |
| 839 raise usage.UsageError("--reason provided in two ways") | |
| 840 self['reason'] = " ".join(args) | |
| 841 | |
| 842 | |
| 843 class TryOptions(OptionsWithOptionsFile): | |
| 844 optParameters = [ | |
| 845 ["connect", "c", None, | |
| 846 "how to reach the buildmaster, either 'ssh' or 'pb'"], | |
| 847 # for ssh, use --tryhost, --username, and --trydir | |
| 848 ["tryhost", None, None, | |
| 849 "the hostname (used by ssh) for the buildmaster"], | |
| 850 ["trydir", None, None, | |
| 851 "the directory (on the tryhost) where tryjobs are deposited"], | |
| 852 ["username", "u", None, "Username performing the trial build"], | |
| 853 # for PB, use --master, --username, and --passwd | |
| 854 ["master", "m", None, | |
| 855 "Location of the buildmaster's PBListener (host:port)"], | |
| 856 ["passwd", None, None, "password for PB authentication"], | |
| 857 | |
| 858 ["diff", None, None, | |
| 859 "Filename of a patch to use instead of scanning a local tree. Use '-' f
or stdin."], | |
| 860 ["patchlevel", "p", 0, | |
| 861 "Number of slashes to remove from patch pathnames, like the -p option t
o 'patch'"], | |
| 862 | |
| 863 ["baserev", None, None, | |
| 864 "Base revision to use instead of scanning a local tree."], | |
| 865 | |
| 866 ["vc", None, None, | |
| 867 "The VC system in use, one of: cvs,svn,tla,baz,darcs"], | |
| 868 ["branch", None, None, | |
| 869 "The branch in use, for VC systems that can't figure it out" | |
| 870 " themselves"], | |
| 871 | |
| 872 ["builder", "b", None, | |
| 873 "Run the trial build on this Builder. Can be used multiple times."], | |
| 874 ["properties", None, None, | |
| 875 "A set of properties made available in the build environment, format:pr
op=value,propb=valueb..."], | |
| 876 | |
| 877 ["try-topfile", None, None, | |
| 878 "Name of a file at the top of the tree, used to find the top. Only need
ed for SVN and CVS."], | |
| 879 ["try-topdir", None, None, | |
| 880 "Path to the top of the working copy. Only needed for SVN and CVS."], | |
| 881 | |
| 882 ] | |
| 883 | |
| 884 optFlags = [ | |
| 885 ["wait", None, "wait until the builds have finished"], | |
| 886 ["dryrun", 'n', "Gather info, but don't actually submit."], | |
| 887 ] | |
| 888 | |
| 889 # here it is, the definitive, quirky mapping of .buildbot/options names to | |
| 890 # command-line options. Design by committee, anyone? | |
| 891 buildbotOptions = [ | |
| 892 [ 'try_connect', 'connect' ], | |
| 893 #[ 'try_builders', 'builders' ], <-- handled in postOptions | |
| 894 [ 'try_vc', 'vc' ], | |
| 895 [ 'try_branch', 'branch' ], | |
| 896 [ 'try_topdir', 'try-topdir' ], | |
| 897 [ 'try_topfile', 'try-topfile' ], | |
| 898 [ 'try_host', 'tryhost' ], | |
| 899 [ 'try_username', 'username' ], | |
| 900 [ 'try_dir', 'trydir' ], | |
| 901 [ 'try_password', 'passwd' ], | |
| 902 [ 'try_master', 'master' ], | |
| 903 #[ 'try_wait', 'wait' ], <-- handled in postOptions | |
| 904 [ 'masterstatus', 'master' ], | |
| 905 ] | |
| 906 | |
| 907 def __init__(self): | |
| 908 OptionsWithOptionsFile.__init__(self) | |
| 909 self['builders'] = [] | |
| 910 self['properties'] = {} | |
| 911 | |
| 912 def opt_builder(self, option): | |
| 913 self['builders'].append(option) | |
| 914 | |
| 915 def opt_properties(self, option): | |
| 916 # We need to split the value of this option into a dictionary of propert
ies | |
| 917 properties = {} | |
| 918 propertylist = option.split(",") | |
| 919 for i in range(0,len(propertylist)): | |
| 920 print propertylist[i] | |
| 921 splitproperty = propertylist[i].split("=") | |
| 922 properties[splitproperty[0]] = splitproperty[1] | |
| 923 self['properties'] = properties | |
| 924 | |
| 925 def opt_patchlevel(self, option): | |
| 926 self['patchlevel'] = int(option) | |
| 927 | |
| 928 def getSynopsis(self): | |
| 929 return "Usage: buildbot try [options]" | |
| 930 | |
| 931 def postOptions(self): | |
| 932 opts = loadOptionsFile() | |
| 933 if not self['builders']: | |
| 934 self['builders'] = opts.get('try_builders', []) | |
| 935 if opts.get('try_wait', False): | |
| 936 self['wait'] = True | |
| 937 | |
| 938 def doTry(config): | |
| 939 from buildbot.scripts import tryclient | |
| 940 t = tryclient.Try(config) | |
| 941 t.run() | |
| 942 | |
| 943 class TryServerOptions(OptionsWithOptionsFile): | |
| 944 optParameters = [ | |
| 945 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], | |
| 946 ] | |
| 947 | |
| 948 def doTryServer(config): | |
| 949 import md5 | |
| 950 jobdir = os.path.expanduser(config["jobdir"]) | |
| 951 job = sys.stdin.read() | |
| 952 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to | |
| 953 # jobdir/new . Rather than come up with a unique name randomly, I'm just | |
| 954 # going to MD5 the contents and prepend a timestamp. | |
| 955 timestring = "%d" % time.time() | |
| 956 jobhash = md5.new(job).hexdigest() | |
| 957 fn = "%s-%s" % (timestring, jobhash) | |
| 958 tmpfile = os.path.join(jobdir, "tmp", fn) | |
| 959 newfile = os.path.join(jobdir, "new", fn) | |
| 960 f = open(tmpfile, "w") | |
| 961 f.write(job) | |
| 962 f.close() | |
| 963 os.rename(tmpfile, newfile) | |
| 964 | |
| 965 | |
| 966 class CheckConfigOptions(OptionsWithOptionsFile): | |
| 967 optFlags = [ | |
| 968 ['quiet', 'q', "Don't display error messages or tracebacks"], | |
| 969 ] | |
| 970 | |
| 971 def getSynopsis(self): | |
| 972 return "Usage :buildbot checkconfig [configFile]\n" + \ | |
| 973 " If not specified, 'master.cfg' will be used as 'configFi
le'" | |
| 974 | |
| 975 def parseArgs(self, *args): | |
| 976 if len(args) >= 1: | |
| 977 self['configFile'] = args[0] | |
| 978 else: | |
| 979 self['configFile'] = 'master.cfg' | |
| 980 | |
| 981 | |
| 982 def doCheckConfig(config): | |
| 983 quiet = config.get('quiet') | |
| 984 configFileName = config.get('configFile') | |
| 985 try: | |
| 986 from buildbot.scripts.checkconfig import ConfigLoader | |
| 987 if os.path.isdir(configFileName): | |
| 988 ConfigLoader(basedir=configFileName) | |
| 989 else: | |
| 990 ConfigLoader(configFileName=configFileName) | |
| 991 except: | |
| 992 if not quiet: | |
| 993 # Print out the traceback in a nice format | |
| 994 t, v, tb = sys.exc_info() | |
| 995 traceback.print_exception(t, v, tb) | |
| 996 sys.exit(1) | |
| 997 | |
| 998 if not quiet: | |
| 999 print "Config file is good!" | |
| 1000 | |
| 1001 | |
| 1002 class Options(usage.Options): | |
| 1003 synopsis = "Usage: buildbot <command> [command options]" | |
| 1004 | |
| 1005 subCommands = [ | |
| 1006 # the following are all admin commands | |
| 1007 ['create-master', None, MasterOptions, | |
| 1008 "Create and populate a directory for a new buildmaster"], | |
| 1009 ['upgrade-master', None, UpgradeMasterOptions, | |
| 1010 "Upgrade an existing buildmaster directory for the current version"], | |
| 1011 ['create-slave', None, SlaveOptions, | |
| 1012 "Create and populate a directory for a new buildslave"], | |
| 1013 ['start', None, StartOptions, "Start a buildmaster or buildslave"], | |
| 1014 ['stop', None, StopOptions, "Stop a buildmaster or buildslave"], | |
| 1015 ['restart', None, RestartOptions, | |
| 1016 "Restart a buildmaster or buildslave"], | |
| 1017 | |
| 1018 ['reconfig', None, ReconfigOptions, | |
| 1019 "SIGHUP a buildmaster to make it re-read the config file"], | |
| 1020 ['sighup', None, ReconfigOptions, | |
| 1021 "SIGHUP a buildmaster to make it re-read the config file"], | |
| 1022 | |
| 1023 ['sendchange', None, SendChangeOptions, | |
| 1024 "Send a change to the buildmaster"], | |
| 1025 | |
| 1026 ['debugclient', None, DebugClientOptions, | |
| 1027 "Launch a small debug panel GUI"], | |
| 1028 | |
| 1029 ['statuslog', None, StatusClientOptions, | |
| 1030 "Emit current builder status to stdout"], | |
| 1031 ['statusgui', None, StatusClientOptions, | |
| 1032 "Display a small window showing current builder status"], | |
| 1033 | |
| 1034 #['force', None, ForceOptions, "Run a build"], | |
| 1035 ['try', None, TryOptions, "Run a build with your local changes"], | |
| 1036 | |
| 1037 ['tryserver', None, TryServerOptions, | |
| 1038 "buildmaster-side 'try' support function, not for users"], | |
| 1039 | |
| 1040 ['checkconfig', None, CheckConfigOptions, | |
| 1041 "test the validity of a master.cfg config file"], | |
| 1042 | |
| 1043 # TODO: 'watch' | |
| 1044 ] | |
| 1045 | |
| 1046 def opt_version(self): | |
| 1047 import buildbot | |
| 1048 print "Buildbot version: %s" % buildbot.version | |
| 1049 usage.Options.opt_version(self) | |
| 1050 | |
| 1051 def opt_verbose(self): | |
| 1052 from twisted.python import log | |
| 1053 log.startLogging(sys.stderr) | |
| 1054 | |
| 1055 def postOptions(self): | |
| 1056 if not hasattr(self, 'subOptions'): | |
| 1057 raise usage.UsageError("must specify a command") | |
| 1058 | |
| 1059 | |
| 1060 def run(): | |
| 1061 config = Options() | |
| 1062 try: | |
| 1063 config.parseOptions() | |
| 1064 except usage.error, e: | |
| 1065 print "%s: %s" % (sys.argv[0], e) | |
| 1066 print | |
| 1067 c = getattr(config, 'subOptions', config) | |
| 1068 print str(c) | |
| 1069 sys.exit(1) | |
| 1070 | |
| 1071 command = config.subCommand | |
| 1072 so = config.subOptions | |
| 1073 | |
| 1074 if command == "create-master": | |
| 1075 createMaster(so) | |
| 1076 elif command == "upgrade-master": | |
| 1077 upgradeMaster(so) | |
| 1078 elif command == "create-slave": | |
| 1079 createSlave(so) | |
| 1080 elif command == "start": | |
| 1081 from buildbot.scripts.startup import start | |
| 1082 start(so) | |
| 1083 elif command == "stop": | |
| 1084 stop(so, wait=True) | |
| 1085 elif command == "restart": | |
| 1086 restart(so) | |
| 1087 elif command == "reconfig" or command == "sighup": | |
| 1088 from buildbot.scripts.reconfig import Reconfigurator | |
| 1089 Reconfigurator().run(so) | |
| 1090 elif command == "sendchange": | |
| 1091 sendchange(so, True) | |
| 1092 elif command == "debugclient": | |
| 1093 debugclient(so) | |
| 1094 elif command == "statuslog": | |
| 1095 statuslog(so) | |
| 1096 elif command == "statusgui": | |
| 1097 statusgui(so) | |
| 1098 elif command == "try": | |
| 1099 doTry(so) | |
| 1100 elif command == "tryserver": | |
| 1101 doTryServer(so) | |
| 1102 elif command == "checkconfig": | |
| 1103 doCheckConfig(so) | |
| 1104 | |
| 1105 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build