| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- test-case-name: buildbot.test.test_dependencies -*- | |
| 2 | |
| 3 import time, os.path | |
| 4 | |
| 5 from zope.interface import implements | |
| 6 from twisted.internet import reactor | |
| 7 from twisted.application import service, internet, strports | |
| 8 from twisted.python import log, runtime | |
| 9 from twisted.protocols import basic | |
| 10 from twisted.cred import portal, checkers | |
| 11 from twisted.spread import pb | |
| 12 | |
| 13 from buildbot import interfaces, buildset, util, pbutil | |
| 14 from buildbot.status import builder | |
| 15 from buildbot.sourcestamp import SourceStamp | |
| 16 from buildbot.changes.maildir import MaildirService | |
| 17 from buildbot.process.properties import Properties | |
| 18 | |
| 19 | |
| 20 class BaseScheduler(service.MultiService, util.ComparableMixin): | |
| 21 """ | |
| 22 A Scheduler creates BuildSets and submits them to the BuildMaster. | |
| 23 | |
| 24 @ivar name: name of the scheduler | |
| 25 | |
| 26 @ivar properties: additional properties specified in this | |
| 27 scheduler's configuration | |
| 28 @type properties: Properties object | |
| 29 """ | |
| 30 implements(interfaces.IScheduler) | |
| 31 | |
| 32 def __init__(self, name, properties={}): | |
| 33 """ | |
| 34 @param name: name for this scheduler | |
| 35 | |
| 36 @param properties: properties to be propagated from this scheduler | |
| 37 @type properties: dict | |
| 38 """ | |
| 39 service.MultiService.__init__(self) | |
| 40 self.name = name | |
| 41 self.properties = Properties() | |
| 42 self.properties.update(properties, "Scheduler") | |
| 43 self.properties.setProperty("scheduler", name, "Scheduler") | |
| 44 | |
| 45 def __repr__(self): | |
| 46 # TODO: why can't id() return a positive number? %d is ugly. | |
| 47 return "<Scheduler '%s' at %d>" % (self.name, id(self)) | |
| 48 | |
| 49 def submitBuildSet(self, bs): | |
| 50 self.parent.submitBuildSet(bs) | |
| 51 | |
| 52 def addChange(self, change): | |
| 53 pass | |
| 54 | |
| 55 class BaseUpstreamScheduler(BaseScheduler): | |
| 56 implements(interfaces.IUpstreamScheduler) | |
| 57 | |
| 58 def __init__(self, name, properties={}): | |
| 59 BaseScheduler.__init__(self, name, properties) | |
| 60 self.successWatchers = [] | |
| 61 | |
| 62 def subscribeToSuccessfulBuilds(self, watcher): | |
| 63 self.successWatchers.append(watcher) | |
| 64 def unsubscribeToSuccessfulBuilds(self, watcher): | |
| 65 self.successWatchers.remove(watcher) | |
| 66 | |
| 67 def submitBuildSet(self, bs): | |
| 68 d = bs.waitUntilFinished() | |
| 69 d.addCallback(self.buildSetFinished) | |
| 70 BaseScheduler.submitBuildSet(self, bs) | |
| 71 | |
| 72 def buildSetFinished(self, bss): | |
| 73 if not self.running: | |
| 74 return | |
| 75 if bss.getResults() == builder.SUCCESS: | |
| 76 ss = bss.getSourceStamp() | |
| 77 for w in self.successWatchers: | |
| 78 w(ss) | |
| 79 | |
| 80 | |
| 81 class Scheduler(BaseUpstreamScheduler): | |
| 82 """The default Scheduler class will run a build after some period of time | |
| 83 called the C{treeStableTimer}, on a given set of Builders. It only pays | |
| 84 attention to a single branch. You can provide a C{fileIsImportant} | |
| 85 function which will evaluate each Change to decide whether or not it | |
| 86 should trigger a new build. | |
| 87 """ | |
| 88 | |
| 89 fileIsImportant = None | |
| 90 compare_attrs = ('name', 'treeStableTimer', 'builderNames', 'branch', | |
| 91 'fileIsImportant', 'properties', 'categories') | |
| 92 | |
| 93 def __init__(self, name, branch, treeStableTimer, builderNames, | |
| 94 fileIsImportant=None, properties={}, categories=None): | |
| 95 """ | |
| 96 @param name: the name of this Scheduler | |
| 97 @param branch: The branch name that the Scheduler should pay | |
| 98 attention to. Any Change that is not in this branch | |
| 99 will be ignored. It can be set to None to only pay | |
| 100 attention to the default branch. | |
| 101 @param treeStableTimer: the duration, in seconds, for which the tree | |
| 102 must remain unchanged before a build is | |
| 103 triggered. This is intended to avoid builds | |
| 104 of partially-committed fixes. | |
| 105 @param builderNames: a list of Builder names. When this Scheduler | |
| 106 decides to start a set of builds, they will be | |
| 107 run on the Builders named by this list. | |
| 108 | |
| 109 @param fileIsImportant: A callable which takes one argument (a Change | |
| 110 instance) and returns True if the change is | |
| 111 worth building, and False if it is not. | |
| 112 Unimportant Changes are accumulated until the | |
| 113 build is triggered by an important change. | |
| 114 The default value of None means that all | |
| 115 Changes are important. | |
| 116 | |
| 117 @param properties: properties to apply to all builds started from this | |
| 118 scheduler | |
| 119 @param categories: A list of categories of changes to accept | |
| 120 """ | |
| 121 | |
| 122 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 123 self.treeStableTimer = treeStableTimer | |
| 124 errmsg = ("The builderNames= argument to Scheduler must be a list " | |
| 125 "of Builder description names (i.e. the 'name' key of the " | |
| 126 "Builder specification dictionary)") | |
| 127 assert isinstance(builderNames, (list, tuple)), errmsg | |
| 128 for b in builderNames: | |
| 129 assert isinstance(b, str), errmsg | |
| 130 self.builderNames = builderNames | |
| 131 self.branch = branch | |
| 132 if fileIsImportant: | |
| 133 assert callable(fileIsImportant) | |
| 134 self.fileIsImportant = fileIsImportant | |
| 135 | |
| 136 self.importantChanges = [] | |
| 137 self.allChanges = [] | |
| 138 self.nextBuildTime = None | |
| 139 self.timer = None | |
| 140 self.categories = categories | |
| 141 | |
| 142 def listBuilderNames(self): | |
| 143 return self.builderNames | |
| 144 | |
| 145 def getPendingBuildTimes(self): | |
| 146 if self.nextBuildTime is not None: | |
| 147 return [self.nextBuildTime] | |
| 148 return [] | |
| 149 | |
| 150 def addChange(self, change): | |
| 151 if change.branch != self.branch: | |
| 152 log.msg("%s ignoring off-branch %s" % (self, change)) | |
| 153 return | |
| 154 if self.categories is not None and change.category not in self.categorie
s: | |
| 155 log.msg("%s ignoring non-matching categories %s" % (self, change)) | |
| 156 return | |
| 157 if not self.fileIsImportant: | |
| 158 self.addImportantChange(change) | |
| 159 elif self.fileIsImportant(change): | |
| 160 self.addImportantChange(change) | |
| 161 else: | |
| 162 self.addUnimportantChange(change) | |
| 163 | |
| 164 def addImportantChange(self, change): | |
| 165 log.msg("%s: change is important, adding %s" % (self, change)) | |
| 166 self.importantChanges.append(change) | |
| 167 self.allChanges.append(change) | |
| 168 self.nextBuildTime = max(self.nextBuildTime, | |
| 169 change.when + self.treeStableTimer) | |
| 170 self.setTimer(self.nextBuildTime) | |
| 171 | |
| 172 def addUnimportantChange(self, change): | |
| 173 log.msg("%s: change is not important, adding %s" % (self, change)) | |
| 174 self.allChanges.append(change) | |
| 175 | |
| 176 def setTimer(self, when): | |
| 177 log.msg("%s: setting timer to %s" % | |
| 178 (self, time.strftime("%H:%M:%S", time.localtime(when)))) | |
| 179 now = util.now() | |
| 180 if when < now: | |
| 181 when = now | |
| 182 if self.timer: | |
| 183 self.timer.cancel() | |
| 184 self.timer = reactor.callLater(when - now, self.fireTimer) | |
| 185 | |
| 186 def stopTimer(self): | |
| 187 if self.timer: | |
| 188 self.timer.cancel() | |
| 189 self.timer = None | |
| 190 | |
| 191 def fireTimer(self): | |
| 192 # clear out our state | |
| 193 self.timer = None | |
| 194 self.nextBuildTime = None | |
| 195 changes = self.allChanges | |
| 196 self.importantChanges = [] | |
| 197 self.allChanges = [] | |
| 198 | |
| 199 # create a BuildSet, submit it to the BuildMaster | |
| 200 bs = buildset.BuildSet(self.builderNames, | |
| 201 SourceStamp(changes=changes), | |
| 202 properties=self.properties) | |
| 203 self.submitBuildSet(bs) | |
| 204 | |
| 205 def stopService(self): | |
| 206 self.stopTimer() | |
| 207 return service.MultiService.stopService(self) | |
| 208 | |
| 209 | |
| 210 class AnyBranchScheduler(BaseUpstreamScheduler): | |
| 211 """This Scheduler will handle changes on a variety of branches. It will | |
| 212 accumulate Changes for each branch separately. It works by creating a | |
| 213 separate Scheduler for each new branch it sees.""" | |
| 214 | |
| 215 schedulerFactory = Scheduler | |
| 216 fileIsImportant = None | |
| 217 | |
| 218 compare_attrs = ('name', 'branches', 'treeStableTimer', 'builderNames', | |
| 219 'fileIsImportant', 'properties', 'categories') | |
| 220 | |
| 221 def __init__(self, name, branches, treeStableTimer, builderNames, | |
| 222 fileIsImportant=None, properties={}, categories=None): | |
| 223 """ | |
| 224 @param name: the name of this Scheduler | |
| 225 @param branches: The branch names that the Scheduler should pay | |
| 226 attention to. Any Change that is not in one of these | |
| 227 branches will be ignored. It can be set to None to | |
| 228 accept changes from any branch. Don't use [] (an | |
| 229 empty list), because that means we don't pay | |
| 230 attention to *any* branches, so we'll never build | |
| 231 anything. | |
| 232 @param treeStableTimer: the duration, in seconds, for which the tree | |
| 233 must remain unchanged before a build is | |
| 234 triggered. This is intended to avoid builds | |
| 235 of partially-committed fixes. | |
| 236 @param builderNames: a list of Builder names. When this Scheduler | |
| 237 decides to start a set of builds, they will be | |
| 238 run on the Builders named by this list. | |
| 239 | |
| 240 @param fileIsImportant: A callable which takes one argument (a Change | |
| 241 instance) and returns True if the change is | |
| 242 worth building, and False if it is not. | |
| 243 Unimportant Changes are accumulated until the | |
| 244 build is triggered by an important change. | |
| 245 The default value of None means that all | |
| 246 Changes are important. | |
| 247 | |
| 248 @param properties: properties to apply to all builds started from this | |
| 249 scheduler | |
| 250 @param categories: A list of categories of changes to accept | |
| 251 """ | |
| 252 | |
| 253 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 254 self.treeStableTimer = treeStableTimer | |
| 255 for b in builderNames: | |
| 256 assert isinstance(b, str) | |
| 257 self.builderNames = builderNames | |
| 258 self.branches = branches | |
| 259 if self.branches == []: | |
| 260 log.msg("AnyBranchScheduler %s: branches=[], so we will ignore " | |
| 261 "all branches, and never trigger any builds. Please set " | |
| 262 "branches=None to mean 'all branches'" % self) | |
| 263 # consider raising an exception here, to make this warning more | |
| 264 # prominent, but I can vaguely imagine situations where you might | |
| 265 # want to comment out branches temporarily and wouldn't | |
| 266 # appreciate it being treated as an error. | |
| 267 if fileIsImportant: | |
| 268 assert callable(fileIsImportant) | |
| 269 self.fileIsImportant = fileIsImportant | |
| 270 self.schedulers = {} # one per branch | |
| 271 self.categories = categories | |
| 272 | |
| 273 def __repr__(self): | |
| 274 return "<AnyBranchScheduler '%s'>" % self.name | |
| 275 | |
| 276 def listBuilderNames(self): | |
| 277 return self.builderNames | |
| 278 | |
| 279 def getPendingBuildTimes(self): | |
| 280 bts = [] | |
| 281 for s in self.schedulers.values(): | |
| 282 if s.nextBuildTime is not None: | |
| 283 bts.append(s.nextBuildTime) | |
| 284 return bts | |
| 285 | |
| 286 def buildSetFinished(self, bss): | |
| 287 # we don't care if a build has finished; one of the per-branch builders | |
| 288 # will take care of it, instead. | |
| 289 pass | |
| 290 | |
| 291 def addChange(self, change): | |
| 292 branch = change.branch | |
| 293 if self.branches is not None and branch not in self.branches: | |
| 294 log.msg("%s ignoring off-branch %s" % (self, change)) | |
| 295 return | |
| 296 if self.categories is not None and change.category not in self.categorie
s: | |
| 297 log.msg("%s ignoring non-matching categories %s" % (self, change)) | |
| 298 return | |
| 299 s = self.schedulers.get(branch) | |
| 300 if not s: | |
| 301 if branch: | |
| 302 name = self.name + "." + branch | |
| 303 else: | |
| 304 name = self.name + ".<default>" | |
| 305 s = self.schedulerFactory(name, branch, | |
| 306 self.treeStableTimer, | |
| 307 self.builderNames, | |
| 308 self.fileIsImportant) | |
| 309 s.successWatchers = self.successWatchers | |
| 310 s.setServiceParent(self) | |
| 311 s.properties = self.properties | |
| 312 # TODO: does this result in schedulers that stack up forever? | |
| 313 # When I make the persistify-pass, think about this some more. | |
| 314 self.schedulers[branch] = s | |
| 315 s.addChange(change) | |
| 316 | |
| 317 | |
| 318 class Dependent(BaseUpstreamScheduler): | |
| 319 """This scheduler runs some set of 'downstream' builds when the | |
| 320 'upstream' scheduler has completed successfully.""" | |
| 321 implements(interfaces.IDownstreamScheduler) | |
| 322 | |
| 323 compare_attrs = ('name', 'upstream', 'builderNames', 'properties') | |
| 324 | |
| 325 def __init__(self, name, upstream, builderNames, properties={}): | |
| 326 assert interfaces.IUpstreamScheduler.providedBy(upstream) | |
| 327 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 328 self.upstream_name = upstream.name | |
| 329 self.upstream = None | |
| 330 self.builderNames = builderNames | |
| 331 | |
| 332 def listBuilderNames(self): | |
| 333 return self.builderNames | |
| 334 | |
| 335 def getPendingBuildTimes(self): | |
| 336 # report the upstream's value | |
| 337 return self.findUpstreamScheduler().getPendingBuildTimes() | |
| 338 | |
| 339 def startService(self): | |
| 340 service.MultiService.startService(self) | |
| 341 self.upstream = self.findUpstreamScheduler() | |
| 342 self.upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt) | |
| 343 | |
| 344 def stopService(self): | |
| 345 d = service.MultiService.stopService(self) | |
| 346 self.upstream.unsubscribeToSuccessfulBuilds(self.upstreamBuilt) | |
| 347 self.upstream = None | |
| 348 return d | |
| 349 | |
| 350 def upstreamBuilt(self, ss): | |
| 351 bs = buildset.BuildSet(self.builderNames, ss, | |
| 352 properties=self.properties) | |
| 353 self.submitBuildSet(bs) | |
| 354 | |
| 355 def findUpstreamScheduler(self): | |
| 356 # find our *active* upstream scheduler (which may not be self.upstream!)
by name | |
| 357 upstream = None | |
| 358 for s in self.parent.allSchedulers(): | |
| 359 if s.name == self.upstream_name and interfaces.IUpstreamScheduler.pr
ovidedBy(s): | |
| 360 upstream = s | |
| 361 if not upstream: | |
| 362 log.msg("ERROR: Couldn't find upstream scheduler of name <%s>" % | |
| 363 self.upstream_name) | |
| 364 return upstream | |
| 365 | |
| 366 def checkUpstreamScheduler(self): | |
| 367 # if we don't already have an upstream, then there's nothing to worry ab
out | |
| 368 if not self.upstream: | |
| 369 return | |
| 370 | |
| 371 upstream = self.findUpstreamScheduler() | |
| 372 | |
| 373 # if it's already correct, we're good to go | |
| 374 if upstream is self.upstream: | |
| 375 return | |
| 376 | |
| 377 # otherwise, associate with the new upstream. We also keep listening | |
| 378 # to the old upstream, in case it's in the middle of a build | |
| 379 upstream.subscribeToSuccessfulBuilds(self.upstreamBuilt) | |
| 380 self.upstream = upstream | |
| 381 log.msg("Dependent <%s> connected to new Upstream <%s>" % | |
| 382 (self.name, up_name)) | |
| 383 | |
| 384 class Periodic(BaseUpstreamScheduler): | |
| 385 """Instead of watching for Changes, this Scheduler can just start a build | |
| 386 at fixed intervals. The C{periodicBuildTimer} parameter sets the number | |
| 387 of seconds to wait between such periodic builds. The first build will be | |
| 388 run immediately.""" | |
| 389 | |
| 390 # TODO: consider having this watch another (changed-based) scheduler and | |
| 391 # merely enforce a minimum time between builds. | |
| 392 | |
| 393 compare_attrs = ('name', 'builderNames', 'periodicBuildTimer', 'branch', 'pr
operties') | |
| 394 | |
| 395 def __init__(self, name, builderNames, periodicBuildTimer, | |
| 396 branch=None, properties={}): | |
| 397 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 398 self.builderNames = builderNames | |
| 399 self.periodicBuildTimer = periodicBuildTimer | |
| 400 self.branch = branch | |
| 401 self.reason = ("The Periodic scheduler named '%s' triggered this build" | |
| 402 % name) | |
| 403 self.timer = internet.TimerService(self.periodicBuildTimer, | |
| 404 self.doPeriodicBuild) | |
| 405 self.timer.setServiceParent(self) | |
| 406 | |
| 407 def listBuilderNames(self): | |
| 408 return self.builderNames | |
| 409 | |
| 410 def getPendingBuildTimes(self): | |
| 411 # TODO: figure out when self.timer is going to fire next and report | |
| 412 # that | |
| 413 return [] | |
| 414 | |
| 415 def doPeriodicBuild(self): | |
| 416 bs = buildset.BuildSet(self.builderNames, | |
| 417 SourceStamp(branch=self.branch), | |
| 418 self.reason, | |
| 419 properties=self.properties) | |
| 420 self.submitBuildSet(bs) | |
| 421 | |
| 422 | |
| 423 | |
| 424 class Nightly(BaseUpstreamScheduler): | |
| 425 """Imitate 'cron' scheduling. This can be used to schedule a nightly | |
| 426 build, or one which runs are certain times of the day, week, or month. | |
| 427 | |
| 428 Pass some subset of minute, hour, dayOfMonth, month, and dayOfWeek; each | |
| 429 may be a single number or a list of valid values. The builds will be | |
| 430 triggered whenever the current time matches these values. Wildcards are | |
| 431 represented by a '*' string. All fields default to a wildcard except | |
| 432 'minute', so with no fields this defaults to a build every hour, on the | |
| 433 hour. | |
| 434 | |
| 435 For example, the following master.cfg clause will cause a build to be | |
| 436 started every night at 3:00am:: | |
| 437 | |
| 438 s = Nightly('nightly', ['builder1', 'builder2'], hour=3, minute=0) | |
| 439 c['schedules'].append(s) | |
| 440 | |
| 441 This scheduler will perform a build each monday morning at 6:23am and | |
| 442 again at 8:23am:: | |
| 443 | |
| 444 s = Nightly('BeforeWork', ['builder1'], | |
| 445 dayOfWeek=0, hour=[6,8], minute=23) | |
| 446 | |
| 447 The following runs a build every two hours:: | |
| 448 | |
| 449 s = Nightly('every2hours', ['builder1'], hour=range(0, 24, 2)) | |
| 450 | |
| 451 And this one will run only on December 24th:: | |
| 452 | |
| 453 s = Nightly('SleighPreflightCheck', ['flying_circuits', 'radar'], | |
| 454 month=12, dayOfMonth=24, hour=12, minute=0) | |
| 455 | |
| 456 For dayOfWeek and dayOfMonth, builds are triggered if the date matches | |
| 457 either of them. All time values are compared against the tuple returned | |
| 458 by time.localtime(), so month and dayOfMonth numbers start at 1, not | |
| 459 zero. dayOfWeek=0 is Monday, dayOfWeek=6 is Sunday. | |
| 460 | |
| 461 onlyIfChanged functionality | |
| 462 s = Nightly('nightly', ['builder1', 'builder2'], | |
| 463 hour=3, minute=0, onlyIfChanged=True) | |
| 464 When the flag is True (False by default), the build is trigged if | |
| 465 the date matches and if the branch has changed | |
| 466 | |
| 467 fileIsImportant parameter is implemented as defined in class Scheduler | |
| 468 """ | |
| 469 | |
| 470 compare_attrs = ('name', 'builderNames', | |
| 471 'minute', 'hour', 'dayOfMonth', 'month', | |
| 472 'dayOfWeek', 'branch', 'onlyIfChanged', | |
| 473 'fileIsImportant', 'properties') | |
| 474 | |
| 475 def __init__(self, name, builderNames, minute=0, hour='*', | |
| 476 dayOfMonth='*', month='*', dayOfWeek='*', | |
| 477 branch=None, fileIsImportant=None, onlyIfChanged=False, propert
ies={}): | |
| 478 # Setting minute=0 really makes this an 'Hourly' scheduler. This | |
| 479 # seemed like a better default than minute='*', which would result in | |
| 480 # a build every 60 seconds. | |
| 481 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 482 self.builderNames = builderNames | |
| 483 self.minute = minute | |
| 484 self.hour = hour | |
| 485 self.dayOfMonth = dayOfMonth | |
| 486 self.month = month | |
| 487 self.dayOfWeek = dayOfWeek | |
| 488 self.branch = branch | |
| 489 self.onlyIfChanged = onlyIfChanged | |
| 490 self.delayedRun = None | |
| 491 self.nextRunTime = None | |
| 492 self.reason = ("The Nightly scheduler named '%s' triggered this build" | |
| 493 % name) | |
| 494 | |
| 495 self.importantChanges = [] | |
| 496 self.allChanges = [] | |
| 497 self.fileIsImportant = None | |
| 498 if fileIsImportant: | |
| 499 assert callable(fileIsImportant) | |
| 500 self.fileIsImportant = fileIsImportant | |
| 501 | |
| 502 def addTime(self, timetuple, secs): | |
| 503 return time.localtime(time.mktime(timetuple)+secs) | |
| 504 def findFirstValueAtLeast(self, values, value, default=None): | |
| 505 for v in values: | |
| 506 if v >= value: return v | |
| 507 return default | |
| 508 | |
| 509 def setTimer(self): | |
| 510 self.nextRunTime = self.calculateNextRunTime() | |
| 511 self.delayedRun = reactor.callLater(self.nextRunTime - time.time(), | |
| 512 self.doPeriodicBuild) | |
| 513 | |
| 514 def startService(self): | |
| 515 BaseUpstreamScheduler.startService(self) | |
| 516 self.setTimer() | |
| 517 | |
| 518 def stopService(self): | |
| 519 BaseUpstreamScheduler.stopService(self) | |
| 520 self.delayedRun.cancel() | |
| 521 | |
| 522 def isRunTime(self, timetuple): | |
| 523 def check(ourvalue, value): | |
| 524 if ourvalue == '*': return True | |
| 525 if isinstance(ourvalue, int): return value == ourvalue | |
| 526 return (value in ourvalue) | |
| 527 | |
| 528 if not check(self.minute, timetuple[4]): | |
| 529 #print 'bad minute', timetuple[4], self.minute | |
| 530 return False | |
| 531 | |
| 532 if not check(self.hour, timetuple[3]): | |
| 533 #print 'bad hour', timetuple[3], self.hour | |
| 534 return False | |
| 535 | |
| 536 if not check(self.month, timetuple[1]): | |
| 537 #print 'bad month', timetuple[1], self.month | |
| 538 return False | |
| 539 | |
| 540 if self.dayOfMonth != '*' and self.dayOfWeek != '*': | |
| 541 # They specified both day(s) of month AND day(s) of week. | |
| 542 # This means that we only have to match one of the two. If | |
| 543 # neither one matches, this time is not the right time. | |
| 544 if not (check(self.dayOfMonth, timetuple[2]) or | |
| 545 check(self.dayOfWeek, timetuple[6])): | |
| 546 #print 'bad day' | |
| 547 return False | |
| 548 else: | |
| 549 if not check(self.dayOfMonth, timetuple[2]): | |
| 550 #print 'bad day of month' | |
| 551 return False | |
| 552 | |
| 553 if not check(self.dayOfWeek, timetuple[6]): | |
| 554 #print 'bad day of week' | |
| 555 return False | |
| 556 | |
| 557 return True | |
| 558 | |
| 559 def calculateNextRunTime(self): | |
| 560 return self.calculateNextRunTimeFrom(time.time()) | |
| 561 | |
| 562 def calculateNextRunTimeFrom(self, now): | |
| 563 dateTime = time.localtime(now) | |
| 564 | |
| 565 # Remove seconds by advancing to at least the next minue | |
| 566 dateTime = self.addTime(dateTime, 60-dateTime[5]) | |
| 567 | |
| 568 # Now we just keep adding minutes until we find something that matches | |
| 569 | |
| 570 # It not an efficient algorithm, but it'll *work* for now | |
| 571 yearLimit = dateTime[0]+2 | |
| 572 while not self.isRunTime(dateTime): | |
| 573 dateTime = self.addTime(dateTime, 60) | |
| 574 #print 'Trying', time.asctime(dateTime) | |
| 575 assert dateTime[0] < yearLimit, 'Something is wrong with this code' | |
| 576 return time.mktime(dateTime) | |
| 577 | |
| 578 def listBuilderNames(self): | |
| 579 return self.builderNames | |
| 580 | |
| 581 def getPendingBuildTimes(self): | |
| 582 # TODO: figure out when self.timer is going to fire next and report | |
| 583 # that | |
| 584 if self.nextRunTime is None: return [] | |
| 585 return [self.nextRunTime] | |
| 586 | |
| 587 def doPeriodicBuild(self): | |
| 588 # Schedule the next run | |
| 589 self.setTimer() | |
| 590 | |
| 591 if self.onlyIfChanged: | |
| 592 if len(self.importantChanges) > 0: | |
| 593 changes = self.allChanges | |
| 594 # And trigger a build | |
| 595 log.msg("Nightly Scheduler <%s>: triggering build" % self.name) | |
| 596 bs = buildset.BuildSet(self.builderNames, | |
| 597 SourceStamp(changes=changes), | |
| 598 self.reason, | |
| 599 properties=self.properties) | |
| 600 self.submitBuildSet(bs) | |
| 601 # Reset the change lists | |
| 602 self.importantChanges = [] | |
| 603 self.allChanges = [] | |
| 604 else: | |
| 605 log.msg("Nightly Scheduler <%s>: skipping build - No important c
hange" % self.name) | |
| 606 else: | |
| 607 # And trigger a build | |
| 608 bs = buildset.BuildSet(self.builderNames, | |
| 609 SourceStamp(branch=self.branch), | |
| 610 self.reason, | |
| 611 properties=self.properties) | |
| 612 self.submitBuildSet(bs) | |
| 613 | |
| 614 def addChange(self, change): | |
| 615 if self.onlyIfChanged: | |
| 616 if change.branch != self.branch: | |
| 617 log.msg("Nightly Scheduler <%s>: ignoring change %s on off-branc
h %s" % (self.name, change.revision, change.branch)) | |
| 618 return | |
| 619 if not self.fileIsImportant: | |
| 620 self.addImportantChange(change) | |
| 621 elif self.fileIsImportant(change): | |
| 622 self.addImportantChange(change) | |
| 623 else: | |
| 624 self.addUnimportantChange(change) | |
| 625 else: | |
| 626 log.msg("Nightly Scheduler <%s>: no add change" % self.name) | |
| 627 pass | |
| 628 | |
| 629 def addImportantChange(self, change): | |
| 630 log.msg("Nightly Scheduler <%s>: change %s from %s is important, adding
it" % (self.name, change.revision, change.who)) | |
| 631 self.allChanges.append(change) | |
| 632 self.importantChanges.append(change) | |
| 633 | |
| 634 def addUnimportantChange(self, change): | |
| 635 log.msg("Nightly Scheduler <%s>: change %s from %s is not important, add
ing it" % (self.name, change.revision, change.who)) | |
| 636 self.allChanges.append(change) | |
| 637 | |
| 638 | |
| 639 class TryBase(BaseScheduler): | |
| 640 def __init__(self, name, builderNames, properties={}): | |
| 641 BaseScheduler.__init__(self, name, properties) | |
| 642 self.builderNames = builderNames | |
| 643 | |
| 644 def listBuilderNames(self): | |
| 645 return self.builderNames | |
| 646 | |
| 647 def getPendingBuildTimes(self): | |
| 648 # we can't predict what the developers are going to do in the future | |
| 649 return [] | |
| 650 | |
| 651 def addChange(self, change): | |
| 652 # Try schedulers ignore Changes | |
| 653 pass | |
| 654 | |
| 655 def processBuilderList(self, builderNames): | |
| 656 # self.builderNames is the configured list of builders | |
| 657 # available for try. If the user supplies a list of builders, | |
| 658 # it must be restricted to the configured list. If not, build | |
| 659 # on all of the configured builders. | |
| 660 if builderNames: | |
| 661 for b in builderNames: | |
| 662 if not b in self.builderNames: | |
| 663 log.msg("%s got with builder %s" % (self, b)) | |
| 664 log.msg(" but that wasn't in our list: %s" | |
| 665 % (self.builderNames,)) | |
| 666 return [] | |
| 667 else: | |
| 668 builderNames = self.builderNames | |
| 669 return builderNames | |
| 670 | |
| 671 class BadJobfile(Exception): | |
| 672 pass | |
| 673 | |
| 674 class JobFileScanner(basic.NetstringReceiver): | |
| 675 def __init__(self): | |
| 676 self.strings = [] | |
| 677 self.transport = self # so transport.loseConnection works | |
| 678 self.error = False | |
| 679 | |
| 680 def stringReceived(self, s): | |
| 681 self.strings.append(s) | |
| 682 | |
| 683 def loseConnection(self): | |
| 684 self.error = True | |
| 685 | |
| 686 class Try_Jobdir(TryBase): | |
| 687 compare_attrs = ( 'name', 'builderNames', 'jobdir', 'properties' ) | |
| 688 | |
| 689 def __init__(self, name, builderNames, jobdir, properties={}): | |
| 690 TryBase.__init__(self, name, builderNames, properties) | |
| 691 self.jobdir = jobdir | |
| 692 self.watcher = MaildirService() | |
| 693 self.watcher.setServiceParent(self) | |
| 694 | |
| 695 def setServiceParent(self, parent): | |
| 696 self.watcher.setBasedir(os.path.join(parent.basedir, self.jobdir)) | |
| 697 TryBase.setServiceParent(self, parent) | |
| 698 | |
| 699 def parseJob(self, f): | |
| 700 # jobfiles are serialized build requests. Each is a list of | |
| 701 # serialized netstrings, in the following order: | |
| 702 # "1", the version number of this format | |
| 703 # buildsetID, arbitrary string, used to find the buildSet later | |
| 704 # branch name, "" for default-branch | |
| 705 # base revision, "" for HEAD | |
| 706 # patchlevel, usually "1" | |
| 707 # patch | |
| 708 # builderNames... | |
| 709 p = JobFileScanner() | |
| 710 p.dataReceived(f.read()) | |
| 711 if p.error: | |
| 712 raise BadJobfile("unable to parse netstrings") | |
| 713 s = p.strings | |
| 714 ver = s.pop(0) | |
| 715 if ver != "1": | |
| 716 raise BadJobfile("unknown version '%s'" % ver) | |
| 717 buildsetID, branch, baserev, patchlevel, diff = s[:5] | |
| 718 builderNames = s[5:] | |
| 719 if branch == "": | |
| 720 branch = None | |
| 721 if baserev == "": | |
| 722 baserev = None | |
| 723 patchlevel = int(patchlevel) | |
| 724 patch = (patchlevel, diff) | |
| 725 ss = SourceStamp(branch, baserev, patch) | |
| 726 return builderNames, ss, buildsetID | |
| 727 | |
| 728 def messageReceived(self, filename): | |
| 729 md = os.path.join(self.parent.basedir, self.jobdir) | |
| 730 if runtime.platformType == "posix": | |
| 731 # open the file before moving it, because I'm afraid that once | |
| 732 # it's in cur/, someone might delete it at any moment | |
| 733 path = os.path.join(md, "new", filename) | |
| 734 f = open(path, "r") | |
| 735 os.rename(os.path.join(md, "new", filename), | |
| 736 os.path.join(md, "cur", filename)) | |
| 737 else: | |
| 738 # do this backwards under windows, because you can't move a file | |
| 739 # that somebody is holding open. This was causing a Permission | |
| 740 # Denied error on bear's win32-twisted1.3 buildslave. | |
| 741 os.rename(os.path.join(md, "new", filename), | |
| 742 os.path.join(md, "cur", filename)) | |
| 743 path = os.path.join(md, "cur", filename) | |
| 744 f = open(path, "r") | |
| 745 | |
| 746 try: | |
| 747 builderNames, ss, bsid = self.parseJob(f) | |
| 748 except BadJobfile: | |
| 749 log.msg("%s reports a bad jobfile in %s" % (self, filename)) | |
| 750 log.err() | |
| 751 return | |
| 752 # Validate/fixup the builder names. | |
| 753 builderNames = self.processBuilderList(builderNames) | |
| 754 if not builderNames: | |
| 755 return | |
| 756 reason = "'try' job" | |
| 757 bs = buildset.BuildSet(builderNames, ss, reason=reason, | |
| 758 bsid=bsid, properties=self.properties) | |
| 759 self.submitBuildSet(bs) | |
| 760 | |
| 761 class Try_Userpass(TryBase): | |
| 762 compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' ) | |
| 763 implements(portal.IRealm) | |
| 764 | |
| 765 def __init__(self, name, builderNames, port, userpass, properties={}): | |
| 766 TryBase.__init__(self, name, builderNames, properties) | |
| 767 if type(port) is int: | |
| 768 port = "tcp:%d" % port | |
| 769 self.port = port | |
| 770 self.userpass = userpass | |
| 771 c = checkers.InMemoryUsernamePasswordDatabaseDontUse() | |
| 772 for user,passwd in self.userpass: | |
| 773 c.addUser(user, passwd) | |
| 774 | |
| 775 p = portal.Portal(self) | |
| 776 p.registerChecker(c) | |
| 777 f = pb.PBServerFactory(p) | |
| 778 s = strports.service(port, f) | |
| 779 s.setServiceParent(self) | |
| 780 | |
| 781 def getPort(self): | |
| 782 # utility method for tests: figure out which TCP port we just opened. | |
| 783 return self.services[0]._port.getHost().port | |
| 784 | |
| 785 def requestAvatar(self, avatarID, mind, interface): | |
| 786 log.msg("%s got connection from user %s" % (self, avatarID)) | |
| 787 assert interface == pb.IPerspective | |
| 788 p = Try_Userpass_Perspective(self, avatarID) | |
| 789 return (pb.IPerspective, p, lambda: None) | |
| 790 | |
| 791 class Try_Userpass_Perspective(pbutil.NewCredPerspective): | |
| 792 def __init__(self, parent, username): | |
| 793 self.parent = parent | |
| 794 self.username = username | |
| 795 | |
| 796 def perspective_try(self, branch, revision, patch, builderNames, properties=
{}): | |
| 797 log.msg("user %s requesting build on builders %s" % (self.username, | |
| 798 builderNames)) | |
| 799 # Validate/fixup the builder names. | |
| 800 builderNames = self.parent.processBuilderList(builderNames) | |
| 801 if not builderNames: | |
| 802 return | |
| 803 ss = SourceStamp(branch, revision, patch) | |
| 804 reason = "'try' job from user %s" % self.username | |
| 805 | |
| 806 # roll the specified props in with our inherited props | |
| 807 combined_props = Properties() | |
| 808 combined_props.updateFromProperties(self.parent.properties) | |
| 809 combined_props.update(properties, "try build") | |
| 810 | |
| 811 bs = buildset.BuildSet(builderNames, | |
| 812 ss, | |
| 813 reason=reason, | |
| 814 properties=combined_props) | |
| 815 | |
| 816 self.parent.submitBuildSet(bs) | |
| 817 | |
| 818 # return a remotely-usable BuildSetStatus object | |
| 819 from buildbot.status.client import makeRemote | |
| 820 return makeRemote(bs.status) | |
| 821 | |
| 822 class Triggerable(BaseUpstreamScheduler): | |
| 823 """This scheduler doesn't do anything until it is triggered by a Trigger | |
| 824 step in a factory. In general, that step will not complete until all of | |
| 825 the builds that I fire have finished. | |
| 826 """ | |
| 827 | |
| 828 compare_attrs = ('name', 'builderNames', 'properties') | |
| 829 | |
| 830 def __init__(self, name, builderNames, properties={}): | |
| 831 BaseUpstreamScheduler.__init__(self, name, properties) | |
| 832 self.builderNames = builderNames | |
| 833 | |
| 834 def listBuilderNames(self): | |
| 835 return self.builderNames | |
| 836 | |
| 837 def getPendingBuildTimes(self): | |
| 838 return [] | |
| 839 | |
| 840 def trigger(self, ss, set_props=None): | |
| 841 """Trigger this scheduler. Returns a deferred that will fire when the | |
| 842 buildset is finished. | |
| 843 """ | |
| 844 | |
| 845 # properties for this buildset are composed of our own properties, | |
| 846 # potentially overridden by anything from the triggering build | |
| 847 props = Properties() | |
| 848 props.updateFromProperties(self.properties) | |
| 849 if set_props: props.updateFromProperties(set_props) | |
| 850 | |
| 851 bs = buildset.BuildSet(self.builderNames, ss, properties=props) | |
| 852 d = bs.waitUntilFinished() | |
| 853 self.submitBuildSet(bs) | |
| 854 return d | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 12207158:
Bye bye buildbot 0.7.12. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build