Delete obsolete masters from tools/buildbot

master.tryserver.webkit/master.cfg

-# -*- python -*-
-# ex: set syntax=python:
-
-# These modules come from scripts/master, which must be in the PYTHONPATH.
-# TODO(maruel): Upstream these to make it webkit.org-friendly.
-from builders_pools import BuildersPools
-from try_job_http import TryJobHTTP
-from try_mail_notifier import TryMailNotifier
-#from try_job_svn import TryJobSubversion
-
-
-c = BuildmasterConfig = {}
-
-from buildbot.buildslave import BuildSlave
-from buildbot.changes.pb import PBChangeSource
-from buildbot.scheduler import AnyBranchScheduler, Triggerable
-from buildbot.status import html
-from buildbot.process import buildstep, factory, properties
-from buildbot.steps import master, shell, source, transfer, trigger
-from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED
-
-from twisted.internet import defer
-
-import re
-import simplejson
-
-WithProperties = properties.WithProperties
-
-class ConfigureBuild(buildstep.BuildStep):
-    name = "configure build"
-    description = ["configuring build"]
-    descriptionDone = ["configured build"]
-    def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs):
-        buildstep.BuildStep.__init__(self, *args, **kwargs)
-        self.platform = platform.split('-', 1)[0]
-        self.fullPlatform = platform
-        self.configuration = configuration
-        self.architecture = architecture
-        self.buildOnly = buildOnly
-        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly)
-
-    def start(self):
-        self.setProperty("platform", self.platform)
-        self.setProperty("fullPlatform", self.fullPlatform)
-        self.setProperty("configuration", self.configuration)
-        self.setProperty("architecture", self.architecture)
-        self.setProperty("buildOnly", self.buildOnly)
-        self.finished(SUCCESS)
-        return defer.succeed(None)
-
-
-class CheckOutSource(source.SVN):
-    baseURL = "http://svn.webkit.org/repository/webkit/"
-    mode = "update"
-    def __init__(self, *args, **kwargs):
-        source.SVN.__init__(self, baseURL=self.baseURL, defaultBranch="trunk", mode=self.mode, *args, **kwargs)
-
-
-class InstallWin32Dependencies(shell.Compile):
-    description = ["installing dependencies"]
-    descriptionDone = ["installed dependencies"]
-    command = ["perl", "./WebKitTools/Scripts/update-webkit-auxiliary-libs"]
-
-
-class InstallChromiumDependencies(shell.ShellCommand):
-    name = "gclient"
-    description = ["updating chromium dependencies"]
-    descriptionDone = ["updated chromium dependencies"]
-    command = ["perl", "./WebKitTools/Scripts/update-webkit-chromium"]
-    haltOnFailure = True
-
-
-def appendCustomBuildFlags(step, platform):
-    if platform in ('gtk', 'wx', 'qt', 'chromium'):
-        step.setCommand(step.command + ['--' + platform])
-
-
-class CompileWebKit(shell.Compile):
-    command = ["perl", "./WebKitTools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
-    env = {'MFLAGS':''}
-    name = "compile-webkit"
-    description = ["compiling"]
-    descriptionDone = ["compiled"]
-    warningPattern = ".*arning: .*"
-
-    def start(self):
-        platform = self.getProperty('platform')
-        buildOnly = self.getProperty('buildOnly')
-        if platform == 'mac' and buildOnly:
-            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
-
-        appendCustomBuildFlags(self, platform)
-        return shell.Compile.start(self)
-
-
-class ArchiveBuiltProduct(shell.ShellCommand):
-    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
-               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
-    name = "archive-built-product"
-    description = ["archiving built product"]
-    descriptionDone = ["archived built product"]
-    haltOnFailure = True
-
-
-class ExtractBuiltProduct(shell.ShellCommand):
-    command = ["python", "./WebKitTools/BuildSlaveSupport/built-product-archive",
-               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "extract"]
-    name = "extract-built-product"
-    description = ["extracting built product"]
-    descriptionDone = ["extracted built product"]
-    haltOnFailure = True
-
-
-class UploadBuiltProduct(transfer.FileUpload):
-    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
-    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
-    haltOnFailure = True
-
-    def __init__(self):
-        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
-
-
-class DownloadBuiltProduct(transfer.FileDownload):
-    slavedest = WithProperties("WebKitBuild/%(configuration)s.zip")
-    mastersrc = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
-    haltOnFailure = True
-
-    def __init__(self):
-        transfer.FileDownload.__init__(self, self.mastersrc, self.slavedest)
-
-
-class RunJavaScriptCoreTests(shell.Test):
-    name = "jscore-test"
-    description = ["jscore-tests running"]
-    descriptionDone = ["jscore-tests"]
-    command = ["perl", "./WebKitTools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
-    logfiles = {'results': 'JavaScriptCore/tests/mozilla/actual.html'}
-
-    def __init__(self, skipBuild=False, *args, **kwargs):
-        self.skipBuild = skipBuild
-        shell.Test.__init__(self, *args, **kwargs)
-        self.addFactoryArguments(skipBuild=skipBuild)
-
-    def start(self):
-        appendCustomBuildFlags(self, self.getProperty('platform'))
-        if self.skipBuild:
-            self.setCommand(self.command + ['--skip-build'])
-        return shell.Test.start(self)
-
-    def commandComplete(self, cmd):
-        shell.Test.commandComplete(self, cmd)
-
-        logText = cmd.logs['stdio'].getText()
-        statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
-        if statusLines and statusLines[0].split()[0] != '0':
-            self.regressionLine = statusLines[0]
-        else:
-            self.regressionLine = None
-
-    def evaluateCommand(self, cmd):
-        if self.regressionLine:
-            return FAILURE
-
-        if cmd.rc != 0:
-            return FAILURE
-
-        return SUCCESS
-
-    def getText(self, cmd, results):
-        return self.getText2(cmd, results)
-
-    def getText2(self, cmd, results):
-        if results != SUCCESS and self.regressionLine:
-            return [self.name, self.regressionLine]
-
-        return [self.name]
-
-
-class RunWebKitTests(shell.Test):
-    name = "layout-test"
-    description = ["layout-tests running"]
-    descriptionDone = ["layout-tests"]
-    command = ["perl", "./WebKitTools/Scripts/run-webkit-tests", "--no-launch-safari", "--no-new-test-results",
-               "--no-sample-on-timeout", "--results-directory", "layout-test-results", "--use-remote-links-to-tests",
-               WithProperties("--%(configuration)s"), "--exit-after-n-failures", "20"]
-
-    def __init__(self, skipBuild=False, *args, **kwargs):
-        self.skipBuild = skipBuild
-        shell.Test.__init__(self, *args, **kwargs)
-        self.addFactoryArguments(skipBuild=skipBuild)
-
-    def start(self):
-        appendCustomBuildFlags(self, self.getProperty('platform'))
-        if self.skipBuild:
-            self.setCommand(self.command + ['--root=WebKitBuild/bin'])
-        return shell.Test.start(self)
-
-    def commandComplete(self, cmd):
-        shell.Test.commandComplete(self, cmd)
-
-        logText = cmd.logs['stdio'].getText()
-        incorrectLayoutLines = []
-        for line in logText.splitlines():
-            if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
-                incorrectLayoutLines.append(line)
-            elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
-                incorrectLayoutLines.append(line)
-            elif line.startswith("WARNING:") and line.find(' leak') >= 0:
-                incorrectLayoutLines.append(line.replace('WARNING: ', ''))
-            elif line.find('Exiting early') >= 0:
-                incorrectLayoutLines.append(line)
-
-            # FIXME: Detect and summarize leaks of RefCounted objects
-
-        self.incorrectLayoutLines = incorrectLayoutLines
-
-    def evaluateCommand(self, cmd):
-        if self.incorrectLayoutLines:
-            if len(self.incorrectLayoutLines) == 1:
-                line = self.incorrectLayoutLines[0]
-                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
-                    return WARNINGS
-
-            return FAILURE
-
-        if cmd.rc != 0:
-            return FAILURE
-
-        return SUCCESS
-
-    def getText(self, cmd, results):
-        return self.getText2(cmd, results)
-
-    def getText2(self, cmd, results):
-        if results != SUCCESS and self.incorrectLayoutLines:
-            return self.incorrectLayoutLines
-
-        return [self.name]
-
-
-class RunWebKitLeakTests(RunWebKitTests):
-    def start(self):
-        self.setCommand(self.command + ["--leaks"])
-        return RunWebKitTests.start(self)
-
-
-class ArchiveTestResults(shell.ShellCommand):
-    command = ["python", "./WebKitTools/BuildSlaveSupport/test-result-archive",
-               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
-    name = "archive-test-results"
-    description = ["archiving test results"]
-    descriptionDone = ["archived test results"]
-    haltOnFailure = True
-
-
-class UploadTestResults(transfer.FileUpload):
-    slavesrc = "layout-test-results.zip"
-    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
-
-    def __init__(self):
-        transfer.FileUpload.__init__(self, self.slavesrc, self.masterdest)
-
-
-class ExtractTestResults(master.MasterShellCommand):
-    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
-    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
-
-    def __init__(self):
-        master.MasterShellCommand.__init__(self, "")
-
-    def start(self):
-        self.command = ["ditto", "-k", "-x", "-V", self.build.getProperties().render(self.zipFile), self.build.getProperties().render(self.resultDirectory)]
-        return master.MasterShellCommand.start(self)
-
-    def finished(self, result):
-        url = self.build.getProperties().render(self.resultDirectory).replace("public_html/", "")
-        self.addURL("view results", url)
-        result = master.MasterShellCommand.finished(self, result)
-        self.step_status.setText(["uploaded results"])
-        return result
-
-
-class Factory(factory.BuildFactory):
-    def __init__(self, platform, configuration, architectures, buildOnly):
-        factory.BuildFactory.__init__(self)
-        self.addStep(ConfigureBuild, platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly)
-        self.addStep(CheckOutSource)
-        if platform == "win":
-            self.addStep(InstallWin32Dependencies)
-        if platform == "chromium":
-            self.addStep(InstallChromiumDependencies)
-
-class BuildFactory(Factory):
-    def __init__(self, platform, configuration, architectures, triggers=None):
-        Factory.__init__(self, platform, configuration, architectures, True)
-        self.addStep(CompileWebKit)
-        if triggers:
-            self.addStep(ArchiveBuiltProduct)
-            self.addStep(UploadBuiltProduct)
-            self.addStep(trigger.Trigger, schedulerNames=triggers)
-
-class TestFactory(Factory):
-    def __init__(self, platform, configuration, architectures):
-        Factory.__init__(self, platform, configuration, architectures, False)
-        self.addStep(DownloadBuiltProduct)
-        self.addStep(ExtractBuiltProduct)
-        self.addStep(RunJavaScriptCoreTests, skipBuild=True)
-        self.addStep(RunWebKitTests, skipBuild=(platform == 'win'))
-        self.addStep(ArchiveTestResults)
-        self.addStep(UploadTestResults)
-        self.addStep(ExtractTestResults)
-
-class BuildAndTestFactory(Factory):
-    TestClass = RunWebKitTests
-    def __init__(self, platform, configuration, architectures):
-        Factory.__init__(self, platform, configuration, architectures, False)
-        self.addStep(CompileWebKit)
-        self.addStep(RunJavaScriptCoreTests)
-        self.addStep(self.TestClass)
-        self.addStep(ArchiveTestResults)
-        self.addStep(UploadTestResults)
-        self.addStep(ExtractTestResults)
-
-class BuildAndTestLeaksFactory(BuildAndTestFactory):
-    TestClass = RunWebKitLeakTests
-
-
-class TryFactory(Factory):
-    """Similar to BuildAndTestFactory but doesn't archive the results."""
-    TestClass = RunWebKitTests
-    def __init__(self, platform, configuration, architectures):
-        Factory.__init__(self, platform, configuration, architectures, False)
-        self.addStep(CompileWebKit)
-        self.addStep(RunJavaScriptCoreTests)
-        self.addStep(self.TestClass)
-
-
-class TryBuildFactory(BuildFactory):
-    """Same as BuildFactory."""
-    pass
-
-
-class TryLeaksFactory(TryFactory):
-    TestClass = RunWebKitLeakTests
-
-
-def convertToString(json_data):
-    """Converts unicode strings to ascii strings in dictionaries and lists.
-
-    Some simplejson version will always return unicode strings. Python
-    doesn't accept unicode strings as dictionary key for kwargs so convert
-    them."""
-    if isinstance(json_data, dict):
-        retval = {}
-        for (k,v) in json_data.iteritems():
-            retval[str(k)] = convertToString(v)
-        return retval
-    elif isinstance(json_data, list):
-        retval = []
-        for i in json_data:
-            retval.append(convertToString(i))
-        return retval
-    elif isinstance(json_data, unicode):
-        return str(json_data)
-    else:
-        return json_data
-
-
-def loadBuilderConfig(c, config_file, passwords_file):
-    passwords = convertToString(simplejson.load(open(passwords_file)))
-
-    config = convertToString(simplejson.load(open(config_file)))
-
-    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
-
-    c['schedulers'] = []
-    for scheduler in config['schedulers']:
-        kls = globals()[scheduler.pop('type')]
-        c['schedulers'].append(kls(**scheduler))
-
-    c['builders'] = []
-    for builder in config['builders']:
-        for slaveName in builder['slavenames']:
-            for slave in config['slaves']:
-                if slave['name'] != slaveName or slave['platform'] == '*':
-                    continue
-
-                if slave['platform'] != builder['platform']:
-                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
-
-                break
-
-        factory = globals()["%sFactory" % builder.pop('type')]
-        factoryArgs = []
-        for key in "platform", "configuration", "architectures", "triggers":
-            value = builder.pop(key, None)
-            if value:
-                factoryArgs.append(value)
-
-        builder["factory"] = factory(*factoryArgs)
-
-        c['builders'].append(builder)
-
-
-loadBuilderConfig(c, 'config.json', 'passwords.json')
-
-c['change_source'] = PBChangeSource()
-pools = BuildersPools('webkit')
-#pools['webkit'].append("Try Apple Leopard Intel Release")
-pools['webkit'].append("Try Apple Leopard Intel Debug")
-#pools['webkit'].append("Try Apple SnowLeopard Intel Release")
-#pools['webkit'].append("Try Apple SnowLeopard Intel Leaks")
-#pools['webkit'].append("Try Apple Windows Release")
-pools['webkit'].append("Try Apple Windows Debug")
-#pools['webkit'].append("Try GTK Linux Release")
-#pools['webkit'].append("Try Qt Linux Release")
-pools['webkit'].append("Try Chromium Win Release")
-pools['webkit'].append("Try Chromium Mac Release")
-pools['webkit'].append("Try Chromium Linux Release")
-c['schedulers'].append(TryJobHTTP(
-      name='try_job_http',
-      port=8017,
-      pools=pools))
-
-c['status'] = []
-c['status'].append(html.WebStatus(http_port=8010, allowForce=True))
-c['status'].append(html.WebStatus(http_port=8044, allowForce=False))
-
-def GetSmtp():
-    # TODO(maruel): Remove me.
-    import chromium_config_private
-    return chromium_config_private.Master.smtp
-
-c['status'].append(TryMailNotifier(
-    fromaddr='tryserver@chromium.org',
-    subject="try %(result)s for %(reason)s on %(builder)s @ r%(revision)s",
-    mode='all',
-    relayhost=GetSmtp(),
-    lookup=''))
-
-
-c['slavePortnum'] = 8031
-c['projectName'] = "WebKit"
-c['projectURL'] = "http://webkit.org"
-c['buildbotURL'] = "http://build.chromium.org/buildbot/webkit-try/"
-
-# Keep last try jobs, the default is too low. Must keep at least a few days
-# worth of try jobs.
-c['buildHorizon'] = 3000
-c['logHorizon'] = 3000
-# Must be at least 2x the number of slaves.
-c['eventHorizon'] = 200
-# Must be at least 2x the number of on-going builds.
-c['buildCacheSize'] = 200
-