path: root/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg

# -*- python -*-
# ex: set syntax=python:

from buildbot.buildslave import BuildSlave
from buildbot.changes.pb import PBChangeSource
from buildbot.scheduler import AnyBranchScheduler, Triggerable
from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter
from buildbot.schedulers.filter import ChangeFilter
from buildbot.status import html
from buildbot.status.web.authz import Authz
from buildbot.process import buildstep, factory, properties
from buildbot.steps import master, shell, source, transfer, trigger
from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED

from twisted.internet import defer

import os
import re
import json
import operator
import urllib

from committer_auth import CommitterAuth
from webkitpy.common.config import build as wkbuild


c = BuildmasterConfig = {}

c['change_source'] = PBChangeSource()

# permissions for WebStatus
authz = Authz(
    auth=CommitterAuth('auth.json'),
    forceBuild='auth',
    forceAllBuilds='auth',
    pingBuilder=True,
    gracefulShutdown=False,
    stopBuild='auth',
    stopAllBuilds='auth',
    cancelPendingBuild='auth',
    stopChange=True,
    cleanShutdown=False)

c['status'] = []
c['status'].append(html.WebStatus(http_port=8710, 
                                  revlink="http://trac.webkit.org/changeset/%s", 
                                  authz=authz))

c['slavePortnum'] = 17000
c['projectName'] = "WebKit"
c['projectURL'] = "http://webkit.org"
c['buildbotURL'] = "http://build.webkit.org/"

c['buildHorizon'] = 1000
c['logHorizon'] = 500
c['eventHorizon'] = 200
c['buildCacheSize'] = 60

WithProperties = properties.WithProperties


class TestWithFailureCount(shell.Test):
    failedTestsFormatString = "%d tests failed"

    def countFailures(self, cmd):
        return 0

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)
        self.failedTestCount = self.countFailures(cmd)

    def evaluateCommand(self, cmd):
        if self.failedTestCount:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.failedTestCount:
            return [self.failedTestsFormatString % self.failedTestCount]

        return [self.name]


class ConfigureBuild(buildstep.BuildStep):
    name = "configure build"
    description = ["configuring build"]
    descriptionDone = ["configured build"]
    def __init__(self, platform, configuration, architecture, buildOnly, SVNMirror, *args, **kwargs):
        buildstep.BuildStep.__init__(self, *args, **kwargs)
        self.platform = platform.split('-', 1)[0]
        self.fullPlatform = platform
        self.configuration = configuration
        self.architecture = architecture
        self.buildOnly = buildOnly
        self.SVNMirror = SVNMirror
        self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, SVNMirror=SVNMirror)

    def start(self):
        self.setProperty("platform", self.platform)
        self.setProperty("fullPlatform", self.fullPlatform)
        self.setProperty("configuration", self.configuration)
        self.setProperty("architecture", self.architecture)
        self.setProperty("buildOnly", self.buildOnly)
        self.setProperty("shouldAbortEarly", True)
        self.setProperty("SVNMirror", self.SVNMirror)
        self.finished(SUCCESS)
        return defer.succeed(None)


class CheckOutSource(source.SVN):
    mode = "update"
    def __init__(self, SVNMirror, **kwargs):
        kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
        kwargs['defaultBranch'] = "trunk"
        kwargs['mode'] = self.mode
        source.SVN.__init__(self, **kwargs)
        self.addFactoryArguments(SVNMirror=SVNMirror)

class WaitForSVNServer(shell.ShellCommand):
    name = "wait-for-svn-server"
    command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
    description = ["waiting for SVN server"]
    descriptionDone = ["SVN server is ready"]
    haltOnFailure = True

class InstallWin32Dependencies(shell.Compile):
    description = ["installing dependencies"]
    descriptionDone = ["installed dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]

class KillOldProcesses(shell.Compile):
    name = "kill old processes"
    description = ["killing old processes"]
    descriptionDone = ["killed old processes"]
    command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes"]

class InstallEflDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating efl dependencies"]
    descriptionDone = ["updated efl dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitefl-libs"]
    haltOnFailure = True

class InstallGtkDependencies(shell.ShellCommand):
    name = "jhbuild"
    description = ["updating gtk dependencies"]
    descriptionDone = ["updated gtk dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
    haltOnFailure = True

class InstallChromiumDependencies(shell.ShellCommand):
    name = "gclient"
    description = ["updating chromium dependencies"]
    descriptionDone = ["updated chromium dependencies"]
    command = ["perl", "./Tools/Scripts/update-webkit-chromium", "--force"]
    haltOnFailure = True
    def start(self):
        if self.getProperty('fullPlatform') == "chromium-android":
            self.setCommand(self.command + ['--chromium-android'])

        return shell.ShellCommand.start(self)

class CleanupChromiumCrashLogs(shell.ShellCommand):
    name = "cleanup crash logs"
    description = ["removing crash logs"]
    descriptionDone = ["removed crash logs"]
    command = ["python", "./Tools/BuildSlaveSupport/chromium/remove-crash-logs"]
    haltOnFailure = False


def appendCustomBuildFlags(step, platform, fullPlatform=""):
    if fullPlatform == "chromium-android":
        step.setCommand(step.command + ['--chromium-android'])
    elif platform in ('chromium', 'efl', 'gtk', 'qt', 'wincairo', 'wince', 'wx'):
        step.setCommand(step.command + ['--' + platform])


class CompileWebKit(shell.Compile):
    command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
    env = {'MFLAGS':''}
    name = "compile-webkit"
    description = ["compiling"]
    descriptionDone = ["compiled"]
    warningPattern = ".*arning: .*"

    def start(self):
        platform = self.getProperty('platform')
        buildOnly = self.getProperty('buildOnly')
        if platform == 'mac' and buildOnly:
            self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])

        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))

        return shell.Compile.start(self)


class ArchiveBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-built-product"
    description = ["archiving built product"]
    descriptionDone = ["archived built product"]
    haltOnFailure = True


class ExtractBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
               WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
    name = "extract-built-product"
    description = ["extracting built product"]
    descriptionDone = ["extracted built product"]
    haltOnFailure = True


class UploadBuiltProduct(transfer.FileUpload):
    slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
    masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
    haltOnFailure = True

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        transfer.FileUpload.__init__(self, **kwargs)


class DownloadBuiltProduct(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
        WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
        WithProperties(c["buildbotURL"] + "archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
    name = "download-built-product"
    description = ["downloading built product"]
    descriptionDone = ["downloaded built product"]
    haltOnFailure = True
    flunkOnFailure = True


class RunJavaScriptCoreTests(shell.Test):
    name = "jscore-test"
    description = ["jscore-tests running"]
    descriptionDone = ["jscore-tests"]
    command = ["perl", "./Tools/Scripts/run-javascriptcore-tests", WithProperties("--%(configuration)s")]
    logfiles = {'actual.html (source)': 'Source/JavaScriptCore/tests/mozilla/actual.html'}

    def __init__(self, buildJSCTool=True, *args, **kwargs):
        self.buildJSCTool = buildJSCTool
        shell.Test.__init__(self, *args, **kwargs)
        self.addFactoryArguments(buildJSCTool=buildJSCTool)

    def start(self):
        appendCustomBuildFlags(self, self.getProperty('platform'))
        if not self.buildJSCTool:
            self.setCommand(self.command + ['--no-build'])
        return shell.Test.start(self)

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        statusLines = [line for line in logText.splitlines() if line.find('regression') >= 0 and line.find(' found.') >= 0]
        if statusLines and statusLines[0].split()[0] != '0':
            self.regressionLine = statusLines[0]
        else:
            self.regressionLine = None

        if 'actual.html (source)' in cmd.logs:
            self.addHTMLLog('actual.html', cmd.logs['actual.html (source)'].getText())

    def evaluateCommand(self, cmd):
        if self.regressionLine:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.regressionLine:
            return [self.name, self.regressionLine]

        return [self.name]


class RunWebKitTests(shell.Test):
    name = "layout-test"
    description = ["layout-tests running"]
    descriptionDone = ["layout-tests"]
    command = ["perl", "./Tools/Scripts/run-webkit-tests",
               "--no-launch-safari",
               "--no-new-test-results",
               "--no-sample-on-timeout",
               "--results-directory", "layout-test-results",
               "--use-remote-links-to-tests",
               "--builder-name", WithProperties("%(buildername)s"),
               "--build-number", WithProperties("%(buildnumber)s"),
               "--master-name", "webkit.org",
               "--test-results-server", "test-results.appspot.com",
               WithProperties("--%(configuration)s")]

    def __init__(self, buildJSCTool=True, *args, **kwargs):
        self.buildJSCTool = buildJSCTool
        shell.Test.__init__(self, *args, **kwargs)
        self.addFactoryArguments(buildJSCTool=buildJSCTool)

    def start(self):
        platform = self.getProperty('platform')
        shouldAbortEarly = self.getProperty('shouldAbortEarly')
        appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
        if platform.startswith('mac'):
            self.setCommand(self.command + ['--no-build'])
        if shouldAbortEarly:
            self.setCommand(self.command + ["--exit-after-n-crashes-or-timeouts", "20", "--exit-after-n-failures", "500"])
            
        if platform == "win":
            rootArgument = ['--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin")]
            self.setCommand(self.command + ['--no-build'])
        else:
            rootArgument = ['--root=WebKitBuild/bin']
        if not self.buildJSCTool:
            self.setCommand(self.command + rootArgument)
        return shell.Test.start(self)

    def _parseOldRunWebKitTestsOutput(self, logText):
        incorrectLayoutLines = []
        for line in logText.splitlines():
            if line.find('had incorrect layout') >= 0 or line.find('were new') >= 0 or line.find('was new') >= 0:
                incorrectLayoutLines.append(line)
            elif line.find('test case') >= 0 and (line.find(' crashed') >= 0 or line.find(' timed out') >= 0):
                incorrectLayoutLines.append(line)
            elif line.startswith("WARNING:") and line.find(' leak') >= 0:
                incorrectLayoutLines.append(line.replace('WARNING: ', ''))
            elif line.find('Exiting early') >= 0:
                incorrectLayoutLines.append(line)

            # FIXME: Detect and summarize leaks of RefCounted objects

        self.incorrectLayoutLines = incorrectLayoutLines

    # FIXME: This will break if new-run-webkit-tests changes its default log formatter.
    nrwt_log_message_regexp = re.compile(r'(?P<log_prefix>.*) (?P<log_level>DEBUG|INFO) (?P<message>.*)')

    def _strip_python_logging_prefix(self, line):
        match_object = self.nrwt_log_message_regexp.match(line)
        if match_object:
            return match_object.group('message')
        return line

    def _parseNewRunWebKitTestsOutput(self, logText):
        incorrectLayoutLines = []
        expressions = [
            ('flakes', re.compile(r'[Uu]nexpected flakiness.+:?\s*\((\d+)\)')),
            ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
            ('missing results', re.compile(r'(?:no expected results found|missing results)\s*:\s+\((\d+)\)')),
            ('failures', re.compile(r'Regressions: [Uu]nexpected.+:?\s*\((\d+)\)')),
        ]
        testFailures = {}

        for line in logText.splitlines():
            if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
                incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
                continue
            for name, expression in expressions:
                match = expression.search(line)

                if match:
                    testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
                    break

                # FIXME: Parse file names and put them in results

        for name in testFailures:
            incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)

        self.incorrectLayoutLines = incorrectLayoutLines

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        if logText.find("Collecting tests ...") >= 0:
            self._parseNewRunWebKitTestsOutput(logText)
        else:
            self._parseOldRunWebKitTestsOutput(logText)

    def evaluateCommand(self, cmd):
        result = SUCCESS

        if self.incorrectLayoutLines:
            if len(self.incorrectLayoutLines) == 1:
                line = self.incorrectLayoutLines[0]
                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
                    return WARNINGS

            for line in self.incorrectLayoutLines:
                if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
                    result = WARNINGS
                else:
                    return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return result

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectLayoutLines:
            return self.incorrectLayoutLines

        return [self.name]


class RunUnitTests(TestWithFailureCount):
    name = "run-api-tests"
    description = ["unit tests running"]
    descriptionDone = ["unit-tests"]
    command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
    failedTestsFormatString = "%d unit tests failed or timed out"

    def start(self):
        platform = self.getProperty('fullPlatform')
        if platform.startswith('win'):
            self.setCommand(self.command + ['--no-build'])
        if platform.startswith('mac'):
            self.setCommand(self.command + ['--no-build'])
        if platform.startswith('chromium'):
            self.setCommand(self.command + ['--chromium'])
        if platform == 'chromium-android':
            self.setCommand(self.command + ['--chromium-android'])
        return shell.Test.start(self)

    def countFailures(self, cmd):
        log_text = cmd.logs['stdio'].getText()
        count = 0

        split = re.split(r'\sTests that timed out:\s', log_text)
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        split = re.split(r'\sTests that failed:\s', split[0])
        if len(split) > 1:
            count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))

        return count


class RunPythonTests(TestWithFailureCount):
    name = "webkitpy-test"
    description = ["python-tests running"]
    descriptionDone = ["python-tests"]
    command = ["python", "./Tools/Scripts/test-webkitpy"]
    failedTestsFormatString = "%d python tests failed"

    def start(self):
        platform = self.getProperty('platform')
        # Python tests are flaky on the GTK builders, running them serially
        # helps and does not significantly prolong the cycle time.
        if platform == 'gtk':
            self.setCommand(self.command + ['--child-processes', '1'])
        return shell.Test.start(self)

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
        regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
        return 0


class RunPerlTests(TestWithFailureCount):
    name = "webkitperl-test"
    description = ["perl-tests running"]
    descriptionDone = ["perl-tests"]
    command = ["perl", "./Tools/Scripts/test-webkitperl"]
    failedTestsFormatString = "%d perl tests failed"

    def countFailures(self, cmd):
        logText = cmd.logs['stdio'].getText()
        # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
        regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
        for line in logText.splitlines():
            match = regex.match(line)
            if not match:
                continue
            return int(match.group('count'))
        return 0


class RunBindingsTests(shell.Test):
    name = "bindings-generation-tests"
    description = ["bindings-tests running"]
    descriptionDone = ["bindings-tests"]
    command = ["python", "./Tools/Scripts/run-bindings-tests"]


class RunEflAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["perl", "./Tools/Scripts/run-efl-tests", WithProperties("--%(configuration)s")]


class RunGtkAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        incorrectLines = []
        for line in logText.splitlines():
            if line.startswith('ERROR'):
                incorrectLines.append(line)

        self.incorrectLines = incorrectLines

    def evaluateCommand(self, cmd):
        if self.incorrectLines:
            return FAILURE

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectLines:
            return ["%d API tests failed" % len(self.incorrectLines)]

        return [self.name]

class RunQtAPITests(shell.Test):
    name = "API tests"
    description = ["API tests running"]
    descriptionDone = ["API tests"]
    command = ["python", "./Tools/Scripts/run-qtwebkit-tests",
               "--output-file=qt-unit-tests.html", "--do-not-open-results", "--timeout=120",
               WithProperties("WebKitBuild/%(configuration_pretty)s/Source/WebKit/qt/tests/", configuration_pretty=lambda build: build.getProperty("configuration").title())
                ]

    def commandComplete(self, cmd):
        shell.Test.commandComplete(self, cmd)

        logText = cmd.logs['stdio'].getText()
        foundItems = re.findall("TOTALS: (?P<passed>\d+) passed, (?P<failed>\d+) failed, (?P<skipped>\d+) skipped, (?P<crashed>\d+) crashed", logText)

        self.incorrectTests = 0
        self.crashedTests = 0
        self.statusLine = []

        if foundItems:
            self.incorrectTests = int(foundItems[0][1])
            self.crashedTests = int(foundItems[0][3])

            if self.incorrectTests > 0 or self.crashedTests > 0:
                self.statusLine = [
                    "%s passed, %s failed, %s skipped, %s crashed" % (foundItems[0][0], foundItems[0][1], foundItems[0][2], foundItems[0][3])
                ]

    def evaluateCommand(self, cmd):
        if self.crashedTests:
            return FAILURE

        if re.findall("Timeout, process", cmd.logs['stdio'].getText()):
            self.statusLine = ["Failure: timeout occured during testing"]
            return FAILURE

        if self.incorrectTests:
            return WARNINGS

        if cmd.rc != 0:
            return FAILURE

        return SUCCESS

    def getText(self, cmd, results):
        return self.getText2(cmd, results)

    def getText2(self, cmd, results):
        if results != SUCCESS and self.incorrectTests:
            return self.statusLine

        return [self.name]

class RunWebKitLeakTests(RunWebKitTests):
    warnOnWarnings = True
    def start(self):
        self.setCommand(self.command + ["--leaks"])
        return RunWebKitTests.start(self)


class RunWebKit2Tests(RunWebKitTests):
    def start(self):
        self.setProperty("shouldAbortEarly", False)
        self.setCommand(self.command + ["--webkit-test-runner"])
        return RunWebKitTests.start(self)


class RunChromiumWebKitUnitTests(shell.Test):
    name = "webkit-unit-tests"
    description = ["webkit-unit-tests running"]
    descriptionDone = ["webkit-unit-tests"]
    command = ["perl", "./Tools/Scripts/run-chromium-webkit-unit-tests",
               WithProperties("--%(configuration)s"), WithProperties("--platform=%(fullPlatform)s")]


class RunAndUploadPerfTests(shell.Test):
   name = "perf-test"
   description = ["perf-tests running"]
   descriptionDone = ["perf-tests"]
   command = ["python", "./Tools/Scripts/run-perf-tests",
              "--output-json-path", "perf-test-results.json",
              "--source-json-path", "../../perf-test-config.json",
              "--test-results-server", "webkit-perf.appspot.com",
              "--builder-name", WithProperties("%(buildername)s"),
              "--build-number", WithProperties("%(buildnumber)s"),
              "--platform", WithProperties("%(fullPlatform)s"),
              WithProperties("--%(configuration)s")]

   def start(self):
       self.setCommand(self.command)
       return shell.Test.start(self)


class RunAndUploadPerfTestsWebKit2(RunAndUploadPerfTests):
    def start(self):
        self.setCommand(self.command + ["--webkit-test-runner"])
        return RunAndUploadPerfTests.start(self)


class ArchiveTestResults(shell.ShellCommand):
    command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
               WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
    name = "archive-test-results"
    description = ["archiving test results"]
    descriptionDone = ["archived test results"]
    haltOnFailure = True


class UploadTestResults(transfer.FileUpload):
    slavesrc = "layout-test-results.zip"
    masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")

    def __init__(self, **kwargs):
        kwargs['slavesrc'] = self.slavesrc
        kwargs['masterdest'] = self.masterdest
        kwargs['mode'] = 0644
        transfer.FileUpload.__init__(self, **kwargs)


class ExtractTestResults(master.MasterShellCommand):
    zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
    resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
    descriptionDone = ["uploaded results"]

    def __init__(self, **kwargs):
        kwargs['command'] = ""
        master.MasterShellCommand.__init__(self, **kwargs)

    def resultDirectoryURL(self):
        return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"

    def start(self):
        self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
        return master.MasterShellCommand.start(self)

    def addCustomURLs(self):
        url = self.resultDirectoryURL() + "results.html"
        self.addURL("view results", url)

    def finished(self, result):
        self.addCustomURLs()
        return master.MasterShellCommand.finished(self, result)


class ExtractTestResultsAndLeaks(ExtractTestResults):
    def addCustomURLs(self):
        ExtractTestResults.addCustomURLs(self)
        url = "/LeaksViewer/?url=" + urllib.quote(self.resultDirectoryURL(), safe="")
        self.addURL("view leaks", url)


class Factory(factory.BuildFactory):
    def __init__(self, platform, configuration, architectures, buildOnly, SVNMirror):
        factory.BuildFactory.__init__(self)
        self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, SVNMirror=SVNMirror))
        if SVNMirror:
            self.addStep(WaitForSVNServer())
        self.addStep(CheckOutSource(SVNMirror=SVNMirror))
        # There are multiple Qt slaves running on same machines, so buildslaves shouldn't kill the processes of other slaves.
        if not platform.startswith("qt"):
            self.addStep(KillOldProcesses())
        if platform == "win":
            self.addStep(InstallWin32Dependencies())
        if platform.startswith("chromium"):
            self.addStep(InstallChromiumDependencies())
        if platform == "gtk":
            self.addStep(InstallGtkDependencies())
        if platform == "efl":
            self.addStep(InstallEflDependencies())


class BuildFactory(Factory):
    def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
        self.addStep(CompileWebKit())
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

def unitTestsSupported(configuration, platform):
    if platform.startswith('mac') and configuration == "release":
        return False; # https://bugs.webkit.org/show_bug.cgi?id=82652
    return (platform == 'win' or platform.startswith('mac') or platform.startswith('chromium'))

def pickLatestBuild(builder, requests):
    return max(requests, key=operator.attrgetter("submittedAt"))

class TestFactory(Factory):
    TestClass = RunWebKitTests
    ExtractTestResultsClass = ExtractTestResults
    def __init__(self, platform, configuration, architectures, SVNMirror=None):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror)
        if platform.startswith("chromium"):
            self.addStep(CleanupChromiumCrashLogs())
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        if not platform.startswith("chromium"):
            self.addStep(RunJavaScriptCoreTests(buildJSCTool=False))
        if platform.startswith("chromium"):
            self.addStep(RunChromiumWebKitUnitTests())
        self.addStep(self.TestClass(buildJSCTool=(platform != 'win')))

        if unitTestsSupported(configuration, platform): 
            self.addStep(RunUnitTests())
        self.addStep(RunPythonTests())
        # Chromium Win runs in non-Cygwin environment, which is not yet fit
        # for running tests. This can be removed once bug 48166 is fixed.
        if platform != "chromium-win":
            self.addStep(RunPerlTests())
            self.addStep(RunBindingsTests())
        self.addStep(ArchiveTestResults())
        self.addStep(UploadTestResults())
        self.addStep(self.ExtractTestResultsClass())
        if platform == "efl":
            self.addStep(RunEflAPITests)
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
        if platform.startswith("qt"):
            self.addStep(RunQtAPITests)

class BuildAndTestFactory(Factory):
    TestClass = RunWebKitTests
    ExtractTestResultsClass = ExtractTestResults
    def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        if platform.startswith("chromium"):
            self.addStep(CleanupChromiumCrashLogs)
        self.addStep(CompileWebKit())
        if not platform.startswith("chromium"):
            self.addStep(RunJavaScriptCoreTests())
        if platform.startswith("chromium"):
            self.addStep(RunChromiumWebKitUnitTests)
        self.addStep(self.TestClass())
        if unitTestsSupported(configuration, platform): 
            self.addStep(RunUnitTests())
        self.addStep(RunPythonTests())
        # Chromium Win runs in non-Cygwin environment, which is not yet fit
        # for running tests. This can be removed once bug 48166 is fixed.
        if platform != "chromium-win":
            self.addStep(RunPerlTests())
            self.addStep(RunBindingsTests())
        self.addStep(ArchiveTestResults())
        self.addStep(UploadTestResults())
        self.addStep(self.ExtractTestResultsClass())
        if platform == "efl":
            self.addStep(RunEflAPITests)
        if platform == "gtk":
            self.addStep(RunGtkAPITests())
        if platform.startswith("qt"):
            self.addStep(RunQtAPITests())
        if triggers:
            self.addStep(ArchiveBuiltProduct())
            self.addStep(UploadBuiltProduct())
            self.addStep(trigger.Trigger(schedulerNames=triggers))

class BuildAndTestWebKit2Factory(BuildAndTestFactory):
    TestClass = RunWebKit2Tests

class BuildAndTestLeaksFactory(BuildAndTestFactory):
    TestClass = RunWebKitLeakTests
    ExtractTestResultsClass = ExtractTestResultsAndLeaks


class TestWebKit2Factory(TestFactory):
    TestClass = RunWebKit2Tests

class BuildAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        if platform.startswith("chromium"):
            self.addStep(CleanupChromiumCrashLogs)
        self.addStep(CompileWebKit())
        self.addStep(RunAndUploadPerfTests())

class BuildAndPerfTestWebKit2Factory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        if platform.startswith("chromium"):
            self.addStep(CleanupChromiumCrashLogs)
        self.addStep(CompileWebKit())
        self.addStep(RunAndUploadPerfTestsWebKit2())

class DownloadAndPerfTestFactory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(DownloadBuiltProduct())
        self.addStep(ExtractBuiltProduct())
        self.addStep(RunAndUploadPerfTests())

class DownloadAndPerfTestWebKit2Factory(Factory):
    def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
        Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
        self.addStep(DownloadBuiltProduct)
        self.addStep(ExtractBuiltProduct)
        self.addStep(RunAndUploadPerfTestsWebKit2)

class PlatformSpecificScheduler(AnyBranchScheduler):
    def __init__(self, platform, branch, **kwargs):
        self.platform = platform
        filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
        AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)

    def filter(self, change):
        return wkbuild.should_build(self.platform, change.files)

trunk_filter = ChangeFilter(branch=["trunk", None])

def loadBuilderConfig(c):
    # FIXME: These file handles are leaked.
    passwords = json.load(open('passwords.json'))
    config = json.load(open('config.json'))

    c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]

    c['schedulers'] = []
    for scheduler in config['schedulers']:
        if "change_filter" in scheduler:
            scheduler["change_filter"] = globals()[scheduler["change_filter"]]
        kls = globals()[scheduler.pop('type')]
        # Python 2.6 can't handle unicode keys as keyword arguments:
        # http://bugs.python.org/issue2646.  Modern versions of json return
        # unicode strings from json.load, so we map all keys to str objects.
        scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))

        c['schedulers'].append(kls(**scheduler))

    forceScheduler = ForceScheduler(
        name="force",
        builderNames=[builder['name'] for builder in config['builders']],
        reason=StringParameter(name="reason", default="", size=40),

        # Validate SVN revision: number or emtpy string
        revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),

        # Disable default enabled input fields: branch, repository, project, additional properties
        branch=FixedParameter(name="branch"),
        repository=FixedParameter(name="repository"),
        project=FixedParameter(name="project"),
        properties=[]
    )
    c['schedulers'].append(forceScheduler)

    c['builders'] = []
    for builder in config['builders']:
        for slaveName in builder['slavenames']:
            for slave in config['slaves']:
                if slave['name'] != slaveName or slave['platform'] == '*':
                    continue

                if slave['platform'] != builder['platform']:
                    raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])

                break

        platform = builder['platform']

        builderType = builder.pop('type')
        factory = globals()["%sFactory" % builderType]
        factorykwargs = {}
        for key in "platform", "configuration", "architectures", "triggers", "SVNMirror":
            value = builder.pop(key, None)
            if value:
                factorykwargs[key] = value

        builder["factory"] = factory(**factorykwargs)

        if platform.startswith('chromium'):
            builder["category"] = 'Chromium'
        elif platform.startswith('mac'):
            builder["category"] = 'AppleMac'
        elif platform == 'win':
            builder["category"] = 'AppleWin'
        elif platform.startswith('gtk'):
            builder["category"] = 'GTK'
        elif platform.startswith('qt'):
            builder["category"] = 'Qt'
        elif platform.startswith('efl'):
            builder["category"] = "EFL"
        else:
            builder["category"] = 'misc'

        if (builder['category'] == 'AppleMac' or builder['category'] == 'AppleWin') and builderType != 'Build':
            builder['nextBuild'] = pickLatestBuild

        c['builders'].append(builder)

loadBuilderConfig(c)