Remove old Lorry Controller to make room for new one

7 files changed, 0 insertions, 1160 deletions

diff --git a/lorry-controller b/lorry-controller
deleted file mode 100755
index 0ae4ceb..0000000
--- a/lorry-controller
+++ /dev/null
@@ -1,355 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import cliapp
-import json
-import logging
-import os
-import time
-import re
-import urllib
-import urllib2
-
-
-from lorrycontroller.confparser import LorryControllerConfig
-from lorrycontroller.workingstate import WorkingStateManager
-from lorrycontroller.htmlstatus import HTMLStatusManager
-
-
-defaults = {
-    'work-area': '/home/lorry/controller-area',
-    'config-name': 'lorry-controller.conf',
-    'lorry': 'lorry',
-}
-
-
-token_finder = re.compile("([0-9a-f]{40})")
-
-
-class LorryController(cliapp.Application):
-
-    def add_settings(self):
-        self.settings.string(['work-area'],
-                             'path to the area for  the controller to work in',
-                             metavar='PATH',
-                             default=defaults['work-area'])
-        self.settings.boolean(['dry-run'],
-                              "do a dry-run and don't actually do anything "
-                              "beyond updating the git tree",
-                              default=False)
-        self.settings.string(['lorry'],
-                             'path to the lorry binary to use',
-                             metavar='LORRY',
-                             default=defaults['lorry'])
-        self.settings.string(['config-name'],
-                             'configuration leafname.  Defaults to '
-                             'lorry-controller.conf',
-                             metavar='CONFNAME',
-                             default=defaults['config-name'])
-        self.settings.boolean(['lorry-verbose'],
-                              'Whether to pass --verbose to lorry',
-                              default=False)
-        self.settings.string(['lorry-log'],
-                             'Log file name for lorry if wanted',
-                             metavar='LORRYLOG',
-                             default=None)
-        self.settings.string(['html-file'],
-                             'HTML filename for lorry controller status',
-                             metavar='HTMLFILE',
-                             default=None)
-
-    def process_args(self, args):
-        logging.info("Starting to control lorry")
-        try:
-            os.chdir(self.settings['work-area'])
-        except OSError, e:
-            logging.error("Unable to chdir() to %s" % 
-                          self.settings['work-area'])
-            raise SystemExit(2)
-        if not os.path.isdir("git"):
-            logging.error("Unable to find git checkout")
-            raise SystemExit(3)
-        if not os.path.isdir("work"):
-            os.mkdir("work")
-
-        logging.info("Updating configuration checkout")
-        self.rungit(['remote', 'update', 'origin'])
-        self.rungit(['reset', '--hard', 'origin/master'])
-        self.rungit(['clean', '-fdx'])
-
-        self.lorrycmd=[self.settings['lorry']]
-        if self.settings['lorry-verbose']:
-            self.lorrycmd += ["--verbose"]
-        if self.settings['lorry-log'] is not None:
-            self.lorrycmd += ["--log", self.settings['lorry-log']]
-
-        if not os.path.exists(os.path.join('git',
-                                           self.settings['config-name'])):
-            logging.error("Unable to find lorry-controller.conf in git")
-            raise SystemExit(4)
-
-        if os.path.isfile('git/proxy.conf'):
-            self.set_proxy('git/proxy.conf')
-            logging.info('Loaded proxy information')
-        self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf')
-        self.html = HTMLStatusManager(self)
-        if self.settings['dry-run']:
-            self.html.series = 0
-        self.html.write_out_status()
-        self.conf.parse_config()
-
-        with WorkingStateManager(self) as mgr:
-            # Update any troves
-            self.html.set_mgr(mgr)
-            self.html.bump_state()
-            self.conf.update_troves(mgr)
-            prev_lorries = set(mgr.lorry_state.keys())
-            cur_lorries = set(self.conf.lorries.keys())
-            logging.info("Starting processing.  Previously %d lorries "
-                         "were handled.  We currently have %d defined." % (
-                    len(prev_lorries), len(cur_lorries)))
-
-            # 1. Handle deletes for any old lorries we no longer want
-            self.html.bump_state()
-            logging.info("Delete any old lorries...")
-            for dead_lorry in prev_lorries - cur_lorries:
-                self.html.set_processing(dead_lorry)
-                logging.info("Dead lorry: %s" % dead_lorry)
-                conf_uuid = mgr.lorry_state[dead_lorry]['conf']
-                if conf_uuid in self.conf.configs:
-                    should_delete = self.conf.configs[conf_uuid]['destroy']
-                else:
-                    # Could not find UUID in config, switch to 'never'
-                    should_delete = "never"
-                want_destroy = (should_delete == "always")
-                if should_delete == "unchanged":
-                    exit, out, err = self.maybe_runcmd(
-                        ['git', 'ls-remote', 'ssh://git@localhost/%s.git' %
-                         dead_lorry], dry=True)
-                    if exit != 0:
-                        logging.error("Unable to ls-remote to decide if "
-                                      "unchanged.  Assuming it is changed.")
-                    else:
-                        logging.debug("TODO: Should decide if unchanged!")
-
-                if want_destroy:
-                    exit, out, err = self.maybe_runcmd(['ssh', 'git@localhost',
-                                                        'destroy', dead_lorry],
-                                                       dry=True)
-                    if exit != 0:
-                        logging.error("Unable to destroy %s" % dead_lorry)
-                    else:
-                        token = token_finder.match(out).group(1)
-                        exit, out, err = self.maybe_runcmd(
-                            ['ssh', 'git@localhost', 'destroy', dead_lorry,
-                             token])
-                        if exit != 0:
-                            logging.error("Unable to destroy %s despite having"
-                                          " the token %s" %
-                                          (dead_lorry, token))
-                        else:
-                            logging.debug("Destroyed")
-                del mgr.lorry_state[dead_lorry]
-
-            # 2. Handle creates for any new lorries we now want
-            self.html.bump_state()
-            logging.info("Create any new lorries...")
-            for new_lorry in cur_lorries - prev_lorries:
-                self.html.set_processing(new_lorry)
-                logging.info("New lorry: %s" % new_lorry)
-                lorry = self.conf.lorries[new_lorry]
-                conf_uuid = lorry['controller-uuid']
-                conf = self.conf.configs[conf_uuid]
-                nextdue = self.conf.duetimes[new_lorry]
-                # Make new lorries overdue.
-                nextdue -= conf['interval-parsed']
-                should_create = conf['create'] == "always"
-                store_state = True
-                if should_create:
-                    exit, out, err = self.maybe_runcmd(["ssh", "git@localhost",
-                                                        "create", new_lorry])
-                    if exit != 0:
-                        if ' already exists' in err:
-                            logging.warn("Repository %s already exists" %
-                                         new_lorry)
-                        else:
-                            logging.error("Unable to create repository %s" %
-                                          new_lorry)
-                            logging.error(err)
-                            store_state = False
-                if store_state:
-                    self.maybe_runcmd(["ssh", "git@localhost", "set-head",
-                                       new_lorry, lorry['source-HEAD']])
-                    mgr.lorry_state[new_lorry] = {
-                        'destroy': conf['destroy'],
-                        'conf': conf_uuid,
-                        'lorry': lorry,
-                        'next-due': nextdue,
-                        }
-                else:
-                    # Remove this from cur_lorries so we don't run it
-                    cur_lorries.remove(new_lorry)
-
-            # 3. For every lorry we have, update the settings if necessary.
-            #    and reset the next-due as appropriate.
-            self.html.bump_state()
-            logging.info("Update active lorry configurations...")
-            updated_count = 0
-            for upd_lorry in cur_lorries:
-                if mgr.lorry_state[upd_lorry]['lorry'] != \
-                        self.conf.lorries[upd_lorry]:
-                    lorry = self.conf.lorries[upd_lorry]
-                    old_lorry = mgr.lorry_state[upd_lorry]["lorry"]
-                    if lorry["source-HEAD"] != \
-                            old_lorry.get("source-HEAD", "refs/heads/master"):
-                        self.maybe_runcmd(['ssh', 'git@localhost', 'set-head',
-                                           upd_lorry, lorry["source-HEAD"]])
-                    conf_uuid = lorry['controller-uuid']
-                    conf = self.conf.configs[conf_uuid]
-                    nextdue = self.conf.duetimes[upd_lorry]
-                    mgr.lorry_state[upd_lorry] = {
-                        'destroy': conf['destroy'],
-                        'conf': conf_uuid,
-                        'lorry': lorry,
-                        'next-due': nextdue,
-                    }
-                    updated_count += 1
-            logging.info("Result: %d/%d lorries needed updating" % (
-                    updated_count, len(cur_lorries)))
-
-            # 3. Iterate all active lorries and see if they're due
-            logging.info("Iterate active lorries looking for work...")
-            now = time.time()
-            lorried = 0
-            earliest_due = None
-            what_early_due = ""
-            lorries_to_run = []
-            for lorry in cur_lorries:
-                state = mgr.lorry_state[lorry]
-                conf_uuid = state['conf']
-                conf = self.conf.configs[conf_uuid]
-                due = state['next-due']
-                if now >= due:
-                    lorries_to_run.append(lorry)
-            lorries_to_run.sort()
-            for lorry in lorries_to_run:
-                state = mgr.lorry_state[lorry]
-                conf_uuid = state['conf']
-                conf = self.conf.configs[conf_uuid]
-                due = state['next-due']
-                lorried += 1
-                logging.info("Running %d/%d. Lorrying: %s" % (
-                        lorried, len(lorries_to_run),lorry))
-                self.html.set_processing(lorry)
-                # Before we run lorry, make sure that Git doesn't verify
-                # SSL certificates. This is a workaround for the fact that
-                # we don't yet have a solution for proper SSL certificates
-                # in Trove yet.
-                os.environ['GIT_SSL_NO_VERIFY'] = 'true'
-                with mgr.runner(lorry) as runner:
-                    runner.run_lorry(*self.lorrycmd)
-                while state['next-due'] <= now:
-                    state['next-due'] += conf['interval-parsed']
-
-            for lorry in cur_lorries:
-                state = mgr.lorry_state[lorry]
-                due = state['next-due']
-                if earliest_due is None or due < earliest_due:
-                    earliest_due = due
-                    what_early_due = lorry
-
-            if earliest_due is None:
-                logging.info("Lorried %d.  No idea what's next." % lorried)
-            else:
-                logging.info("Lorried %d.  %s due in %d seconds" % (
-                        lorried, what_early_due, int(earliest_due - now)))
-            logging.info("All done.")
-        self.html.bump_state()
-
-    def rungit(self, args):
-        self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'],
-                                                   'git'))
-
-    def maybe_http_request(self, url, auth=None, dry=False):
-        """If not a dry run, make an HTTP request and return its output."""
-        if (not self.settings['dry-run']) or dry:
-            return self.http_request(url, auth)
-        else:
-            logging.debug('DRY-RUN: Not sending a request to %s' % url)
-            return 0, 'DRY-RUN', 'DRY-RUN'
-
-    def maybe_runcmd(self, cmdline, dry=False, *args, **kwargs):
-        if (not self.settings['dry-run']) or dry:
-            return self.runcmd_unchecked(cmdline, *args, **kwargs)
-        else:
-            logging.debug("DRY-RUN: Not running %r" % cmdline)
-            return 0, 'DRY-RUN', 'DRY-RUN'
-
-    def http_request(self, url, auth=None):
-        """Make an HTTP request to the given url, return the output.
-
-        Make an HTTP request to `url`. If the request succeeds (response code
-        200) then return an exit code 0, the data from the response and the
-        response code. Otherwise return the response code, any data in the
-        repsonse and a string containing the response code.
-
-        """
-        request = urllib2.Request(url, None, {})
-        if auth:
-            password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
-            password_mgr.add_password(
-                None, url, auth['username'], auth['password'])
-            auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr)
-            opener = urllib2.build_opener(auth_handler)
-            response = opener.open(url)
-        else:
-            response = urllib2.urlopen(request)
-        code = response.getcode()
-        if code == 200:
-            return 0, response.read(), '200'
-        else:
-            return code, response.read(), str(code)
-
-    def set_proxy(self, proxy_def):
-        """Tell urllib2 to use a proxy for http action by lorry-controller.
-
-        Load the proxy information from the JSON file given by proxy_def, then
-        set urllib2's url opener to open urls via an authenticated proxy.
-
-        """
-        with open(proxy_def, 'r') as proxy_info:
-            proxy = json.load(proxy_info)
-
-        # set the required environment variables
-        hostname = urllib.quote(proxy['hostname'])
-        user = '%s:%s' % (proxy['username'], proxy['password'])
-        url = '%s:%s' % (hostname, proxy['port'])
-        os.environ['http_proxy'] = 'http://%s@%s' % (user, url)
-        os.environ['https_proxy'] = 'https://%s@%s' % (user, url)
-
-        # create a ProxyHandler
-        proxies = {'http_proxy': 'http://%s@%s' % (user, url),
-                   'https_proxy': 'https://%s@%s' % (user, url)}
-        proxy_handler = urllib2.ProxyHandler(proxies)
-
-        # install an opener to use the proxy
-        opener = urllib2.build_opener(proxy_handler)
-        urllib2.install_opener(opener)
-
-if __name__ == '__main__':
-    LorryController(version='1').run()
diff --git a/lorry-controller.conf b/lorry-controller.conf
deleted file mode 100644
index 3d46b23..0000000
--- a/lorry-controller.conf
+++ /dev/null
@@ -1,20 +0,0 @@
-[
-    {
-	"type": "trove",
-	"uuid": "default-staggered-short",
-	"trovehost": "git.baserock.org",
-    "protocol": "ssh",
-	"ls-interval": "1H",
-	"prefixmap": {
-	    "baserock": "baserock"
-	},
-	"ignore": [
-	    "baserock/lorries",
-            "baserock/tests/*"
-	],
-	"create": "never",
-	"destroy": "unchanged",
-	"interval": "30M",
-	"stagger": true
-    }
-]
diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py
deleted file mode 100644
index 0fe0b33..0000000
--- a/lorrycontroller/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import confparser
-import workingstate
diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py
deleted file mode 100644
index 403b768..0000000
--- a/lorrycontroller/confparser.py
+++ /dev/null
@@ -1,335 +0,0 @@
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import json
-import logging
-import re
-import glob
-import os
-import time
-import fnmatch
-import urllib
-
-default_values = [
-    ( u'create', u'never' ),
-    ( u'destroy', u'never' ),
-    ( u'interval', u'1m' ),
-    ( u'stagger', False ),
-    ( u'tarball', u'never' ),
-    ( u'type', u'invalid_type' ),
-]
-
-valid_interval = re.compile(r"^([1-9][0-9]*)([mhd])?$")
-interval_mults = {
-    None: 1,
-    'm': 60,
-    'h': 60 * 60,
-    'd': 60 * 60 * 24,
-}
-class LorryControllerConfig(object):
-    '''This encapsulates the configuration for lorry-controller.'''
-
-    def __init__(self, app, confpath):
-        self.app = app
-        self.confpath = confpath
-        self.lorries = {}
-        self.configs = {}
-        self.duetimes = {}
-        self.troves = []
-
-    def parse_config(self):
-        confpath = os.path.join(self.app.settings['work-area'], self.confpath)
-        logging.info("Parsing configuration: %s" % confpath)
-        try:
-            with open(confpath, "r") as fh:
-                self._raw_conf = json.load(fh)
-        except Exception, e:
-            logging.error("Unable to parse: %r" % e)
-            raise
-        logging.debug("Validating configuration semantics")
-        self._validate__raw_conf()
-        logging.info("Configuration loaded")
-
-    def _validate__raw_conf(self):
-        '''Validate the entire raw config.'''
-        if type(self._raw_conf) != list:
-            self._give_up("Configuration was not a list.")
-        for entry in self._raw_conf:
-            if type(entry) != dict:
-                self._give_up("Configuration entry was not a dict.")
-            if type(entry.get('type', None)) != unicode:
-                self._give_up("Configuration entry lacked a suitable 'type' "
-                             "field.")
-            # Set the defaults
-            for key, defval in default_values:
-                entry[key] = entry.get(key, defval)
-            # And validate the generic values
-            self._validate__generics(entry)
-            # Now validate the rest
-            validator = getattr(self, '_validate_' + entry['type'], None)
-            if validator is None:
-                self._give_up("Configuration entry had unknown type: %s" %
-                             entry['type'])
-            validator(entry)
-
-    def _validate__generics(self, entry):
-        '''Validate the generic entries such as 'uuid'.'''
-        if type(entry.get('uuid', None)) != unicode:
-            self._give_up("UUID missing, cannot reconcile without it!")
-        if entry['uuid'] in self.configs:
-            self._give_up("UUID is not unique")
-        self.configs[entry['uuid']] = entry
-        for key, defval in default_values:
-            if type(defval) != type(entry[key]):
-                self._give_up("Invalid type for '%s': %r" % (key, entry[key]))
-        self._validate__when(entry, 'create', ["always", "never"])
-        self._validate__when(entry, 'destroy',
-                             ["always", "never", "unchanged"])
-        self._validate__when(entry, 'tarball', ["always", "never", "first"])
-        entry['interval-parsed'] = self._parse_interval(entry['interval'])
-        if 'ls-interval' in entry:
-            entry['ls-interval-parsed'] = \
-                self._parse_interval(entry['ls-interval'])
-
-    def _validate__when(self, entry, key, valid_whens):
-        if entry[key] not in valid_whens:
-            self._give_up("Invalid value for %s: %s" % (key, entry[key]))
-
-    def _parse_interval(self, interval):
-        m = valid_interval.match(interval.lower())
-        if m is None:
-            self._give_up("Unable to parse '%s' as an interval" % interval)
-        num, mult = m.groups()
-        num = int(num)
-        mult = interval_mults.get(mult, None)
-        if mult is None:
-            self._give_up("Somehow, '%s' managed to appear as a multiplier!" %
-                         m.group(2))
-        logging.debug("Converted interval %r to %r", interval, (num * mult))
-        return num * mult
-
-    def _validate_lorries(self, entry):
-        '''Validate a 'lorries' stanza.'''
-        if type(entry.get('globs', None)) != list:
-            self._give_up("Lorries stanzas need lists for their 'globs'")
-        if entry.get('prefix', None) is None:
-            entry['prefix'] = u""
-        if type(entry['prefix']) != unicode:
-            self._give_up("Lorry prefixes should be strings.")
-        my_lorries = set()
-        git_base = os.path.join(self.app.settings['work-area'], 'git')
-        for glob_entry in entry['globs']:
-            if type(glob_entry) != unicode:
-                self._give_up("Lorries globs should be strings")
-            fullglob = os.path.join(git_base, glob_entry)
-            my_lorries = my_lorries.union(set(glob.iglob(fullglob)))
-        for lorry in my_lorries:
-            if not lorry.startswith(git_base):
-                self._give_up("Glob found %s which is outside the git base")
-
-        logging.debug("Expanded globs in entry to %d lorry files" % 
-                      len(my_lorries))
-        logging.debug("Loading lorries into memory, please wait...")
-
-        my_lorry_names = set()
-        for lorry in my_lorries:
-            try:
-                with open(lorry, "r") as fh:
-                    lorry_json = json.load(fh)
-                for name, content in lorry_json.iteritems():
-                    fullname = os.path.join(entry['prefix'], name)
-                    if self.lorries.get(fullname, None) is not None:
-                        self._give_up("Lorry repeated: %s" % fullname)
-                    content['controller-uuid'] = entry['uuid']
-                    if not content.has_key('source-HEAD'):
-                        content['source-HEAD'] = 'refs/heads/master'
-                    my_lorry_names.add(fullname)
-                    self.lorries[fullname] = content
-            except Exception, e:
-                logging.warning("Unable to parse %s, because of %s. "
-                                "Moving on" % (lorry, e))
-
-        # Now calculate the 'next due' time for every lorry we just parsed
-        starttime = time.time() - 1
-        endtime = starttime + entry['interval-parsed']
-        step = 0
-        if entry['stagger']:
-            step = (endtime - starttime) / (len(my_lorry_names) + 1)
-        for lorry_name in my_lorry_names:
-            self.duetimes[lorry_name] = starttime
-            starttime += step
-
-        logging.debug("Now loaded %d lorries" % len(self.lorries.keys()))
-
-    def _validate_trove(self, entry):
-        # Validate top levels
-        if type(entry.get('trovehost', None)) != unicode:
-            self._give_up("Trove host %r is not a string" %
-                          entry.get('trovehost', None))
-        if 'ls-interval-parsed' not in entry:
-            self._give_up("No ls-interval specified for %s" %
-                          entry['trovehost'])
-        if type(entry.get('prefixmap', None)) != dict:
-            self._give_up("Prefixmap not a dict for %s" %
-                          entry['trovehost'])
-        if type(entry.get('ignore', [])) != list:
-            self._give_up("Ignore is not a list for %s" %
-                          entry['trovehost'])
-        protocol = entry.get('protocol')
-        auth = entry.get('auth')
-        if protocol == 'https' and not auth:
-            self._give_up('Trove access protocol requires authorisation '
-                    'details but none were defined.')
-        elif not protocol:
-            self._give_up('Trove access protocol not defined.')
-        # Validate prefixmap
-        for local, remote in entry['prefixmap'].iteritems():
-            if type(local) != unicode:
-                self._give_up("Local part of prefixmap is not a string: %r" %
-                              local)
-            if type(remote) != unicode:
-                self._give_up("Remote part of prefixmap is not a string: %r" %
-                              remote)
-        # Validate ignore
-        for ign in entry.get('ignore', []):
-            if type(ign) != unicode:
-                self._give_up("Part of ignore list is not a string: %r" % ign)
-
-        self.troves.append(entry)
-
-    def update_trove(self, trove, state):
-        logging.info("Processing trove %s (%s)" % (trove['trovehost'],
-                                                   trove['uuid']))
-        # 1. Ensure that if we need to 'ls' the trove, we do it
-        now = time.time()
-        state['next-vls'] = state.get('next-vls', now - 1)
-        if state['next-vls'] < now:
-            exit, out, err = self.run_gitano_command(trove, True, 'ls', '--verbose')
-            if exit == 0:
-                repo_info = {}
-                for entry in [x for x in out.split("\n") if x != ""]:
-                    while entry.find("  ") > -1:
-                        entry = entry.replace("  ", " ")
-                    elems = entry.split(" ")
-                    this_repo = {
-                        "perm": elems[0],
-                        "name": elems[1],
-                        "head": elems[2],
-                        "desc": " ".join(elems[3:]),
-                        }
-                    repo_info[elems[1]] = this_repo
-                state['last-ls-output'] = repo_info
-                logging.info("ls interval %d" % trove['ls-interval-parsed'])
-                logging.info("next-vls was %s" % time.asctime(time.gmtime(state['next-vls'])))
-                while state['next-vls'] < now:
-                    state['next-vls'] += trove['ls-interval-parsed']
-                logging.info("next-vls now %s" % time.asctime(time.gmtime(state['next-vls'])))
-            else:
-                # Pass through unchanged
-                state['last-ls-output'] = state.get('last-ls-output', {})
-
-        def ignored(reponame):
-            for pattern in trove['ignore']:
-                if fnmatch.fnmatch(reponame, pattern):
-                    return True
-            return False
-
-        # 2. For every entry in last-ls-output, construct a lorry if we want it
-        lorries_made = set()
-        for remotereponame, info in state['last-ls-output'].iteritems():
-            localreponame = None
-            for local, remote in trove['prefixmap'].iteritems():
-                if remotereponame.startswith(remote+"/"):
-                    localreponame = "%s/%s" % (local,
-                                               remotereponame[len(remote)+1:])
-            if ((not ignored(remotereponame)) and (localreponame is not None)):
-                # Make the url in the correct form for the given protocol
-                if trove['protocol'] == 'ssh':
-                    url = 'ssh://git@%s/%s.git' % (trove['trovehost'],
-                                                   remotereponame)
-                elif trove['protocol'] == 'https':
-                    auth = trove['auth']
-                    url = 'https://%s:%s@%s/git/%s.git' % (auth['username'],
-                                                           auth['password'],
-                                                           trove['trovehost'],
-                                                           remotereponame)
-                else:
-                    url = 'http://%s/git/%s.git' % (trove['trovehost'],
-                                                    remotereponame)
-                # Construct a lorry for this one.
-                lorry = {
-                    "type": "git",
-                    "url": url,
-                    "controller-uuid": trove['uuid'],
-                    "source-HEAD": info["head"],
-                    "refspecs": [ "+refs/heads/*:refs/heads/*",
-                                  "+refs/tags/*:refs/tags/*" ]
-                    }
-                if localreponame in self.lorries:
-                    logging.warn("Skipping %s (%s from %s) because we already "
-                                 "have something for that." % (
-                            localreponame, remotereponame, trove['trovehost']))
-                else:
-                    self.lorries[localreponame] = lorry
-                    lorries_made.add(localreponame)
-
-        # 3. Now schedule all those lorries in case they're new
-        starttime = time.time() - 1
-        endtime = starttime + trove['interval-parsed']
-        step = 0
-        if trove['stagger']:
-            step = (endtime - starttime) / (len(lorries_made)+1)
-        for lorry_name in lorries_made:
-            self.duetimes[lorry_name] = starttime
-            starttime += step
-
-        logging.debug("Generated %d lorries from that trove" %
-                      len(lorries_made))
-
-    def update_troves(self, statemgr):
-        # Now that we have a state manager we can look at the trove data.
-        for trove in self.troves:
-            self.app.html.set_processing(trove['uuid'])
-            trove_state = statemgr.get_trove(trove['uuid'])
-            self.update_trove(trove, trove_state)
-
-    def run_gitano_command(self, trove, dry, command, *args):
-        """Run a gitano command on the trove, and return the output."""
-        if trove['protocol'] == 'ssh':
-            # construct list to run command over ssh
-            cmdargs = ['ssh',
-                       '-oStrictHostKeyChecking=no',
-                       '-oBatchMode=yes',
-                       'git@%s' % trove['trovehost'],
-                       command]
-            cmdargs.extend(args)
-            # run the command
-            exit, out, err = self.app.maybe_runcmd(cmdargs, dry=dry)
-        else:
-            # construct a url which will return the command output
-            query_string = '%s %s' % (command, ' '.join(args))
-            query_string = urllib.quote(query_string)
-            trovehost = urllib.quote(trove['trovehost'])
-            url = '%s://%s/gitano-command.cgi?cmd=%s' % (
-                trove['protocol'], trovehost, query_string)
-            auth = trove.get('auth', None)
-            # make an http request to the url
-            exit, out, err = self.app.maybe_http_request(url, auth=auth, dry=dry)
-        return exit, out, err
-
-    def _give_up(self, *args, **kwargs):
-        logging.error(*args, **kwargs)
-        raise SystemExit(5)
diff --git a/lorrycontroller/htmlstatus.py b/lorrycontroller/htmlstatus.py
deleted file mode 100644
index 30b52f0..0000000
--- a/lorrycontroller/htmlstatus.py
+++ /dev/null
@@ -1,286 +0,0 @@
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-import time
-from cgi import escape
-
-state_names = [
-    "Initialisation",
-    "Load Troves",
-    "Remove old repos",
-    "Create new repos",
-    "Process Lorries",
-    "Finished"
-    ]
-
-def format_time(time_t):
-    return time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(time_t))
-
-class HTMLStatusManager(object):
-    '''Manage the HTML status page for lorry-controller.'''
-
-
-    def __init__(self, app):
-        self.app = app
-        self.state = 0
-        self.series = None
-        self.filename = self.app.settings['html-file']
-        self.mgr = None
-        self.all_processing = set()
-        self.processing = None
-        self.processing_time = time.time()
-        self.failing = None
-        self.all_lorries_ever = set()
-        self.bump_time = time.time()
-
-    def set_failing(self, failmsg):
-        self.failing = failmsg
-        self.write_out_status()
-
-    def set_mgr(self, mgr):
-        self.mgr = mgr
-
-    def set_processing(self, proc):
-        if self.processing is not None:
-            self.all_processing.add(self.processing)
-        self.processing = proc
-        self.processing_time = time.time()
-        self.write_out_status()
-
-    def bump_state(self):
-        self.state = self.state + 1
-        self.bump_time = time.time()
-        self.processing = None
-        self.write_out_status()
-
-    def write_out_status(self):
-        if self.filename is None: return
-        try:
-            with open(self.filename + ".new", "w") as ofh:
-                ofh.write("<!DOCTYPE html>\n")
-                ofh.write(self.gen_html())
-                ofh.write("\n")
-            target = self.filename
-            if self.series is not None:
-                target += ".%d" % self.series
-                self.series += 1
-            os.rename(self.filename + ".new", target)
-        except:
-            os.unlink(self.filename + ".new")
-            raise
-
-    def gen_html(self):
-        head = self.gen_head()
-        body = self.gen_body()
-        return self.tag("html", content=head+"\n"+body, gap=True)
-
-    def gen_head(self):
-        title = self.tag("title", content="Lorry Controller")
-        css = self.tag("link", href="trove.css", rel="stylesheet",
-                       type="text/css")
-        script = self.tag(
-            "script", type="text/javascript", src="/table.js", content="")
-        return self.tag("head", content=title+css+script)
-
-    def gen_body(self):
-        # 1. Rough header, branded as trove
-        curtime = format_time(time.time())
-        link = "/"
-        if self.series is not None:
-            link = self.filename + ".%d" % (self.series + 1)
-        header = '''
-<table id='header'><tr><td class='logo' rowspan='2'>
-<a href='%s'><img src='trove.png' alt='trove logo'/></a></td>
-<td class='main'>Status of Lorry Controller</td></tr>
-<tr><td class='sub'>Updated at %s</td></tr></table>
-''' % (link, curtime)
-        # 2. List of steps and where we are currently
-        steps = self.gen_steps()
-
-        # 4. Main content
-        content = self.gen_content()
-
-        # 5. footer
-        footer = self.gen_footer()
-        
-        # magic args
-        return self.tag("body", content=self.tag(
-                "div", content=(header+steps+content+footer),
-                Class="lorrycontroller"))
-
-    def gen_content(self):
-        if self.failing is not None:
-            return self.tag("div", Class="failure", content=self.failing)
-        # 1. Troves known
-        troves = self.gen_troves()
-        # 2. Lorries known
-        lorries = self.gen_lorries()
-
-        return self.tag("div", Class="content", content=
-                        self.tag("div", id="troves", content=troves) +
-                        self.tag("div", id="lorries", content=lorries))
-
-    def gen_troves(self):
-        troves = []
-        now = time.time()
-        for trove in self.app.conf.troves:
-            troveinfo = {}
-            if self.mgr is not None:
-                troveinfo = self.mgr.trove_state.get(trove['uuid'], {})
-            uuid = self.tag("td", content=escape(trove['uuid']))
-            state = "Up to date"
-            if self.processing == trove['uuid']:
-                state = "Processing since " + \
-                    format_time(self.processing_time)
-            elif troveinfo.get('next-vls', now - 1) < now:
-                if self.state < len(state_names) - 1:
-                    state = "Due to be checked this run."
-                else:
-                    state = "Due to be checked on the next run"
-            state = self.tag("td", content=escape(state))
-            nextdue = self.tag("td", content=escape(format_time(
-                        troveinfo.get('next-vls', now - 1))))
-            lorrycount = len([l for l in self.app.conf.lorries.itervalues()
-                              if l['controller-uuid'] == trove['uuid']])
-            lorrycount = self.tag("td", content=str(lorrycount))
-
-            troves.append(self.tag("tr", content=
-                                   uuid+state+nextdue+lorrycount))
-        if len(troves) == 0:
-            content = "No troves detected"
-        else:
-            header = self.tag("tr", content=
-                              self.tag("th", content="Trove UUID") +
-                              self.tag("th", content="Status") +
-                              self.tag("th", content="Next due") +
-                              self.tag("th", content="Lorries created"))
-            content = self.tag("table", content=
-                               header + "".join(troves))
-
-        return content
-
-    def gen_lorries(self):
-        lorries = []
-        now = time.time()
-        all_lorry_names = set(self.app.conf.lorries.keys())
-        if self.mgr is not None:
-            all_lorry_names.update(set(self.mgr.lorry_state.keys()))
-        self.all_lorries_ever.update(all_lorry_names)
-        all_lorry_names = list(self.all_lorries_ever)
-        all_lorry_names.sort()
-        for lorry_name in all_lorry_names:
-            lorry = self.app.conf.lorries.get(lorry_name, None)
-            dead_lorry = False
-            dead_and_gone = False
-            if lorry is None:
-                lorrystate = self.mgr.lorry_state.get(lorry_name, None)
-                if lorrystate is None:
-                    dead_and_gone = True
-                    lorry = {}
-                else:
-                    lorry = lorrystate['lorry']
-                dead_lorry = True
-            lorryinfo = {}
-            if self.mgr is not None:
-                lorryinfo = self.mgr.lorry_state.get(lorry_name, {})
-            uuid = self.tag("td", content=
-                            escape(lorry.get('controller-uuid', 'Dead')))
-            state = "Waiting "
-            if self.processing == lorry_name:
-                state = "Processing since " + \
-                    format_time(self.processing_time)
-            elif lorryinfo.get('next-due', now - 1) < self.bump_time:
-                if self.state < len(state_names) - 1:
-                    state = "Due to be checked this run."
-                else:
-                    state = "Due to be checked on the next run"
-                if self.mgr is not None:
-                    if self.mgr.lorry_state.get(lorry_name, None) is None:
-                        state = "Needs creating"
-            elif lorry_name in self.all_processing:
-                state = "Processed"
-            if dead_lorry:
-                state = "Dead - To be removed"
-                if dead_and_gone:
-                    state = "Dead"
-                if self.processing == lorry_name:
-                    state = "Removing since " + \
-                        format_time(self.processing_time)
-            state = self.tag("td", content=escape(state))
-            lastresult = self.tag("td", content=self.tag(
-                    "pre", content=escape(lorryinfo.get('result', '-'))))
-            nextdue = self.tag("td", content=escape(format_time(
-                        lorryinfo.get('next-due', now - 1))))
-            lorryname = self.tag("td", content=escape(lorry_name))
-            lorries.append(self.tag("tr", content=
-                                   lorryname+uuid+state+lastresult+nextdue))
-        if len(lorries) == 0:
-            content = "No lorries detected yet"
-        else:
-            header = self.tag("tr", content=
-                              self.tag("th",
-                                       Class="table-sortable:alphanumeric",
-                                       content="Lorry Name") +
-                              self.tag("th",
-                                       Class="table-sortable:alphanumeric",
-                                       content="Comes From") +
-                              self.tag("th",
-                                       Class="table-sortable:alphanumeric",
-                                       content="Status") +
-                              self.tag("th",
-                                       Class="table-sortable:alphanumeric",
-                                       content="Last result") +
-                              self.tag("th",
-                                       Class="table-sortable:alphanumeric",
-                                       content="Next due"))
-            header = self.tag("thead", content=header)
-            content = self.tag("table", Class="table-autosort:4", content=
-                               header + "\n" + "\n".join(lorries))
-
-        return content
-
-
-    def gen_footer(self):
-        curtime = format_time(time.time())
-        return self.tag("div", Class="footer", content=
-                        "Generated by Lorry Controller at " + curtime)
-
-    def gen_steps(self):
-        steps = []
-        for idx in xrange(len(state_names)):
-            if idx < self.state:
-                Class = "donestep"
-            elif idx == self.state:
-                Class = "activestep"
-            else:
-                Class = "pendingstep"
-            steps.append(self.tag("span", Class=Class,
-                                  content=state_names[idx]))
-        return self.tag("table", Class="steps", content=
-                        self.tag("tr", content=
-                                 self.tag("td", content=
-                                          "".join(steps))))
-
-    def tag(self, tagname, content=None, gap=False, **kwargs):
-        tagval = " ".join([tagname] + 
-                          ["%s=%r" % (k.lower(), v) for k, v in kwargs.iteritems()])
-        gap = "\n" if gap else ""
-        if content is None:
-            return "<%s />" % tagval
-        else:
-            return "<%s>%s%s%s</%s>" % (tagval, gap, content, gap, tagname)
-
diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py
deleted file mode 100644
index b8dc751..0000000
--- a/lorrycontroller/workingstate.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import json
-import os
-import logging
-import string
-
-class LorryFileRunner(object):
-    def __init__(self, mgr, lorryname):
-        self.mgr = mgr
-        self.lorryname = lorryname
-        self.lorryfile = os.path.join(self.mgr.workdir,
-                                      self._esc(lorryname) + ".lorry")
-
-    def _esc(self, name):
-        valid_chars = string.digits + string.letters + '%_'
-        transl = lambda x: x if x in valid_chars else '_'
-        return ''.join([transl(x) for x in name])
-
-    def __enter__(self):
-        lorry_obj = { self.lorryname:
-                      self.mgr.lorry_state[self.lorryname]['lorry'] }
-        with open(self.lorryfile, "w") as fh:
-            json.dump(lorry_obj, fh)
-            fh.write("\n")
-        return self
-
-    def __exit__(self, exctype, excvalue, exctraceback):
-        os.unlink(self.lorryfile)
-
-    def run_lorry(self, *args):
-        cmdargs = list(args)
-        cmdargs.append(self.lorryfile)
-        conf_uuid = self.mgr.lorry_state[self.lorryname]['conf']
-        conf = self.mgr.app.conf.configs[conf_uuid]
-        cmdargs.append("--tarball=%s" % conf['tarball'])
-        exit, out, err = self.mgr.app.maybe_runcmd(cmdargs)
-        if exit == 0:
-            logging.debug("Lorry of %s succeeded: %s" % (self.lorryname, out))
-            self.mgr.lorry_state[self.lorryname]['result'] = "OK"
-        else:
-            logging.warn("Lorry of %s failed: %s" % (self.lorryname, err))
-            self.mgr.lorry_state[self.lorryname]['result'] = err
-
-class WorkingStateManager(object):
-    '''Manage the working state of lorry-controller'''
-
-    def __init__(self, app):
-        self.app = app
-        self.workdir = os.path.join(self.app.settings['work-area'], 'work')
-
-    def __enter__(self):
-        self._load_state()
-        return self
-
-    def __exit__(self, exctype, excvalue, exctraceback):
-        self.purge_dead_troves()
-        if not self.app.settings['dry-run']:
-            self.save_state()
-        else:
-            logging.debug("DRY-RUN: Not saving state again")
-
-    def purge_dead_troves(self):
-        old_trove_count = len(self.trove_state.keys())
-        all_troves = self.trove_state
-        self.trove_state = {}
-        new_trove_count = 0
-        for uuid, trove in all_troves.iteritems():
-            self.trove_state[uuid] = trove
-            new_trove_count += 1
-        if old_trove_count != new_trove_count:
-            trove_diff = old_trove_count - new_trove_count
-            logging.info("Purged %d dead trove entr%s from the state file" % (
-                    trove_diff, ("y" if trove_diff == 1 else "ies")))
-
-    def _load_state(self):
-        self.lorry_state_file = os.path.join(self.workdir,
-                                             "last-lorry-state.json")
-        self.trove_state_file = os.path.join(self.workdir,
-                                             "last-trove-state.json")
-        if os.path.exists(self.lorry_state_file):
-            logging.info("Loading lorry state file: %s" %
-                         self.lorry_state_file)
-            with open(self.lorry_state_file, "r") as fh:
-                self.lorry_state = json.load(fh)
-        else:
-            self.lorry_state = dict()
-
-        if os.path.exists(self.trove_state_file):
-            logging.info("Loading trove state file: %s" %
-                         self.trove_state_file)
-            with open(self.trove_state_file, "r") as fh:
-                self.trove_state = json.load(fh)
-        else:
-            self.trove_state = dict()
-
-    def save_state(self):
-        logging.info("Serialising lorry state: %s" % self.lorry_state_file)
-        with open(self.lorry_state_file, "w") as fh:
-            json.dump(self.lorry_state, fh, sort_keys=True, indent=4)
-            fh.write("\n")
-        logging.info("Serialising trove state: %s" % self.trove_state_file)
-        with open(self.trove_state_file, "w") as fh:
-            json.dump(self.trove_state, fh, sort_keys=True, indent=4)
-            fh.write("\n")
-
-    def get_trove(self, troveuuid):
-        if troveuuid not in self.trove_state:
-            self.trove_state[troveuuid] = {}
-        return self.trove_state[troveuuid]
-
-    def runner(self, lorryname):
-        return LorryFileRunner(self, lorryname)
diff --git a/setup.py b/setup.py
deleted file mode 100644
index b27b9d5..0000000
--- a/setup.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (C) 2012  Codethink Limited
-
-
-from distutils.core import setup
-
-
-setup(name='lorry-controller',
-      description='FIXME',
-      long_description='''\
-FIXME
-''',
-      author='Daniel Silverstone',
-      author_email='daniel.silverstne@codethink.co.uk',
-      url='http://www.baserock.com/',
-      scripts=['lorry-controller'],
-      packages=['lorrycontroller'],
-     )