Initial crack at trove support

3 files changed, 71 insertions, 7 deletions

diff --git a/lorry-controller b/lorry-controller
index ce60989..2a18d8c 100755
--- a/lorry-controller
+++ b/lorry-controller
@@ -75,8 +75,7 @@ class LorryController(cliapp.Application):
             logging.error("Unable to find lorry-controller.conf in git")
             raise SystemExit(4)
 
-        self.conf = LorryControllerConfig(self.settings,
-                                          'git/lorry-controller.conf')
+        self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf')
 
         with WorkingStateManager(self) as mgr:
             # Update any troves
diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py
index cc8f0e5..a5f1e3e 100644
--- a/lorrycontroller/confparser.py
+++ b/lorrycontroller/confparser.py
@@ -28,12 +28,13 @@ interval_mults = {
 class LorryControllerConfig(object):
     '''This encapsulates the configuration for lorry-controller.'''
 
-    def __init__(self, settings, confpath):
-        self.settings = settings
+    def __init__(self, app, confpath):
+        self.app = app
         self.lorries = {}
         self.configs = {}
         self.duetimes = {}
-        confpath = os.path.join(settings['work-area'], confpath)
+        self.troves = []
+        confpath = os.path.join(app.settings['work-area'], confpath)
         logging.debug("Parsing configuration: %s" % confpath)
         with open(confpath, "r") as fh:
             self._raw_conf = json.load(fh)
@@ -105,7 +106,7 @@ class LorryControllerConfig(object):
         if type(entry['prefix']) != unicode:
             self._give_up("Lorry prefixes should be strings.")
         my_lorries = set()
-        git_base = os.path.join(self.settings['work-area'], 'git')
+        git_base = os.path.join(self.app.settings['work-area'], 'git')
         for glob_entry in entry['globs']:
             if type(glob_entry) != unicode:
                 self._give_up("Lorries globs should be strings")
@@ -175,9 +176,68 @@ class LorryControllerConfig(object):
             if type(ign) != unicode:
                 self._give_up("Part of ignore list is not a string: %r" % ign)
 
+        self.troves.append(entry)
+
+    def update_trove(self, trove, state):
+        logging.debug("Processing trove %s (%s)" % (trove['trovehost'],
+                                                    trove['uuid']))
+        # 1. Ensure that if we need to 'ls' the trove, we do it
+        now = time.time()
+        listcmdargs = ["ssh", "git@" + trove['trovehost'], "ls"]
+        state['next-ls'] = state.get('next-ls', now - 1)
+        if state['next-ls'] < now:
+            exit, out, err = self.app.maybe_runcmd(listcmdargs)
+            if exit == 0:
+                state['last-ls-output'] = [x[3:] for x in out.split("\n")]
+                while state['next-ls'] < now:
+                    state['next-ls'] += trove['ls-interval-parsed']
+            else:
+                # Pass through unchanged
+                state['last-ls-output'] = state.get('last-ls-output', [])
+
+        # 2. For every entry in last-ls-output, construct a lorry if we want it
+        lorries_made = set()
+        for remotereponame in state['last-ls-output']:
+            localreponame = None
+            for local, remote in trove['prefixmap'].iteritems():
+                if remotereponame.startswith(remote+"/"):
+                    localreponame = "%s/%s" % (local,
+                                               remotereponame[len(remote)+1:])
+            if ((remotereponame not in trove['ignore']) and
+                (localreponame is not None)):
+                # Construct a lorry for this one.
+                lorry = {
+                    "type": "git",
+                    "url": "ssh://git@%s/%s.git" % (trove['trovehost'],
+                                                        remotereponame)
+                    }
+                if localreponame in self.lorries:
+                    logging.warn("Skipping %s (%s from %s) because we already "
+                                 "have something for that." % (
+                            localreponame, remotereponame, trove['trovehost']))
+                else:
+                    self.lorries[localreponame] = lorry
+                    self.configs[localreponame] = trove
+                    lorries_made.add(localreponame)
+
+        # 3. Now schedule all those lorries in case they're new
+        starttime = time.time() - 1
+        endtime = starttime + trove['interval-parsed']
+        step = 0
+        if trove['stagger']:
+            step = (endtime - starttime) / len(lorries_made)
+        for lorry_name in lorries_made:
+            self.duetimes[lorry_name] = starttime
+            starttime += step
+
+        logging.debug("Generated %d lorries from that trove" %
+                      len(lorries_made))
+
     def update_troves(self, statemgr):
         # Now that we have a state manager we can look at the trove data.
-        pass
+        for trove in self.troves:
+            trove_state = statemgr.get_trove(trove['uuid'])
+            self.update_trove(trove, trove_state)
 
     def _give_up(self, *args, **kwargs):
         logging.error(*args, **kwargs)
diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py
index c4b031f..1ef2e9b 100644
--- a/lorrycontroller/workingstate.py
+++ b/lorrycontroller/workingstate.py
@@ -84,5 +84,10 @@ class WorkingStateManager(object):
             json.dump(self.trove_state, fh, sort_keys=True, indent=4)
             fh.write("\n")
 
+    def get_trove(self, troveuuid):
+        if troveuuid not in self.trove_state:
+            self.trove_state[troveuuid] = {}
+        return self.trove_state[troveuuid]
+
     def runner(self, lorryname):
         return LorryFileRunner(self, lorryname)