Make /read-configuration populate STATEDB from .lorry files

3 files changed, 135 insertions, 16 deletions

diff --git a/lorry-controller-webapp b/lorry-controller-webapp
index 6a3b0e2..edad578 100755
--- a/lorry-controller-webapp
+++ b/lorry-controller-webapp
@@ -17,6 +17,7 @@
 
 
 import errno
+import glob
 import logging
 import json
 import os
@@ -65,23 +66,26 @@ class StateDB(object):
             self._initialise_tables()
 
     def _initialise_tables(self):
+        logging.debug('Initialising tables (if necessary) in database')
         c = self._conn.cursor()
         c.execute('BEGIN TRANSACTION')
 
         c.execute('CREATE TABLE IF NOT EXISTS running_queue (running INT)')
         c.execute('INSERT INTO running_queue VALUES (0)')
 
-        c.execute('CREATE TABLE IF NOT EXISTS lorries (path TEXT, text TEXT, generated INT)')
+        c.execute('CREATE TABLE IF NOT EXISTS lorries (path TEXT PRIMARY KEY, text TEXT, generated INT)')
 
         self._conn.commit()
 
     def get_running_queue(self):
+        logging.debug('StateDB.get_running_queue called')
         self._open()
         c = self._conn.cursor()
         for (running,) in c.execute('SELECT running FROM running_queue'):
             return bool(running)
 
     def set_running_queue(self, new_status):
+        logging.debug('StateDB.set_running_queue(%r) called', new_status)
         if new_status:
             new_value = 1
         else:
@@ -92,9 +96,30 @@ class StateDB(object):
         self._conn.commit()
 
     def get_lorries(self):
+        logging.debug('StateDB.get_lorries called')
         self._open()
         c = self._conn.cursor()
-        return c.execute('SELECT * FROM lorries')
+        return c.execute('SELECT path, text, generated FROM lorries')
+
+    def add_to_lorries(self, path=None, text=None, generated=None):
+        logging.debug(
+            'StateDB.add_to_lorries(path=%r, text=%, generated=%r called',
+            path,
+            text,
+            generated)
+
+        assert path is not None
+        assert text is not None
+        assert generated is not None
+
+        self._open()
+        c = self._conn.cursor()
+        c.execute('BEGIN TRANSACTION')
+        c.execute(
+            'INSERT OR REPLACE INTO lorries (path, text, generated) '
+            'VALUES (?, ?, ?)',
+            (path, text, generated))
+        self._conn.commit()
 
 
 class LorryControllerRoute(object):
@@ -164,6 +189,7 @@ class Status(LorryControllerRoute):
     path = '/1.0/status'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
         renderer = StatusRenderer()
         status = renderer.get_status_as_dict(self.statedb, self.app_settings['statedb'])
         renderer.write_status_as_html(status, self.app_settings['status-html'])
@@ -176,6 +202,7 @@ class StatusHTML(LorryControllerRoute):
     path = '/1.0/status-html'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
         renderer = StatusRenderer()
         status = renderer.get_status_as_dict(self.statedb, self.app_settings['statedb'])
         renderer.write_status_as_html(status, self.app_settings['status-html'])
@@ -188,25 +215,32 @@ class ReadConfiguration(LorryControllerRoute):
     path = '/1.0/read-configuration'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
+
         # FIXME: This doesn't update the git repo from upstream, yet.
 
         conf_obj = self.read_config_file()
-        troves = [x for x in conf_obj if x['type'] == 'trove']
-        lorries = [x for x in conf_obj if x['type'] == 'lorries']
 
-        # FIXME: put the troves and the lorries into STATEDB, but
-        # don't generate lorries from troves, yet. Don't, in fact,
-        # scan for local lorries either, yet. Just make sure STATEDB
-        # has the right tables and columns so that /1.0/list-queue
-        # will return something reasonable-looking.
-        return { 'troves': troves, 'lorries': lorries }
+        for section in conf_obj:
+            if section['type'] == 'lorries':
+                self.add_matching_lorries_to_statedb(section)
 
-    def read_config_file(self):
-        '''Read the configuration file, return as Python object.'''
+        return 'Configuration has been updated.'
 
-        filename = os.path.join(
+    @property
+    def config_file_name(self):
+        return os.path.join(
             self.app_settings['configuration-directory'],
             'lorry-controller.conf')
+
+    def read_config_file(self):
+        '''Read the configuration file, return as Python object.'''
+
+        filename = self.config_file_name
+        logging.debug(
+            '%s %s: reading config file %s',
+            self.http_method, self.path, filename)
+
         try:
             with open(filename) as f:
                 return json.load(f)
@@ -216,6 +250,43 @@ class ReadConfiguration(LorryControllerRoute):
                 return []
             bottle.abort(500, 'Error reading %s: %s' % (filename, e))
 
+    def add_matching_lorries_to_statedb(self, section):
+        filenames = self.find_lorry_files_for_section(section)
+        for filename in filenames:
+            for lorry_spec in self.get_lorry_specs(filename):
+                path = self.deduce_repo_path(section, lorry_spec)
+                text = self.serialise_lorry_spec(lorry_spec)
+                logging.debug(
+                    '%s %s: adding lorry spec %r: %r',
+                    path, text)
+                self.statedb.add_to_lorries(path=path, text=text, generated=0)
+
+    def find_lorry_files_for_section(self, section):
+        result = []
+        dirname = os.path.dirname(self.config_file_name)
+        for base_pattern in section['globs']:
+            pattern = os.path.join(dirname, base_pattern)
+            result.extend(glob.glob(pattern))
+        return result
+    
+    def get_lorry_specs(self, filename):
+        logging.debug(
+            '%s %s: reading Lorry specs from %s',
+            self.http_method, self.path, filename)
+
+        with open(filename) as f:
+            obj = json.load(f)
+        return obj.items()
+
+    def deduce_repo_path(self, section, lorry_spec):
+        base_path, details = lorry_spec
+        return '%s/%s' % (section['prefix'], base_path)
+
+    def serialise_lorry_spec(self, lorry_spec):
+        key, details = lorry_spec
+        obj = { key: details }
+        return json.dumps(obj)
+
 
 class ListQueue(LorryControllerRoute):
 
@@ -223,8 +294,9 @@ class ListQueue(LorryControllerRoute):
     path = '/1.0/list-queue'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
         return {
-            'queue': [row.path for row in self.statedb.get_lorries()],
+            'queue': [row[0] for row in self.statedb.get_lorries()],
             }
 
 
@@ -234,6 +306,7 @@ class StartQueue(LorryControllerRoute):
     path = '/1.0/start-queue'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
         self.statedb.set_running_queue(1)
         return 'Queue set to run'
 
@@ -244,6 +317,7 @@ class StopQueue(LorryControllerRoute):
     path = '/1.0/stop-queue'
 
     def run(self, **kwargs):
+        logging.debug('%s %s called', self.http_method, self.path)
         self.statedb.set_running_queue(0)
         return 'Queue set to not run'
 
diff --git a/yarns.webapp/030-queue-management.yarn b/yarns.webapp/030-queue-management.yarn
index 32ad7cd..4404e20 100644
--- a/yarns.webapp/030-queue-management.yarn
+++ b/yarns.webapp/030-queue-management.yarn
@@ -63,6 +63,15 @@ that does not match any existing `.lorry` files.
     AND admin makes request GET /1.0/list-queue
     THEN response has queue set to []
 
+Add a `.lorry` file, with one Lorry spec, and make sure reading the
+configuration makes `/list-queue` report it.
+
+    GIVEN Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}}
+    AND lorry-controller.conf in CONFGIT adds lorries *.lorry using prefix upstream
+    WHEN admin makes request GET /1.0/read-configuration
+    AND admin makes request GET /1.0/list-queue
+    THEN response has queue set to ["upstream/foo"]
+
 Finally, clean up.
 
     FINALLY WEBAPP terminates
diff --git a/yarns.webapp/900-implementations.yarn b/yarns.webapp/900-implementations.yarn
index 003096f..4c1ddfe 100644
--- a/yarns.webapp/900-implementations.yarn
+++ b/yarns.webapp/900-implementations.yarn
@@ -30,7 +30,7 @@ be running until it crashes or is explicitly killed.
     start-stop-daemon -S -x "$SRCDIR/lorry-controller-webapp" \
         -b -p "$DATADIR/webapp.pid" -m --verbose \
         -- \
-        --configuration-directory "$DATADIR/confgit" \
+        --config "$DATADIR/webapp.conf" \
         --statedb "$DATADIR/webapp.db" \
         --status-html "$DATADIR/lc-status.html" \
         --log-level debug \
@@ -82,7 +82,43 @@ an empty list object.
     IMPLEMENTS GIVEN an empty lorry-controller.conf in (\S+)
     printf '[]\n' > "$DATADIR/$MATCH_1/lorry-controller.conf"
 
-Further, we need to be able to tell WEBAPP, when it runs, where the
+Add a `.lorry` file to be used by a `lorry-controller.conf`.
+
+    IMPLEMENTS GIVEN Lorry file (\S+) with (.*)
+    printf '%s\n' "$MATCH_2" > "$DATADIR/$MATCH_1"
+
+Add a `lorries` section to a `lorry-controller.conf`. This hardcodes
+most of the configuration.
+
+    IMPLEMENTS GIVEN (\S+) in (\S+) adds lorries (\S+) using prefix (\S+)
+    python -c '
+    import os
+    import json
+    
+    DATADIR = os.environ["DATADIR"]
+    MATCH_1 = os.environ["MATCH_1"]
+    MATCH_2 = os.environ["MATCH_2"]
+    MATCH_3 = os.environ["MATCH_3"]
+    MATCH_4 = os.environ["MATCH_4"]
+    
+    new = {
+        "type": "lorries",
+        "interval": "1h",
+        "prefix": MATCH_4,
+        "globs": [
+            MATCH_3,
+        ],
+    }
+
+    filename = os.path.join(DATADIR, MATCH_2, MATCH_1)
+    with open(filename, "r") as f:
+        obj = json.load(f)
+    obj.append(new)
+    with open(filename, "w") as f:
+        json.dump(obj, f)
+    '
+
+We need to be able to tell WEBAPP, when it runs, where the
 configuration directory is.
 
     IMPLEMENTS GIVEN WEBAPP uses (\S+) as its configuration directory