Add new Lorry Controller

1 files changed, 347 insertions, 0 deletions

diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py
new file mode 100644
index 0000000..b6f7333
--- /dev/null
+++ b/lorrycontroller/readconf.py
@@ -0,0 +1,347 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import errno
+import glob
+import json
+import logging
+import os
+import re
+
+import bottle
+import cliapp
+
+import lorrycontroller
+
+
+class LorryControllerConfParseError(Exception):
+
+    def __init__(self, filename, exc):
+        Exception.__init__(
+            self, 'ERROR reading %s: %s' % (filename, str(exc)))
+
+
+class ReadConfiguration(lorrycontroller.LorryControllerRoute):
+
+    http_method = 'POST'
+    path = '/1.0/read-configuration'
+
+    DEFAULT_LORRY_TIMEOUT = 3600 # in seconds
+
+    def run(self, **kwargs):
+        logging.info('%s %s called', self.http_method, self.path)
+
+        self.get_confgit()
+
+        try:
+            conf_obj = self.read_config_file()
+        except LorryControllerConfParseError as e:
+            return str(e)
+
+        error = self.validate_config(conf_obj)
+        if error:
+            return 'ERROR: %s: %r' % (error, conf_obj)
+
+        self.fix_up_parsed_fields(conf_obj)
+
+        statedb = self.open_statedb()
+        with statedb:
+            existing_lorries = set(statedb.get_lorries_paths())
+            existing_troves = set(statedb.get_troves())
+
+            for section in conf_obj:
+                if not 'type' in section:
+                    return 'ERROR: no type field in section'
+                if section['type'] == 'lorries':
+                    added = self.add_matching_lorries_to_statedb(
+                        statedb, section)
+                    existing_lorries = existing_lorries.difference(added)
+                elif section['type'] in ('trove', 'troves'):
+                    self.add_trove(statedb, section)
+                    if section['trovehost'] in existing_troves:
+                        existing_troves.remove(section['trovehost'])
+                        existing_lorries = self.without_lorries_for_trovehost(
+                            statedb, existing_lorries, section['trovehost'])
+                else:
+                    logging.error(
+                        'Unknown section in configuration: %r', section)
+                    return (
+                        'ERROR: Unknown section type in configuration: %r' % 
+                        section)
+
+            for path in existing_lorries:
+                statedb.remove_lorry(path)
+
+            for trovehost in existing_troves:
+                statedb.remove_trove(trovehost)
+                statedb.remove_lorries_for_trovehost(trovehost)
+
+
+            if 'redirect' in bottle.request.forms:
+                bottle.redirect(bottle.request.forms.redirect)
+
+            return 'Configuration has been updated.'
+
+    def without_lorries_for_trovehost(self, statedb, lorries, trovehost):
+        for_trovehost = statedb.get_lorries_for_trove(trovehost)
+        return set(x for x in lorries if x not in for_trovehost)
+
+    def get_confgit(self):
+        if self.app_settings['debug-real-confgit']:
+            confdir = self.app_settings['configuration-directory']
+            if not os.path.exists(confdir):
+                self.git_clone_confgit(confdir)
+            else:
+                self.git_pull_confgit(confdir)
+
+    def git_clone_confgit(self, confdir):
+        url = self.app_settings['confgit-url']
+        branch = self.app_settings['confgit-branch']
+        logging.info('Cloning %s to %s', url, confdir)
+        cliapp.runcmd(['git', 'clone', '-b', branch, url, confdir])
+
+    def git_pull_confgit(self, confdir):
+        logging.info('Updating CONFGIT in %s', confdir)
+        cliapp.runcmd(['git', 'pull'], cwd=confdir)
+
+    @property
+    def config_file_name(self):
+        return os.path.join(
+            self.app_settings['configuration-directory'],
+            'lorry-controller.conf')
+
+    def read_config_file(self):
+        '''Read the configuration file, return as Python object.'''
+
+        filename = self.config_file_name
+        logging.debug('Reading configuration file %s', filename)
+
+        try:
+            with open(filename) as f:
+                return json.load(f)
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                logging.debug(
+                    '%s: does not exist, returning empty config', filename)
+                return []
+            bottle.abort(500, 'Error reading %s: %s' % (filename, e))
+        except ValueError as e:
+            logging.error('Error parsing configuration: %s', e)
+            raise LorryControllerConfParseError(filename, e)
+
+    def validate_config(self, obj):
+        validator = LorryControllerConfValidator()
+        return validator.validate_config(obj)
+
+    def fix_up_parsed_fields(self, obj):
+        for item in obj:
+            item['interval'] = self.fix_up_interval(item.get('interval'))
+            item['ls-interval'] = self.fix_up_interval(item.get('ls-interval'))
+
+    def fix_up_interval(self, value):
+        default_interval = 86400 # 1 day
+        if not value:
+            return default_interval
+        m = re.match('(\d+)\s*(s|m|h|d)?', value, re.I)
+        if not m:
+            return default_value
+
+        number, factor = m.groups()
+        factors = {
+            's': 1,
+            'm': 60,
+            'h': 60*60,
+            'd': 60*60*24,
+            }
+        if factor is None:
+            factor = 's'
+        factor = factors.get(factor.lower(), 1)
+        return int(number) * factor
+
+    def add_matching_lorries_to_statedb(self, statedb, section):
+        logging.debug('Adding matching lorries to STATEDB')
+
+        added_paths = set()
+
+        filenames = self.find_lorry_files_for_section(section)
+        logging.debug('filenames=%r', filenames)
+        lorry_specs = []
+        for filename in sorted(filenames):
+            logging.debug('Reading .lorry: %s', filename)
+            for subpath, obj in self.get_valid_lorry_specs(filename):
+                self.add_refspecs_if_missing(obj)
+                lorry_specs.append((subpath, obj))
+
+        for subpath, obj in sorted(lorry_specs):
+            path = self.deduce_repo_path(section, subpath)
+            text = self.serialise_lorry_spec(path, obj)
+            interval = section['interval']
+            timeout = section.get(
+                'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT)
+
+            try:
+                old_lorry_info = statedb.get_lorry_info(path)
+            except lorrycontroller.LorryNotFoundError:
+                old_lorry_info = None
+
+            statedb.add_to_lorries(
+                path=path, text=text, from_trovehost='', from_path='',
+                interval=interval, timeout=timeout)
+
+            added_paths.add(path)
+
+        return added_paths
+
+    def find_lorry_files_for_section(self, section):
+        result = []
+        dirname = os.path.dirname(self.config_file_name)
+        for base_pattern in section['globs']:
+            pattern = os.path.join(dirname, base_pattern)
+            result.extend(glob.glob(pattern))
+        return result
+    
+    def get_valid_lorry_specs(self, filename):
+        # We do some basic validation of the .lorry file and the Lorry
+        # specs contained within it. We silently ignore anything that
+        # doesn't look OK. We don't have a reasonable mechanism to
+        # communicate any problems to the user, but we do log them to
+        # the log file.
+
+        try:
+            with open(filename) as f:
+                obj = json.load(f)
+        except ValueError as e:
+            logging.error('JSON problem in %s', filename)
+            return []
+
+        if type(obj) != dict:
+            logging.error('%s: does not contain a dict', filename)
+            return []
+
+        items = []
+        for key in obj:
+            if type(obj[key]) != dict:
+                logging.error(
+                    '%s: key %s does not map to a dict', filename, key)
+                continue
+
+            if 'type' not in obj[key]:
+                logging.error(
+                    '%s: key %s does not have type field', filename, key)
+                continue
+
+            logging.debug('Happy with Lorry spec %r: %r', key, obj[key])
+            items.append((key, obj[key]))
+
+        return items
+
+    def add_refspecs_if_missing(self, obj):
+        if 'refspecs' not in obj:
+            obj['refspecs'] = [
+                '+refs/heads/*',
+                '+refs/tags/*',
+                ]
+
+    def deduce_repo_path(self, section, subpath):
+        return '%s/%s' % (section['prefix'], subpath)
+
+    def serialise_lorry_spec(self, path, obj):
+        new_obj = { path: obj }
+        return json.dumps(new_obj, indent=4)
+
+    def add_trove(self, statedb, section):
+        username = None
+        password = None
+        if 'auth' in section:
+            auth = section['auth']
+            username = auth.get('username')
+            password = auth.get('password')
+
+        statedb.add_trove(
+            trovehost=section['trovehost'],
+            protocol=section['protocol'],
+            username=username,
+            password=password,
+            lorry_interval=section['interval'],
+            lorry_timeout=section.get(
+                'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT),
+            ls_interval=section['ls-interval'],
+            prefixmap=json.dumps(section['prefixmap']),
+            ignore=json.dumps(section['ignore']))
+
+
+class LorryControllerConfValidator(object):
+
+    def validate_config(self, conf_obj):
+        try:
+            self._check_is_list(conf_obj)
+            self._check_is_list_of_dicts(conf_obj)
+
+            for section in conf_obj:
+                if 'type' not in section:
+                    raise ValidationError(
+                        'section without type: %r' % section)
+                elif section['type'] in ('trove', 'troves'):
+                    self._check_troves_section(section)
+                elif section['type'] == 'lorries':
+                    self._check_lorries_section(section)
+                else:
+                    raise ValidationError(
+                        'unknown section type %r' % section['type'])
+        except ValidationError as e:
+            return str(e)
+
+        return None
+
+    def _check_is_list(self, conf_obj):
+        if type(conf_obj) is not list:
+            raise ValidationError(
+                'type %r is not a JSON list' % type(conf_obj))
+        
+    def _check_is_list_of_dicts(self, conf_obj):
+        for item in conf_obj:
+            if type(item) is not dict:
+                raise ValidationError('all items must be dicts')
+
+    def _check_troves_section(self, section):
+        self._check_has_required_fields(
+            section,
+            ['trovehost', 'protocol', 'interval', 'ls-interval', 'prefixmap'])
+        self._check_prefixmap(section)
+
+    def _check_prefixmap(self, section):
+        # FIXME: We should be checking the prefixmap for things like
+        # mapping to a prefix that starts with the local Trove ID, but
+        # since we don't have easy access to that, we don't do that
+        # yet. This should be fixed later.
+        pass
+
+    def _check_lorries_section(self, section):
+        self._check_has_required_fields(
+            section, ['interval', 'prefix', 'globs'])
+
+    def _check_has_required_fields(self, section, fields):
+        for field in fields:
+            if field not in section:
+                raise ValidationError(
+                    'mandatory field %s missing in section %r' %
+                    (field, section))
+
+
+class ValidationError(Exception):
+
+    def __init__(self, msg):
+        Exception.__init__(self, msg)