From 4fc162b07b2e9d8489e16ed647e5d96f5c66e10a Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 20 Jan 2014 14:24:27 +0000 Subject: Add new Lorry Controller --- lorrycontroller/readconf.py | 347 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 lorrycontroller/readconf.py (limited to 'lorrycontroller/readconf.py') diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py new file mode 100644 index 0000000..b6f7333 --- /dev/null +++ b/lorrycontroller/readconf.py @@ -0,0 +1,347 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import errno +import glob +import json +import logging +import os +import re + +import bottle +import cliapp + +import lorrycontroller + + +class LorryControllerConfParseError(Exception): + + def __init__(self, filename, exc): + Exception.__init__( + self, 'ERROR reading %s: %s' % (filename, str(exc))) + + +class ReadConfiguration(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/read-configuration' + + DEFAULT_LORRY_TIMEOUT = 3600 # in seconds + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + self.get_confgit() + + try: + conf_obj = self.read_config_file() + except LorryControllerConfParseError as e: + return str(e) + + error = self.validate_config(conf_obj) + if error: + return 'ERROR: %s: %r' % (error, conf_obj) + + self.fix_up_parsed_fields(conf_obj) + + statedb = self.open_statedb() + with statedb: + existing_lorries = set(statedb.get_lorries_paths()) + existing_troves = set(statedb.get_troves()) + + for section in conf_obj: + if not 'type' in section: + return 'ERROR: no type field in section' + if section['type'] == 'lorries': + added = self.add_matching_lorries_to_statedb( + statedb, section) + existing_lorries = existing_lorries.difference(added) + elif section['type'] in ('trove', 'troves'): + self.add_trove(statedb, section) + if section['trovehost'] in existing_troves: + existing_troves.remove(section['trovehost']) + existing_lorries = self.without_lorries_for_trovehost( + statedb, existing_lorries, section['trovehost']) + else: + logging.error( + 'Unknown section in configuration: %r', section) + return ( + 'ERROR: Unknown section type in configuration: %r' % + section) + + for path in existing_lorries: + statedb.remove_lorry(path) + + for trovehost in existing_troves: + statedb.remove_trove(trovehost) + statedb.remove_lorries_for_trovehost(trovehost) + + + if 'redirect' in bottle.request.forms: + bottle.redirect(bottle.request.forms.redirect) + + return 'Configuration has been updated.' + + def without_lorries_for_trovehost(self, statedb, lorries, trovehost): + for_trovehost = statedb.get_lorries_for_trove(trovehost) + return set(x for x in lorries if x not in for_trovehost) + + def get_confgit(self): + if self.app_settings['debug-real-confgit']: + confdir = self.app_settings['configuration-directory'] + if not os.path.exists(confdir): + self.git_clone_confgit(confdir) + else: + self.git_pull_confgit(confdir) + + def git_clone_confgit(self, confdir): + url = self.app_settings['confgit-url'] + branch = self.app_settings['confgit-branch'] + logging.info('Cloning %s to %s', url, confdir) + cliapp.runcmd(['git', 'clone', '-b', branch, url, confdir]) + + def git_pull_confgit(self, confdir): + logging.info('Updating CONFGIT in %s', confdir) + cliapp.runcmd(['git', 'pull'], cwd=confdir) + + @property + def config_file_name(self): + return os.path.join( + self.app_settings['configuration-directory'], + 'lorry-controller.conf') + + def read_config_file(self): + '''Read the configuration file, return as Python object.''' + + filename = self.config_file_name + logging.debug('Reading configuration file %s', filename) + + try: + with open(filename) as f: + return json.load(f) + except IOError as e: + if e.errno == errno.ENOENT: + logging.debug( + '%s: does not exist, returning empty config', filename) + return [] + bottle.abort(500, 'Error reading %s: %s' % (filename, e)) + except ValueError as e: + logging.error('Error parsing configuration: %s', e) + raise LorryControllerConfParseError(filename, e) + + def validate_config(self, obj): + validator = LorryControllerConfValidator() + return validator.validate_config(obj) + + def fix_up_parsed_fields(self, obj): + for item in obj: + item['interval'] = self.fix_up_interval(item.get('interval')) + item['ls-interval'] = self.fix_up_interval(item.get('ls-interval')) + + def fix_up_interval(self, value): + default_interval = 86400 # 1 day + if not value: + return default_interval + m = re.match('(\d+)\s*(s|m|h|d)?', value, re.I) + if not m: + return default_value + + number, factor = m.groups() + factors = { + 's': 1, + 'm': 60, + 'h': 60*60, + 'd': 60*60*24, + } + if factor is None: + factor = 's' + factor = factors.get(factor.lower(), 1) + return int(number) * factor + + def add_matching_lorries_to_statedb(self, statedb, section): + logging.debug('Adding matching lorries to STATEDB') + + added_paths = set() + + filenames = self.find_lorry_files_for_section(section) + logging.debug('filenames=%r', filenames) + lorry_specs = [] + for filename in sorted(filenames): + logging.debug('Reading .lorry: %s', filename) + for subpath, obj in self.get_valid_lorry_specs(filename): + self.add_refspecs_if_missing(obj) + lorry_specs.append((subpath, obj)) + + for subpath, obj in sorted(lorry_specs): + path = self.deduce_repo_path(section, subpath) + text = self.serialise_lorry_spec(path, obj) + interval = section['interval'] + timeout = section.get( + 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT) + + try: + old_lorry_info = statedb.get_lorry_info(path) + except lorrycontroller.LorryNotFoundError: + old_lorry_info = None + + statedb.add_to_lorries( + path=path, text=text, from_trovehost='', from_path='', + interval=interval, timeout=timeout) + + added_paths.add(path) + + return added_paths + + def find_lorry_files_for_section(self, section): + result = [] + dirname = os.path.dirname(self.config_file_name) + for base_pattern in section['globs']: + pattern = os.path.join(dirname, base_pattern) + result.extend(glob.glob(pattern)) + return result + + def get_valid_lorry_specs(self, filename): + # We do some basic validation of the .lorry file and the Lorry + # specs contained within it. We silently ignore anything that + # doesn't look OK. We don't have a reasonable mechanism to + # communicate any problems to the user, but we do log them to + # the log file. + + try: + with open(filename) as f: + obj = json.load(f) + except ValueError as e: + logging.error('JSON problem in %s', filename) + return [] + + if type(obj) != dict: + logging.error('%s: does not contain a dict', filename) + return [] + + items = [] + for key in obj: + if type(obj[key]) != dict: + logging.error( + '%s: key %s does not map to a dict', filename, key) + continue + + if 'type' not in obj[key]: + logging.error( + '%s: key %s does not have type field', filename, key) + continue + + logging.debug('Happy with Lorry spec %r: %r', key, obj[key]) + items.append((key, obj[key])) + + return items + + def add_refspecs_if_missing(self, obj): + if 'refspecs' not in obj: + obj['refspecs'] = [ + '+refs/heads/*', + '+refs/tags/*', + ] + + def deduce_repo_path(self, section, subpath): + return '%s/%s' % (section['prefix'], subpath) + + def serialise_lorry_spec(self, path, obj): + new_obj = { path: obj } + return json.dumps(new_obj, indent=4) + + def add_trove(self, statedb, section): + username = None + password = None + if 'auth' in section: + auth = section['auth'] + username = auth.get('username') + password = auth.get('password') + + statedb.add_trove( + trovehost=section['trovehost'], + protocol=section['protocol'], + username=username, + password=password, + lorry_interval=section['interval'], + lorry_timeout=section.get( + 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT), + ls_interval=section['ls-interval'], + prefixmap=json.dumps(section['prefixmap']), + ignore=json.dumps(section['ignore'])) + + +class LorryControllerConfValidator(object): + + def validate_config(self, conf_obj): + try: + self._check_is_list(conf_obj) + self._check_is_list_of_dicts(conf_obj) + + for section in conf_obj: + if 'type' not in section: + raise ValidationError( + 'section without type: %r' % section) + elif section['type'] in ('trove', 'troves'): + self._check_troves_section(section) + elif section['type'] == 'lorries': + self._check_lorries_section(section) + else: + raise ValidationError( + 'unknown section type %r' % section['type']) + except ValidationError as e: + return str(e) + + return None + + def _check_is_list(self, conf_obj): + if type(conf_obj) is not list: + raise ValidationError( + 'type %r is not a JSON list' % type(conf_obj)) + + def _check_is_list_of_dicts(self, conf_obj): + for item in conf_obj: + if type(item) is not dict: + raise ValidationError('all items must be dicts') + + def _check_troves_section(self, section): + self._check_has_required_fields( + section, + ['trovehost', 'protocol', 'interval', 'ls-interval', 'prefixmap']) + self._check_prefixmap(section) + + def _check_prefixmap(self, section): + # FIXME: We should be checking the prefixmap for things like + # mapping to a prefix that starts with the local Trove ID, but + # since we don't have easy access to that, we don't do that + # yet. This should be fixed later. + pass + + def _check_lorries_section(self, section): + self._check_has_required_fields( + section, ['interval', 'prefix', 'globs']) + + def _check_has_required_fields(self, section, fields): + for field in fields: + if field not in section: + raise ValidationError( + 'mandatory field %s missing in section %r' % + (field, section)) + + +class ValidationError(Exception): + + def __init__(self, msg): + Exception.__init__(self, msg) -- cgit v1.2.1