diff options
Diffstat (limited to 'lorrycontroller/lstroves.py')
| -rw-r--r-- | lorrycontroller/lstroves.py | 217 |
1 files changed, 217 insertions, 0 deletions
diff --git a/lorrycontroller/lstroves.py b/lorrycontroller/lstroves.py new file mode 100644 index 0000000..e69dce2 --- /dev/null +++ b/lorrycontroller/lstroves.py @@ -0,0 +1,217 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import time + +import bottle +import cliapp + +import lorrycontroller + + +class GitanoLsError(Exception): + + def __init__(self, trovehost, output): + Exception.__init__( + self, + 'Failed to get list of git repositories ' + 'on remote host %s:\n%s' % (trovehost, output)) + self.trovehost = trovehost + + +class TroveRepositoryLister(object): + + def __init__(self, app_settings, route): + self.app_settings = app_settings + self.route = route + + def list_trove_into_statedb(self, statedb, trove_info): + remote_paths = self.ls(statedb, trove_info) + remote_paths = self.skip_ignored_repos(trove_info, remote_paths) + repo_map = self.map_remote_repos_to_local_ones( + trove_info, remote_paths) + + with statedb: + self.update_lorries_for_trove(statedb, trove_info, repo_map) + now = statedb.get_current_time() + statedb.set_trove_ls_last_run(trove_info['trovehost'], now) + + def ls(self, statedb, trove_info): + if self.app_settings['debug-fake-trove']: + repo_paths = self.get_fake_ls_output(trove_info) + else: + repo_paths = self.get_real_ls_output(statedb, trove_info) + + return repo_paths + + def get_fake_ls_output(self, trove_info): + trovehost = trove_info['trovehost'] + for item in self.app_settings['debug-fake-trove']: + host, path = item.split('=', 1) + if host == trovehost: + with open(path) as f: + obj = json.load(f) + return obj['ls-output'] + return None + + def get_real_ls_output(self, statedb, trove_info): + gitano = lorrycontroller.new_gitano_command(statedb, trove_info['trovehost']) + output = gitano.ls() + return self.parse_ls_output(output) + + def parse_ls_output(self, ls_output): + repo_paths = [] + for line in ls_output.splitlines(): + words = line.split(None, 1) + if words[0].startswith('R') and len(words) == 2: + repo_paths.append(words[1]) + return repo_paths + + def skip_ignored_repos(self, trovehost, repo_paths): + ignored_paths = json.loads(trovehost['ignore']) + return [x for x in repo_paths if x not in ignored_paths] + + def map_remote_repos_to_local_ones(self, trove_info, remote_paths): + '''Return a dict that maps each remote repo path to a local one.''' + prefixmap = self.parse_prefixmap(trove_info['prefixmap']) + repo_map = {} + for remote_path in remote_paths: + local_path = self.map_one_remote_repo_to_local_one( + remote_path, prefixmap) + if local_path: + repo_map[remote_path] = local_path + else: + logging.debug('Remote repo %r not in prefixmap', remote_path) + return repo_map + + def parse_prefixmap(self, prefixmap_string): + return json.loads(prefixmap_string) + + def map_one_remote_repo_to_local_one(self, remote_path, prefixmap): + for remote_prefix in prefixmap: + if self.path_starts_with_prefix(remote_path, remote_prefix): + local_prefix = prefixmap[remote_prefix] + relative_path = remote_path[len(remote_prefix):] + local_path = local_prefix + relative_path + return local_path + return None + + def path_starts_with_prefix(self, path, prefix): + return path.startswith(prefix) and path[len(prefix):].startswith('/') + + def update_lorries_for_trove(self, statedb, trove_info, repo_map): + trovehost = trove_info['trovehost'] + for remote_path, local_path in repo_map.items(): + lorry = self.construct_lorry(trove_info, local_path, remote_path) + statedb.add_to_lorries( + path=local_path, + text=json.dumps(lorry, indent=4), + from_trovehost=trovehost, + from_path=remote_path, + interval=trove_info['lorry_interval'], + timeout=trove_info['lorry_timeout']) + + all_local_paths = set(statedb.get_lorries_for_trove(trovehost)) + wanted_local_paths = set(repo_map.values()) + delete_local_paths = all_local_paths.difference(wanted_local_paths) + for local_path in delete_local_paths: + statedb.remove_lorry(local_path) + + def construct_lorry(self, trove_info, local_path, remote_path): + return { + local_path: { + 'type': 'git', + 'url': self.construct_lorry_url(trove_info, remote_path), + 'refspecs': [ + "+refs/heads/*", + "+refs/tags/*", + ], + } + } + + def construct_lorry_url(self, trove_info, remote_path): + vars = dict(trove_info) + vars['remote_path'] = remote_path + + patterns = { + 'ssh': 'ssh://git@{trovehost}/{remote_path}', + 'https': + 'https://{username}:{password}@{trovehost}/git/{remote_path}', + 'http': 'http://{trovehost}/git/{remote_path}', + } + + return patterns[trove_info['protocol']].format(**vars) + + +class ForceLsTrove(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/force-ls-trove' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + trovehost = bottle.request.forms.trovehost + + statedb = self.open_statedb() + lister = TroveRepositoryLister(self.app_settings, self) + trove_info = statedb.get_trove_info(trovehost) + try: + updated = lister.list_trove_into_statedb(statedb, trove_info) + except GitanoLsError as e: + raise bottle.abort(500, str(e)) + + return { 'updated-troves': updated } + + +class LsTroves(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/ls-troves' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + statedb = self.open_statedb() + lister = TroveRepositoryLister(self.app_settings, self) + + trove_infos = self.get_due_troves(statedb) + for trove_info in trove_infos: + logging.info('Trove %r is due an ls', trove_info['trovehost']) + try: + lister.list_trove_into_statedb(statedb, trove_info) + except GitanoLsError as e: + bottle.abort(500, str(e)) + + return { + 'updated-troves': [trove_info['trovehost'] for trove_info in trove_infos], + } + + def get_due_troves(self, statedb): + trove_infos = [ + statedb.get_trove_info(trovehost) + for trovehost in statedb.get_troves()] + now = statedb.get_current_time() + return [ + trove_info + for trove_info in trove_infos + if self.is_due(trove_info, now)] + + def is_due(self, trove_info, now): + ls_due = trove_info['ls_last_run'] + trove_info['ls_interval'] + return ls_due <= now |