diff options
Diffstat (limited to 'lorrycontroller/lsupstreams.py')
| -rw-r--r-- | lorrycontroller/lsupstreams.py | 232 |
1 files changed, 232 insertions, 0 deletions
diff --git a/lorrycontroller/lsupstreams.py b/lorrycontroller/lsupstreams.py new file mode 100644 index 0000000..a64a496 --- /dev/null +++ b/lorrycontroller/lsupstreams.py @@ -0,0 +1,232 @@ +# Copyright (C) 2014-2019 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import fnmatch +import json +import logging + +import bottle + +import lorrycontroller + + +class ServerLsError(Exception): + + def __init__(self, remote_host, output): + Exception.__init__( + self, + 'Failed to get list of git repositories ' + 'on remote host %s:\n%s' % (remote_host, output)) + self.remote_host = remote_host + + +class HostRepositoryLister(object): + + def __init__(self, app_settings, route): + self.app_settings = app_settings + self.route = route + + def list_host_into_statedb(self, statedb, host_info): + remote_paths = self.ls(statedb, host_info) + remote_paths = self.skip_ignored_repos(host_info, remote_paths) + repo_map = self.map_remote_repos_to_local_ones( + host_info, remote_paths) + + with statedb: + self.update_lorries_for_host(statedb, host_info, repo_map) + now = statedb.get_current_time() + statedb.set_host_ls_last_run(host_info['host'], now) + + def ls(self, statedb, host_info): + if self.app_settings['debug-fake-upstream-host']: + repo_paths = self.get_fake_ls_output(host_info) + else: + repo_paths = self.get_real_ls_output(statedb, host_info) + + return repo_paths + + def get_fake_ls_output(self, host_info): + host = host_info['host'] + for item in self.app_settings['debug-fake-upstream-host']: + fake_host, path = item.split('=', 1) + if fake_host == host: + with open(path) as f: + obj = json.load(f) + return obj['ls-output'] + return None + + def get_real_ls_output(self, statedb, host_info): + gitlab_token = host_info.get('gitlab_token') + if gitlab_token: + return lorrycontroller.Gitlab( + host_info['host'], gitlab_token).list_projects() + + gitano = lorrycontroller.new_gitano_command( + statedb, host_info['host']) + output = gitano.ls() + return self.parse_ls_output(output) + + def parse_ls_output(self, ls_output): + repo_paths = [] + for line in ls_output.splitlines(): + words = line.split(None, 1) + if words[0].startswith('R') and len(words) == 2: + repo_paths.append(words[1]) + return repo_paths + + def skip_ignored_repos(self, host, repo_paths): + ignored_patterns = json.loads(host['ignore']) + + ignored_paths = set() + for pattern in ignored_patterns: + ignored_paths.update(fnmatch.filter(repo_paths, pattern)) + + return set(repo_paths).difference(ignored_paths) + + def map_remote_repos_to_local_ones(self, host_info, remote_paths): + '''Return a dict that maps each remote repo path to a local one.''' + prefixmap = self.parse_prefixmap(host_info['prefixmap']) + repo_map = {} + for remote_path in remote_paths: + local_path = self.map_one_remote_repo_to_local_one( + remote_path, prefixmap) + if local_path: + repo_map[remote_path] = local_path + else: + logging.debug('Remote repo %r not in prefixmap', remote_path) + return repo_map + + def parse_prefixmap(self, prefixmap_string): + return json.loads(prefixmap_string) + + def map_one_remote_repo_to_local_one(self, remote_path, prefixmap): + for remote_prefix in prefixmap: + if self.path_starts_with_prefix(remote_path, remote_prefix): + local_prefix = prefixmap[remote_prefix] + relative_path = remote_path[len(remote_prefix):] + local_path = local_prefix + relative_path + return local_path + return None + + def path_starts_with_prefix(self, path, prefix): + return path.startswith(prefix) and path[len(prefix):].startswith('/') + + def update_lorries_for_host(self, statedb, host_info, repo_map): + host = host_info['host'] + for remote_path, local_path in list(repo_map.items()): + lorry = self.construct_lorry(host_info, local_path, remote_path) + statedb.add_to_lorries( + path=local_path, + text=json.dumps(lorry, indent=4), + from_host=host, + from_path=remote_path, + interval=host_info['lorry_interval'], + timeout=host_info['lorry_timeout']) + + all_local_paths = set(statedb.get_lorries_for_host(host)) + wanted_local_paths = set(repo_map.values()) + delete_local_paths = all_local_paths.difference(wanted_local_paths) + for local_path in delete_local_paths: + statedb.remove_lorry(local_path) + + def construct_lorry(self, host_info, local_path, remote_path): + return { + local_path: { + 'type': 'git', + 'url': self.construct_lorry_url(host_info, remote_path), + 'refspecs': [ + "+refs/heads/*", + "+refs/tags/*", + ], + } + } + + def construct_lorry_url(self, host_info, remote_path): + gitlab_token = host_info.get('gitlab_token') + if gitlab_token: + return lorrycontroller.Gitlab( + host_info['host'], gitlab_token).get_project_url( + host_info['protocol'], remote_path) + + vars = dict(host_info) + vars['remote_path'] = remote_path + + patterns = { + 'ssh': 'ssh://git@{host}/{remote_path}', + 'https':'https://{username}:{password}@{host}/git/{remote_path}', + 'http': 'http://{host}/git/{remote_path}', + } + + return patterns[host_info['protocol']].format(**vars) + + +class ForceLsUpstream(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/force-ls-trove' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + host = bottle.request.forms.host + + statedb = self.open_statedb() + lister = HostRepositoryLister(self.app_settings, self) + host_info = statedb.get_host_info(host) + try: + updated = lister.list_host_into_statedb(statedb, host_info) + except ServerLsError as e: + raise bottle.abort(500, str(e)) + + return { 'updated-troves': updated } + + +class LsUpstreams(lorrycontroller.LorryControllerRoute): + + http_method = 'POST' + path = '/1.0/ls-troves' + + def run(self, **kwargs): + logging.info('%s %s called', self.http_method, self.path) + + statedb = self.open_statedb() + lister = HostRepositoryLister(self.app_settings, self) + + host_infos = self.get_due_hosts(statedb) + for host_info in host_infos: + logging.info('Host %r is due an ls', host_info['host']) + try: + lister.list_host_into_statedb(statedb, host_info) + except ServerLsError as e: + bottle.abort(500, str(e)) + + return { + 'updated-troves': [host_info['host'] for host_info in host_infos], + } + + def get_due_hosts(self, statedb): + host_infos = [ + statedb.get_host_info(host) + for host in statedb.get_hosts()] + now = statedb.get_current_time() + return [ + host_info + for host_info in host_infos + if self.is_due(host_info, now)] + + def is_due(self, host_info, now): + ls_due = host_info['ls_last_run'] + host_info['ls_interval'] + return ls_due <= now |