# Copyright (C) 2014-2020 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import fnmatch import json import logging import bottle import lorrycontroller class ServerLsError(Exception): def __init__(self, remote_host, output): Exception.__init__( self, 'Failed to get list of git repositories ' 'on remote host %s:\n%s' % (remote_host, output)) self.remote_host = remote_host class HostRepositoryLister(object): def __init__(self, app_settings, route): self.app_settings = app_settings self.route = route def list_host_into_statedb(self, statedb, host_info): remote_paths = self.ls(statedb, host_info) remote_paths = self.skip_ignored_repos(host_info, remote_paths) repo_map = self.map_remote_repos_to_local_ones( host_info, remote_paths) with statedb: self.update_lorries_for_host(statedb, host_info, repo_map) now = statedb.get_current_time() statedb.set_host_ls_last_run(host_info['host'], now) def ls(self, statedb, host_info): if self.app_settings['debug-fake-upstream-host']: repo_paths = self.get_fake_ls_output(host_info) else: repo_paths = self.get_real_ls_output(host_info) return repo_paths def get_fake_ls_output(self, host_info): host = host_info['host'] for item in self.app_settings['debug-fake-upstream-host']: fake_host, path = item.split('=', 1) if fake_host == host: with open(path) as f: obj = json.load(f) return obj['ls-output'] return None def get_real_ls_output(self, host_info): return lorrycontroller.get_upstream_host(host_info).list_repos() def skip_ignored_repos(self, host, repo_paths): ignored_patterns = json.loads(host['ignore']) ignored_paths = set() for pattern in ignored_patterns: ignored_paths.update(fnmatch.filter(repo_paths, pattern)) return set(repo_paths).difference(ignored_paths) def map_remote_repos_to_local_ones(self, host_info, remote_paths): '''Return a dict that maps each remote repo path to a local one.''' prefixmap = self.parse_prefixmap(host_info['prefixmap']) repo_map = {} for remote_path in remote_paths: local_path = self.map_one_remote_repo_to_local_one( remote_path, prefixmap) if local_path: repo_map[remote_path] = local_path else: logging.debug('Remote repo %r not in prefixmap', remote_path) return repo_map def parse_prefixmap(self, prefixmap_string): # Sort prefixes in reverse order, so more specific prefixes # come first return sorted(json.loads(prefixmap_string).items(), reverse=True) def map_one_remote_repo_to_local_one(self, remote_path, prefixmap): for remote_prefix, local_prefix in prefixmap: if self.path_starts_with_prefix(remote_path, remote_prefix): relative_path = remote_path[len(remote_prefix):] local_path = local_prefix + relative_path return local_path return None def path_starts_with_prefix(self, path, prefix): return path.startswith(prefix) and path[len(prefix):].startswith('/') def update_lorries_for_host(self, statedb, host_info, repo_map): host = host_info['host'] for remote_path, local_path in list(repo_map.items()): lorry = self.construct_lorry(host_info, local_path, remote_path) statedb.add_to_lorries( path=local_path, text=json.dumps(lorry, indent=4), from_host=host, from_path=remote_path, interval=host_info['lorry_interval'], timeout=host_info['lorry_timeout']) all_local_paths = set(statedb.get_lorries_for_host(host)) wanted_local_paths = set(repo_map.values()) delete_local_paths = all_local_paths.difference(wanted_local_paths) for local_path in delete_local_paths: statedb.remove_lorry(local_path) def construct_lorry(self, host_info, local_path, remote_path): return { local_path: { 'type': 'git', 'url': self.construct_lorry_url(host_info, remote_path), 'refspecs': [ "+refs/heads/*", "+refs/tags/*", ], } } def construct_lorry_url(self, host_info, remote_path): return lorrycontroller.get_upstream_host(host_info) \ .get_repo_url(remote_path) class ForceLsUpstream(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/force-ls-trove' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) host = bottle.request.forms.host statedb = self.open_statedb() lister = HostRepositoryLister(self.app_settings, self) host_info = statedb.get_host_info(host) try: updated = lister.list_host_into_statedb(statedb, host_info) except ServerLsError as e: raise bottle.abort(500, str(e)) return { 'updated-troves': updated } class LsUpstreams(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/ls-troves' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) statedb = self.open_statedb() lister = HostRepositoryLister(self.app_settings, self) host_infos = self.get_due_hosts(statedb) for host_info in host_infos: logging.info('Host %r is due an ls', host_info['host']) try: lister.list_host_into_statedb(statedb, host_info) except ServerLsError as e: bottle.abort(500, str(e)) return { 'updated-troves': [host_info['host'] for host_info in host_infos], } def get_due_hosts(self, statedb): host_infos = [ statedb.get_host_info(host) for host in statedb.get_hosts()] now = statedb.get_current_time() return [ host_info for host_info in host_infos if self.is_due(host_info, now)] def is_due(self, host_info, now): ls_due = host_info['ls_last_run'] + host_info['ls_interval'] return ls_due <= now