diff options
Diffstat (limited to 'lorrycontroller')
| -rw-r--r-- | lorrycontroller/__init__.py | 33 | ||||
| -rw-r--r-- | lorrycontroller/gerrit.py | 41 | ||||
| -rw-r--r-- | lorrycontroller/gitano.py | 104 | ||||
| -rw-r--r-- | lorrycontroller/gitlab.py | 183 | ||||
| -rw-r--r-- | lorrycontroller/givemejob.py | 123 | ||||
| -rw-r--r-- | lorrycontroller/hosts.py | 121 | ||||
| -rw-r--r-- | lorrycontroller/local.py | 56 | ||||
| -rw-r--r-- | lorrycontroller/lsupstreams.py (renamed from lorrycontroller/lstroves.py) | 141 | ||||
| -rw-r--r-- | lorrycontroller/migrations/0003-generalise-troves.py | 41 | ||||
| -rw-r--r-- | lorrycontroller/readconf.py | 78 | ||||
| -rw-r--r-- | lorrycontroller/statedb.py | 159 | ||||
| -rw-r--r-- | lorrycontroller/status.py | 22 |
12 files changed, 696 insertions, 406 deletions
diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py index c5bf0ad..ddc2f74 100644 --- a/lorrycontroller/__init__.py +++ b/lorrycontroller/__init__.py @@ -18,7 +18,7 @@ from .statedb import ( StateDB, LorryNotFoundError, WrongNumberLorriesRunningJob, - TroveNotFoundError) + HostNotFoundError) from .route import LorryControllerRoute from .readconf import ReadConfiguration from .status import Status, StatusHTML, StatusRenderer @@ -34,18 +34,33 @@ from .listjobs import ListAllJobs, ListAllJobsHTML from .showjob import ShowJob, ShowJobHTML, JobShower from .removeghostjobs import RemoveGhostJobs from .removejob import RemoveJob -from .lstroves import LsTroves, ForceLsTrove +from .lsupstreams import LsUpstreams, ForceLsUpstream from .pretendtime import PretendTime from .maxjobs import GetMaxJobs, SetMaxJobs -from .gitano import ( - GitanoCommand, - LocalTroveGitanoCommand, - GitanoCommandFailure, - new_gitano_command) from .static import StaticFile from .proxy import setup_proxy -from .gerrit import Gerrit -from .gitlab import Gitlab +from . import gerrit +from . import gitano +from . import gitlab +from . import local + + +downstream_types = { + 'gerrit': gerrit.GerritDownstream, + 'gitano': gitano.GitanoDownstream, + 'gitlab': gitlab.GitlabDownstream, + 'local': local.LocalDownstream, +} + + +upstream_types = { + 'gitlab': gitlab.GitlabUpstream, + 'trove': gitano.TroveUpstream, +} + + +def get_upstream_host(host_info): + return upstream_types[host_info['type']](host_info) __all__ = locals() diff --git a/lorrycontroller/gerrit.py b/lorrycontroller/gerrit.py index 4e77ba7..c18f2ed 100644 --- a/lorrycontroller/gerrit.py +++ b/lorrycontroller/gerrit.py @@ -13,11 +13,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import logging import cliapp +from . import hosts -class Gerrit(object): + +class GerritDownstream(hosts.DownstreamHost): '''Run commands on a Gerrit instance. @@ -28,7 +31,12 @@ class Gerrit(object): ''' - def __init__(self, host, user, port=29418): + def __init__(self, app_settings): + # XXX These need to be configurable + host = 'localhost' + port = 29418 + user = 'lorry' + self._ssh_command_args = [ 'ssh', '-oStrictHostKeyChecking=no', '-oBatchMode=yes', '-p%i' % port, '%s@%s' % (user, host)] @@ -40,7 +48,7 @@ class Gerrit(object): out = out.decode('utf-8', errors='replace') return out - def has_project(self, name): + def _has_project(self, name): # There's no 'does this project exist' command in Gerrit 2.9.4; 'list # all projects with this prefix' is as close we can get. @@ -53,5 +61,28 @@ class Gerrit(object): else: return False - def create_project(self, name): - self._ssh_command(['gerrit', 'create-project', name]) + def prepare_repo(self, name, metadata): + '''Create a project in the local Gerrit server. + + The 'lorry' user must have createProject capability in the Gerrit. + + ''' + + if self._has_project(name): + logging.info('Project %s exists in local Gerrit already.', + name) + else: + self._ssh_command(['gerrit', 'create-project', name]) + logging.info('Created %s project in local Gerrit.', name) + + # We can only set this metadata if we're the owner of the + # repository. For now, ignore failures. + try: + if 'head' in metadata: + self._ssh_command(['gerrit', 'set-head', name, + '--new-head', metadata['head']]) + if 'description' in metadata: + self._ssh_command(['gerrit', 'set-project', name, + '-d', metadata['description']]) + except cliapp.AppException: + pass diff --git a/lorrycontroller/gitano.py b/lorrycontroller/gitano.py index 7d9c436..499bb5d 100644 --- a/lorrycontroller/gitano.py +++ b/lorrycontroller/gitano.py @@ -23,9 +23,10 @@ import cliapp import requests import lorrycontroller +from . import hosts -class GitanoCommandFailure(Exception): +class _GitanoCommandFailure(Exception): def __init__(self, trovehost, command, stderr): Exception.__init__( @@ -34,7 +35,7 @@ class GitanoCommandFailure(Exception): (command, trovehost, stderr)) -class GitanoCommand(object): +class _GitanoCommand(object): '''Run a Gitano command on a Trove.''' @@ -49,7 +50,7 @@ class GitanoCommand(object): elif protocol in ('http', 'https'): self._command = self._http_command else: - raise GitanoCommandFailure( + raise _GitanoCommandFailure( self.trovehost, '__init__', 'unknown protocol %s' % protocol) def whoami(self): @@ -98,7 +99,7 @@ class GitanoCommand(object): logging.error( 'Failed to run "%s" for %s:\n%s', quoted_args, self.trovehost, stdout + stderr) - raise GitanoCommandFailure( + raise _GitanoCommandFailure( self.trovehost, ' '.join(gitano_args), stdout + stderr) @@ -122,13 +123,13 @@ class GitanoCommand(object): else: response = requests.get(url) except (requests.exceptions.RequestException) as e: - raise GitanoCommandFailure( + raise _GitanoCommandFailure( self.trovehost, ' '.join(gitano_args), str(e)) return response.text -class LocalTroveGitanoCommand(GitanoCommand): +class _LocalTroveGitanoCommand(_GitanoCommand): '''Run commands on the local Trove's Gitano. @@ -138,14 +139,89 @@ class LocalTroveGitanoCommand(GitanoCommand): ''' def __init__(self): - GitanoCommand.__init__(self, 'localhost', 'ssh', '', '') + _GitanoCommand.__init__(self, 'localhost', 'ssh', '', '') -def new_gitano_command(statedb, trovehost): - trove_info = statedb.get_trove_info(trovehost) - return lorrycontroller.GitanoCommand( - trovehost, - trove_info['protocol'], - trove_info['username'], - trove_info['password']) +class GitanoDownstream(hosts.DownstreamHost): + def __init__(self, app_settings): + self._gitano = _LocalTroveGitanoCommand() + + def prepare_repo(self, repo_path, metadata): + # Create repository on local Trove. If it fails, assume + # it failed because the repository already existed, and + # ignore the failure (but log message). + + try: + self._gitano.create(repo_path) + except _GitanoCommandFailure as e: + logging.debug( + 'Ignoring error creating %s on local Trove: %s', + repo_path, e) + else: + logging.info('Created %s on local repo', repo_path) + + try: + local_config = self._gitano.get_gitano_config(repo_path) + if 'head' in metadata \ + and metadata['head'] != local_config['project.head']: + self._gitano.set_gitano_config(repo_path, + 'project.head', + metadata['head']) + if 'description' in metadata \ + and metadata['description'] != \ + local_config['project.description']: + self._gitano.set_gitano_config(repo_path, + 'project.description', + metadata['description']) + except _GitanoCommandFailure as e: + logging.error('ERROR: %s' % str(e)) + # FIXME: We need a good way to report these errors to the + # user. However, we probably don't want to fail the + # request, so that's not the way to do this. Needs + # thinking. + + +class TroveUpstream(hosts.UpstreamHost): + def __init__(self, host_info): + self._host_info = host_info + self._gitano = _GitanoCommand(host_info['host'], + host_info['protocol'], + host_info['username'], + host_info['password']) + + def list_repos(self): + ls_output = self._gitano.ls() + repo_paths = [] + for line in ls_output.splitlines(): + words = line.split(None, 1) + if words[0].startswith('R') and len(words) == 2: + repo_paths.append(words[1]) + return repo_paths + + def get_repo_url(self, remote_path): + vars = dict(self._host_info) + vars['remote_path'] = remote_path + + patterns = { + 'ssh': 'ssh://git@{host}/{remote_path}', + 'https':'https://{username}:{password}@{host}/git/{remote_path}', + 'http': 'http://{host}/git/{remote_path}', + } + + return patterns[self._host_info['protocol']].format(**vars) + + def get_repo_metadata(self, repo_path): + try: + remote_config = self._gitano.get_gitano_config(repo_path) + return { + 'head': remote_config['project.head'], + 'description': remote_config['project.description'], + } + except _GitanoCommandFailure as e: + logging.error('ERROR: %s' % str(e)) + # FIXME: We need a good way to report these errors to the + # user. However, we probably don't want to fail the + # request, so that's not the way to do this. Needs + # thinking. + return {} diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py index 6938cae..4f70f0a 100644 --- a/lorrycontroller/gitlab.py +++ b/lorrycontroller/gitlab.py @@ -13,113 +13,133 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +''' +Run commands on a GitLab instance. +This uses the python wrapper around the GitLab API. +Use of the API requires the private token of a user with master access +to the targetted group. +''' + +import logging import re import urllib.parse -import itertools + try: import gitlab except ImportError: gitlab = None +from . import hosts + class MissingGitlabModuleError(Exception): pass -class Gitlab(object): +def _init_gitlab(host, token): + if gitlab: + url = "http://" + host + return gitlab.Gitlab(url, token) + else: + raise MissingGitlabModuleError('gitlab module missing\n' + '\tpython-gitlab is required with GitLab as the git server') - '''Run commands on a GitLab instance. - This uses the python wrapper around the GitLab API. - Use of the API requires the private token of a user with master access - to the targetted group. +class GitlabDownstream(hosts.DownstreamHost): + @staticmethod + def add_app_settings(app_settings): + app_settings.string( + ['gitlab-private-token'], + 'private token for GitLab API access') - ''' + @staticmethod + def check_app_settings(app_settings): + if not app_settings['gitlab-private-token']: + logging.error('A private token must be provided to create ' + 'repositories on a GitLab instance.') + app_settings.require('gitlab-private-token') - def __init__(self, host, token): - if gitlab: - url = "http://" + host - self.gl = gitlab.Gitlab(url, token) - else: - raise MissingGitlabModuleError('gitlab module missing\n' - '\tpython-gitlab is required with GitLab as the git server') + def __init__(self, app_settings): + # XXX This needs to be configurable + host = 'localhost' - def first(self, predicate, iterable): - return next(filter(predicate, iterable)) + self.gl = _init_gitlab(host, app_settings['gitlab-private-token']) - def split_and_unslashify_path(self, path): - group, project = path.split('/', 1) - return group, project.replace('/', '_') + def prepare_repo(self, repo_path, metadata): - def find_project(self, repo_path): - group, project = self.split_and_unslashify_path(repo_path) - predicate = lambda x: x.namespace.name == group and x.name == project - - return self.first(predicate, self.gl.projects.search(project)) - - def has_project(self, repo_path): - try: - return bool(self.find_project(repo_path)) - except StopIteration: - return False - - def create_project(self, repo_path): - # GitLab only supports one level of namespacing. - group_name, project_name = self.split_and_unslashify_path(repo_path) - group = None try: - group = self.gl.groups.get(group_name) - except gitlab.GitlabGetError as e: - if e.response_code == 404: - group = self.gl.groups.create( - {'name': group_name, 'path': group_name}) + project = self.gl.projects.get(repo_path) + except gitlab.GitlabGetError: + pass + else: + logging.info('Project %s exists in local GitLab already.', + repo_path) + if 'head' in metadata \ + and project.default_branch != metadata['head']: + project.default_branch = metadata['head'] + if 'description' in metadata \ + and project.description != metadata['description']: + project.description = metadata['description'] + project.save() + return + + path_comps = repo_path.split('/') + + if len(path_comps) < 2: + raise ValueError('cannot create GitLab project outside a group') + + # Create hierarchy of groups as necessary + parent_group = None + for group_name in path_comps[:-1]: + if parent_group is None: + group_path = group_name else: - raise + group_path = parent_group.full_path + '/' + group_name + try: + group = self.gl.groups.get(group_path) + except gitlab.GitlabGetError as e: + if e.response_code != 404: + raise + data = {'name': group_name, 'path': group_name} + if parent_group is not None: + data['parent_id'] = parent_group.id + group = self.gl.groups.create(data) + parent_group = group project = { - 'name': project_name, + 'name': path_comps[-1], 'public': True, 'merge_requests_enabled': False, 'namespace_id': group.id, - # Set the original path in the description. We will use this to - # work around lack of multi-level namespacing. - 'description': 'original_path: %s' % repo_path + 'default_branch': metadata.get('head'), + 'description': metadata.get('description'), } self.gl.projects.create(project) - def try_get_original_path(self, project_description): - match = re.search('original_path:\s(.*)', str(project_description)) - if match: - return match.groups()[0] - - def suitable_path(self, project): - '''Return a path for a downstream Lorry Controller instance to consume. - - Should the path that was lorried have contained more than one level of - namespacing (more than one '/' within the repository path), then for - GitLab to handle this, we replace any '/'s (remaining in the project - name after extracting the group name) with underscores (_). To preserve - the original path, we set the 'original_path' within the project - description. - This method will attempt to return 'original_path' if it was set, - otherwise it will return the 'path_with_namespace', being of the format - 'group_name/project_name', rather than 'group_name/project/name'. - ''' - return (self.try_get_original_path(project.description) or - project.path_with_namespace) + logging.info('Created %s project in local GitLab.', repo_path) - def list_projects(self): - '''List projects on a GitLab instance. - In attempt to handle GitLab's current lack of multi-level namespacing - (see: https://gitlab.com/gitlab-org/gitlab-ce/issues/2772), return - the 'original_path' stored in a project's description, if it exists. - ''' +class GitlabUpstream(hosts.UpstreamHost): + @staticmethod + def check_host_type_params(validator, section): + validator.check_has_required_fields(section, ['private-token']) + + @staticmethod + def get_host_type_params(section): + return {'private-token': section['private-token']} + + def __init__(self, host_info): + self._protocol = host_info['protocol'] + self.gl = _init_gitlab(host_info['host'], + host_info['type_params']['private-token']) + + def list_repos(self): + '''List projects on a GitLab instance.''' - return [self.suitable_path(x) for x in self.gl.projects.list()] + return [x.path_with_namespace for x in self.gl.projects.list()] - def get_project_url(self, protocol, project_path): + def get_repo_url(self, repo_path): '''Return the clone url for a GitLab project. Depending on the protocol specified, will return a suitable clone url. @@ -132,11 +152,20 @@ class Gitlab(object): format matching 'http(s)://host/group/project.git'. ''' - project = self.find_project(project_path) + project = self.gl.projects.get(repo_path) - if protocol == 'ssh': + if self._protocol == 'ssh': return project.ssh_url_to_repo - elif protocol in ('http', 'https'): + elif self._protocol in ('http', 'https'): split = urllib.parse.urlsplit(project.http_url_to_repo) return urllib.parse.urlunsplit(( - protocol, split.netloc, split.path, '', '')) + self._protocol, split.netloc, split.path, '', '')) + + def get_repo_metadata(self, repo_path): + project = self.gl.projects.get(repo_path) + metadata = {} + if project.default_branch is not None: + metadata['head'] = project.default_branch + if project.description is not None: + metadata['description'] = project.description + return metadata diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index a893036..6736b35 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -36,9 +36,12 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): now = statedb.get_current_time() for lorry_info in lorry_infos: if self.ready_to_run(lorry_info, now): - self.create_repository(statedb, lorry_info) - if lorry_info['from_trovehost']: - self.copy_repository_metadata(statedb, lorry_info) + metadata = self.get_repo_metadata(statedb, lorry_info) + downstream_type = lorrycontroller.downstream_types[ + self.app_settings['git-server-type']] + downstream_type(self.app_settings) \ + .prepare_repo(lorry_info['path'], metadata) + self.give_job_to_minion(statedb, lorry_info, now) logging.info( 'Giving job %s to lorry %s to MINION %s:%s', @@ -62,98 +65,36 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): due = lorry_info['last_run'] + lorry_info['interval'] return (lorry_info['running_job'] is None and due <= now) - def create_repository(self, statedb, lorry_info): - api = self.app_settings['git-server-type'] - if api == 'gitano': - self.create_repository_in_local_trove(statedb, lorry_info) - elif api == 'gerrit': - self.create_gerrit_project(statedb, lorry_info) - elif api == 'gitlab': - self.create_gitlab_project(statedb, lorry_info) - elif api == 'local': - pass - - def create_repository_in_local_trove(self, statedb, lorry_info): - # Create repository on local Trove. If it fails, assume - # it failed because the repository already existed, and - # ignore the failure (but log message). - - local = lorrycontroller.LocalTroveGitanoCommand() - try: - local.create(lorry_info['path']) - except lorrycontroller.GitanoCommandFailure as e: - logging.debug( - 'Ignoring error creating %s on local Trove: %s', - lorry_info['path'], e) - else: - logging.info('Created %s on local repo', lorry_info['path']) - - def create_gerrit_project(self, statedb, lorry_info): - '''Create a project in the local Gerrit server. - - The 'lorry' user must have createProject capability in the Gerrit. - - ''' - gerrit = lorrycontroller.Gerrit( - host='localhost', user='lorry') - project_name = lorry_info['path'] - - if gerrit.has_project(project_name): - logging.info('Project %s exists in local Gerrit already.', - project_name) - else: - gerrit.create_project(project_name) - logging.info('Created %s project in local Gerrit.', project_name) - - def create_gitlab_project(self, statedb, lorry_info): - gitlab = lorrycontroller.Gitlab( - 'localhost', self.app_settings['gitlab-private-token']) - project_name = lorry_info['path'] - - if gitlab.has_project(project_name): - logging.info('Project %s exists in local GitLab already.', - project_name) - else: - gitlab.create_project(lorry_info['path']) - logging.info('Created %s project in local GitLab.', project_name) - - def copy_repository_metadata(self, statedb, lorry_info): - '''Copy project.head and project.description to the local Trove.''' - - assert lorry_info['from_trovehost'] - assert lorry_info['from_path'] + def get_repo_metadata(self, statedb, lorry_info): + '''Get repository head and description.''' - if self.app_settings['git-server-type'] != 'gitano': - # FIXME: would be good to have this info in Gerrit too - return + if not lorry_info['from_host']: + return {} - remote = lorrycontroller.new_gitano_command(statedb, lorry_info['from_trovehost']) - local = lorrycontroller.LocalTroveGitanoCommand() + assert lorry_info['from_path'] try: - remote_config = remote.get_gitano_config(lorry_info['from_path']) - local_config = local.get_gitano_config(lorry_info['path']) - - if remote_config['project.head'] != local_config['project.head']: - local.set_gitano_config( - lorry_info['path'], - 'project.head', - remote_config['project.head']) - - if not local_config['project.description']: - desc = '{host}: {desc}'.format( - host=lorry_info['from_trovehost'], - desc=remote_config['project.description']) - local.set_gitano_config( - lorry_info['path'], - 'project.description', - desc) - except lorrycontroller.GitanoCommandFailure as e: - logging.error('ERROR: %s' % str(e)) - # FIXME: We need a good way to report these errors to the - # user. However, we probably don't want to fail the - # request, so that's not the way to do this. Needs - # thinking. + host_info = statedb.get_host_info(lorry_info['from_host']) + except lorrycontroller.HostNotFoundError: + # XXX We don't know whether upstream is Trove. It should be + # possible to set host type for single repositories. + host_info = { + 'host': lorry_info['from_host'], + 'protocol': 'ssh', + 'username': None, + 'password': None, + 'type': 'trove', + 'type_params': {}, + } + + metadata = lorrycontroller.get_upstream_host(host_info) \ + .get_repo_metadata(lorry_info['from_path']) + if 'description' in metadata: + # Prepend Upstream Host name + metadata['description'] = '{host}: {desc}'.format( + host=lorry_info['from_host'], + desc=metadata['description']) + return metadata def give_job_to_minion(self, statedb, lorry_info, now): path = lorry_info['path'] diff --git a/lorrycontroller/hosts.py b/lorrycontroller/hosts.py new file mode 100644 index 0000000..39cae57 --- /dev/null +++ b/lorrycontroller/hosts.py @@ -0,0 +1,121 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import abc + + +class DownstreamHost(abc.ABC): + @staticmethod + def add_app_settings(app_settings): + '''Add any application settings that are specific to this Downstream + Host type. + ''' + pass + + @staticmethod + def check_app_settings(app_settings): + '''Validate any fields in the application settings that are specific + to this Downstream Host type. + ''' + pass + + @abc.abstractmethod + def __init__(self, app_settings): + '''Construct a Downstream Host connector from the application + settings. + ''' + pass + + @abc.abstractmethod + def prepare_repo(self, repo_path, metadata): + '''Prepare a repository on the Host. If the repository does not + exist, this method must create it. It should also set any + given metadata on the repository, whether or not it already + exists. + + repo_path is the path that the repository should appear at + within the Host. + + metadata is a dictionary with the following (optional) keys + defined: + + - head: Name of the default branch (a.k.a. HEAD) + - description: Short string describing the repository + ''' + pass + + +class UpstreamHost(abc.ABC): + @staticmethod + def check_host_type_params(validator, section): + '''Validate any type-specific fields in a CONFGIT host section. + + validator is an instance of LorryControllerConfValidator that + may be used to check the types of configuration fields. + + section is the dictionary of fields for the section. + + Returns None if the configuration is valid; raises an + exception on error. + ''' + pass + + @staticmethod + def get_host_type_params(section): + '''Convert any type-specific fields in a CONFGIT host section into a + dictionary that will be stored in STATEDB. + + section is the dictionary of fields for the section. + + Returns a dictionary, which may be empty. This will be stored + in STATEDB as the type_params of the host. + ''' + return {} + + @abc.abstractmethod + def __init__(self, host_info): + '''Construct an Upstream Host connector from the given host_info. + The host_info comes directly from STATEDB. + ''' + pass + + @abc.abstractmethod + def list_repos(self): + '''List all visible repositories on the Host. + + Returns a list of path strings. + ''' + pass + + @abc.abstractmethod + def get_repo_url(self, repo_path): + '''Get URL for a repository. + + repo_path is the path to the repository within the Host. + + Returns a URL string suitable for passing to git clone. + ''' + pass + + @abc.abstractmethod + def get_repo_metadata(self, repo_path): + '''Get metadata for a repository. + + repo_path is the path to the repository within the Host. + + Returns a dictionary of metadata suitable for passing to + DownstreamHost.prepare_repo. + ''' + pass diff --git a/lorrycontroller/local.py b/lorrycontroller/local.py new file mode 100644 index 0000000..d55214d --- /dev/null +++ b/lorrycontroller/local.py @@ -0,0 +1,56 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging +import os +import os.path + +import cliapp + +from . import hosts + + +class LocalDownstream(hosts.DownstreamHost): + @staticmethod + def add_app_settings(app_settings): + app_settings.string( + ['local-base-directory'], + 'Base directory for local Downstream Host') + + @staticmethod + def check_app_settings(app_settings): + if not app_settings['local-base-directory']: + logging.error('A base directory must be provided to create ' + 'repositories on a local filesystem.') + app_settings.require('local-base-directory') + + def __init__(self, app_settings): + self._base_dir = app_settings['local-base-directory'] + + def prepare_repo(self, repo_path, metadata): + repo_path = '%s/%s.git' % (self._base_dir, repo_path) + + # These are idempotent, so we don't need to explicitly check + # whether the repository already exists + os.makedirs(repo_path, exist_ok=True) + cliapp.runcmd(['git', 'init', '--bare', repo_path]) + + if 'head' in metadata: + cliapp.runcmd(['git', '--git-dir', repo_path, + 'symbolic-ref', 'HEAD', + 'refs/heads/' + metadata['head']]) + if 'description' in metadata: + with open(os.path.join(repo_path, 'description'), 'w') as f: + print(metadata['description'], file=f) diff --git a/lorrycontroller/lstroves.py b/lorrycontroller/lsupstreams.py index 34648cb..a535174 100644 --- a/lorrycontroller/lstroves.py +++ b/lorrycontroller/lsupstreams.py @@ -33,62 +33,46 @@ class ServerLsError(Exception): self.remote_host = remote_host -class TroveRepositoryLister(object): +class HostRepositoryLister(object): def __init__(self, app_settings, route): self.app_settings = app_settings self.route = route - def list_trove_into_statedb(self, statedb, trove_info): - remote_paths = self.ls(statedb, trove_info) - remote_paths = self.skip_ignored_repos(trove_info, remote_paths) + def list_host_into_statedb(self, statedb, host_info): + remote_paths = self.ls(statedb, host_info) + remote_paths = self.skip_ignored_repos(host_info, remote_paths) repo_map = self.map_remote_repos_to_local_ones( - trove_info, remote_paths) + host_info, remote_paths) with statedb: - self.update_lorries_for_trove(statedb, trove_info, repo_map) + self.update_lorries_for_host(statedb, host_info, repo_map) now = statedb.get_current_time() - statedb.set_trove_ls_last_run(trove_info['trovehost'], now) + statedb.set_host_ls_last_run(host_info['host'], now) - def ls(self, statedb, trove_info): - if self.app_settings['debug-fake-trove']: - repo_paths = self.get_fake_ls_output(trove_info) + def ls(self, statedb, host_info): + if self.app_settings['debug-fake-upstream-host']: + repo_paths = self.get_fake_ls_output(host_info) else: - repo_paths = self.get_real_ls_output(statedb, trove_info) + repo_paths = self.get_real_ls_output(host_info) return repo_paths - def get_fake_ls_output(self, trove_info): - trovehost = trove_info['trovehost'] - for item in self.app_settings['debug-fake-trove']: - host, path = item.split('=', 1) - if host == trovehost: + def get_fake_ls_output(self, host_info): + host = host_info['host'] + for item in self.app_settings['debug-fake-upstream-host']: + fake_host, path = item.split('=', 1) + if fake_host == host: with open(path) as f: obj = json.load(f) return obj['ls-output'] return None - def get_real_ls_output(self, statedb, trove_info): - gitlab_token = trove_info.get('gitlab_token') - if gitlab_token: - return lorrycontroller.Gitlab( - trove_info['trovehost'], gitlab_token).list_projects() - - gitano = lorrycontroller.new_gitano_command( - statedb, trove_info['trovehost']) - output = gitano.ls() - return self.parse_ls_output(output) - - def parse_ls_output(self, ls_output): - repo_paths = [] - for line in ls_output.splitlines(): - words = line.split(None, 1) - if words[0].startswith('R') and len(words) == 2: - repo_paths.append(words[1]) - return repo_paths + def get_real_ls_output(self, host_info): + return lorrycontroller.get_upstream_host(host_info).list_repos() - def skip_ignored_repos(self, trovehost, repo_paths): - ignored_patterns = json.loads(trovehost['ignore']) + def skip_ignored_repos(self, host, repo_paths): + ignored_patterns = json.loads(host['ignore']) ignored_paths = set() for pattern in ignored_patterns: @@ -96,9 +80,9 @@ class TroveRepositoryLister(object): return set(repo_paths).difference(ignored_paths) - def map_remote_repos_to_local_ones(self, trove_info, remote_paths): + def map_remote_repos_to_local_ones(self, host_info, remote_paths): '''Return a dict that maps each remote repo path to a local one.''' - prefixmap = self.parse_prefixmap(trove_info['prefixmap']) + prefixmap = self.parse_prefixmap(host_info['prefixmap']) repo_map = {} for remote_path in remote_paths: local_path = self.map_one_remote_repo_to_local_one( @@ -124,29 +108,29 @@ class TroveRepositoryLister(object): def path_starts_with_prefix(self, path, prefix): return path.startswith(prefix) and path[len(prefix):].startswith('/') - def update_lorries_for_trove(self, statedb, trove_info, repo_map): - trovehost = trove_info['trovehost'] + def update_lorries_for_host(self, statedb, host_info, repo_map): + host = host_info['host'] for remote_path, local_path in list(repo_map.items()): - lorry = self.construct_lorry(trove_info, local_path, remote_path) + lorry = self.construct_lorry(host_info, local_path, remote_path) statedb.add_to_lorries( path=local_path, text=json.dumps(lorry, indent=4), - from_trovehost=trovehost, + from_host=host, from_path=remote_path, - interval=trove_info['lorry_interval'], - timeout=trove_info['lorry_timeout']) + interval=host_info['lorry_interval'], + timeout=host_info['lorry_timeout']) - all_local_paths = set(statedb.get_lorries_for_trove(trovehost)) + all_local_paths = set(statedb.get_lorries_for_host(host)) wanted_local_paths = set(repo_map.values()) delete_local_paths = all_local_paths.difference(wanted_local_paths) for local_path in delete_local_paths: statedb.remove_lorry(local_path) - def construct_lorry(self, trove_info, local_path, remote_path): + def construct_lorry(self, host_info, local_path, remote_path): return { local_path: { 'type': 'git', - 'url': self.construct_lorry_url(trove_info, remote_path), + 'url': self.construct_lorry_url(host_info, remote_path), 'refspecs': [ "+refs/heads/*", "+refs/tags/*", @@ -154,27 +138,12 @@ class TroveRepositoryLister(object): } } - def construct_lorry_url(self, trove_info, remote_path): - gitlab_token = trove_info.get('gitlab_token') - if gitlab_token: - return lorrycontroller.Gitlab( - trove_info['trovehost'], gitlab_token).get_project_url( - trove_info['protocol'], remote_path) - - vars = dict(trove_info) - vars['remote_path'] = remote_path - - patterns = { - 'ssh': 'ssh://git@{trovehost}/{remote_path}', - 'https': - 'https://{username}:{password}@{trovehost}/git/{remote_path}', - 'http': 'http://{trovehost}/git/{remote_path}', - } - - return patterns[trove_info['protocol']].format(**vars) + def construct_lorry_url(self, host_info, remote_path): + return lorrycontroller.get_upstream_host(host_info) \ + .get_repo_url(remote_path) -class ForceLsTrove(lorrycontroller.LorryControllerRoute): +class ForceLsUpstream(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/force-ls-trove' @@ -182,20 +151,20 @@ class ForceLsTrove(lorrycontroller.LorryControllerRoute): def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) - trovehost = bottle.request.forms.trovehost + host = bottle.request.forms.host statedb = self.open_statedb() - lister = TroveRepositoryLister(self.app_settings, self) - trove_info = statedb.get_trove_info(trovehost) + lister = HostRepositoryLister(self.app_settings, self) + host_info = statedb.get_host_info(host) try: - updated = lister.list_trove_into_statedb(statedb, trove_info) + updated = lister.list_host_into_statedb(statedb, host_info) except ServerLsError as e: raise bottle.abort(500, str(e)) return { 'updated-troves': updated } -class LsTroves(lorrycontroller.LorryControllerRoute): +class LsUpstreams(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/ls-troves' @@ -204,30 +173,30 @@ class LsTroves(lorrycontroller.LorryControllerRoute): logging.info('%s %s called', self.http_method, self.path) statedb = self.open_statedb() - lister = TroveRepositoryLister(self.app_settings, self) + lister = HostRepositoryLister(self.app_settings, self) - trove_infos = self.get_due_troves(statedb) - for trove_info in trove_infos: - logging.info('Trove %r is due an ls', trove_info['trovehost']) + host_infos = self.get_due_hosts(statedb) + for host_info in host_infos: + logging.info('Host %r is due an ls', host_info['host']) try: - lister.list_trove_into_statedb(statedb, trove_info) + lister.list_host_into_statedb(statedb, host_info) except ServerLsError as e: bottle.abort(500, str(e)) return { - 'updated-troves': [trove_info['trovehost'] for trove_info in trove_infos], + 'updated-troves': [host_info['host'] for host_info in host_infos], } - def get_due_troves(self, statedb): - trove_infos = [ - statedb.get_trove_info(trovehost) - for trovehost in statedb.get_troves()] + def get_due_hosts(self, statedb): + host_infos = [ + statedb.get_host_info(host) + for host in statedb.get_hosts()] now = statedb.get_current_time() return [ - trove_info - for trove_info in trove_infos - if self.is_due(trove_info, now)] + host_info + for host_info in host_infos + if self.is_due(host_info, now)] - def is_due(self, trove_info, now): - ls_due = trove_info['ls_last_run'] + trove_info['ls_interval'] + def is_due(self, host_info, now): + ls_due = host_info['ls_last_run'] + host_info['ls_interval'] return ls_due <= now diff --git a/lorrycontroller/migrations/0003-generalise-troves.py b/lorrycontroller/migrations/0003-generalise-troves.py new file mode 100644 index 0000000..bacd2ad --- /dev/null +++ b/lorrycontroller/migrations/0003-generalise-troves.py @@ -0,0 +1,41 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import yoyo + + +yoyo.step('CREATE TABLE hosts (' + 'host TEXT PRIMARY KEY, ' + 'protocol TEXT, ' + 'username TEXT, ' + 'password TEXT, ' + 'type TEXT NOT NULL, ' + 'type_params TEXT NOT NULL, ' + 'lorry_interval INT, ' + 'lorry_timeout INT, ' + 'ls_interval INT, ' + 'ls_last_run INT, ' + 'prefixmap TEXT, ' + 'ignore TEXT ' + ')') +yoyo.step('INSERT INTO hosts ' + 'SELECT trovehost, protocol, username, password, ' + "CASE WHEN gitlab_token IS NULL THEN 'trove' ELSE 'gitlab' END, " + "CASE WHEN gitlab_token IS NULL THEN '{}' " + "ELSE json_object('private-token', gitlab_token) END, " + 'lorry_interval, lorry_timeout, ls_interval, ls_last_run, prefixmap, ' + 'ignore ' + 'FROM troves') +yoyo.step('DROP TABLE troves') diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py index 4e162a9..3303f68 100644 --- a/lorrycontroller/readconf.py +++ b/lorrycontroller/readconf.py @@ -61,7 +61,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): statedb = self.open_statedb() with statedb: lorries_to_remove = set(statedb.get_lorries_paths()) - troves_to_remove = set(statedb.get_troves()) + hosts_to_remove = set(statedb.get_hosts()) for section in conf_obj: if not 'type' in section: @@ -70,13 +70,13 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): added = self.add_matching_lorries_to_statedb( statedb, section) lorries_to_remove = lorries_to_remove.difference(added) - elif section['type'] in ('trove', 'troves', 'gitlab'): - self.add_trove(statedb, section) - trovehost = section.get('host') or section['trovehost'] - if trovehost in troves_to_remove: - troves_to_remove.remove(trovehost) + elif section['type'] in lorrycontroller.upstream_types: + self.add_host(statedb, section) + host = section.get('host') or section['trovehost'] + if host in hosts_to_remove: + hosts_to_remove.remove(host) lorries_to_remove = lorries_to_remove.difference( - statedb.get_lorries_for_trove(trovehost)) + statedb.get_lorries_for_host(host)) else: logging.error( 'Unknown section in configuration: %r', section) @@ -87,9 +87,9 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): for path in lorries_to_remove: statedb.remove_lorry(path) - for trovehost in troves_to_remove: - statedb.remove_trove(trovehost) - statedb.remove_lorries_for_trovehost(trovehost) + for host in hosts_to_remove: + statedb.remove_host(host) + statedb.remove_lorries_for_host(host) if 'redirect' in bottle.request.forms: bottle.redirect(bottle.request.forms.redirect) @@ -215,7 +215,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): old_lorry_info = None statedb.add_to_lorries( - path=path, text=text, from_trovehost='', from_path='', + path=path, text=text, from_host='', from_path='', interval=interval, timeout=timeout) added_paths.add(path) @@ -282,7 +282,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): new_obj = { path: obj } return json.dumps(new_obj, indent=4) - def add_trove(self, statedb, section): + def add_host(self, statedb, section): username = None password = None if 'auth' in section: @@ -290,22 +290,22 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): username = auth.get('username') password = auth.get('password') - gitlab_token = None - if section['type'] == 'gitlab': - gitlab_token = section['private-token'] + type_params = lorrycontroller.upstream_types[section['type']] \ + .get_host_type_params(section) - statedb.add_trove( - trovehost=section.get('host') or section['trovehost'], + statedb.add_host( + host=section.get('host') or section['trovehost'], protocol=section['protocol'], username=username, password=password, + host_type=section['type'], + type_params=type_params, lorry_interval=section['interval'], lorry_timeout=section.get( 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT), ls_interval=section['ls-interval'], prefixmap=json.dumps(section['prefixmap']), - ignore=json.dumps(section.get('ignore', [])), - gitlab_token=gitlab_token) + ignore=json.dumps(section.get('ignore', []))) class ValidationError(Exception): @@ -318,19 +318,22 @@ class LorryControllerConfValidator(object): def validate_config(self, conf_obj): try: - self._check_is_list(conf_obj) - self._check_is_list_of_dicts(conf_obj) + self.check_is_list(conf_obj) + self.check_is_list_of_dicts(conf_obj) for section in conf_obj: if 'type' not in section: raise ValidationError( 'section without type: %r' % section) - elif section['type'] in ('trove', 'troves'): - self._check_troves_section(section) + # Backward compatibility + if section['type'] == 'troves': + section['type'] = 'trove' + if section['type'] in lorrycontroller.upstream_types: + self._check_host_section(section) + lorrycontroller.upstream_types[section['type']] \ + .check_host_type_params(self, section) elif section['type'] == 'lorries': self._check_lorries_section(section) - elif section['type'] == 'gitlab': - self._check_gitlab_section(section) else: raise ValidationError( 'unknown section type %r' % section['type']) @@ -339,31 +342,26 @@ class LorryControllerConfValidator(object): return None - def _check_is_list(self, conf_obj): + def check_is_list(self, conf_obj): if type(conf_obj) is not list: raise ValidationError( 'type %r is not a JSON list' % type(conf_obj)) - def _check_is_list_of_dicts(self, conf_obj): + def check_is_list_of_dicts(self, conf_obj): for item in conf_obj: if type(item) is not dict: raise ValidationError('all items must be dicts') - def _check_gitlab_section(self, section): - # gitlab section inherits trove configurations, perform the same checks. - self._check_troves_section(section) - self._check_has_required_fields(section, ['private-token']) - - def _check_troves_section(self, section): + def _check_host_section(self, section): if not any(i in ('trovehost', 'host') for i in section): - self._check_has_required_fields(section, ['host']) - self._check_has_required_fields( + self.check_has_required_fields(section, ['host']) + self.check_has_required_fields( section, ['protocol', 'interval', 'ls-interval', 'prefixmap']) self._check_protocol(section) self._check_prefixmap(section) if 'ignore' in section: - self._check_is_list_of_strings(section, 'ignore') + self.check_is_list_of_strings(section, 'ignore') def _check_protocol(self, section): valid = ('ssh', 'http', 'https') @@ -380,18 +378,18 @@ class LorryControllerConfValidator(object): pass def _check_lorries_section(self, section): - self._check_has_required_fields( + self.check_has_required_fields( section, ['interval', 'prefix', 'globs']) - self._check_is_list_of_strings(section, 'globs') + self.check_is_list_of_strings(section, 'globs') - def _check_has_required_fields(self, section, fields): + def check_has_required_fields(self, section, fields): for field in fields: if field not in section: raise ValidationError( 'mandatory field %s missing in section %r' % (field, section)) - def _check_is_list_of_strings(self, section, field): + def check_is_list_of_strings(self, section, field): obj = section[field] if not isinstance(obj, list) or not all( isinstance(s, str) for s in obj): diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index b2bac7e..2dd30f0 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -13,7 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +import json import logging import os import sqlite3 @@ -39,11 +39,11 @@ class WrongNumberLorriesRunningJob(Exception): (row_count, job_id)) -class TroveNotFoundError(Exception): +class HostNotFoundError(Exception): - def __init__(self, trovehost): + def __init__(self, host): Exception.__init__( - self, 'Trove %s not known in STATEDB' % trovehost) + self, 'Host %s not known in STATEDB' % host) class StateDB(object): @@ -57,20 +57,20 @@ class StateDB(object): self._transaction_started = None self.initial_lorries_fields = [ - ('path', 'TEXT PRIMARY KEY'), - ('text', 'TEXT'), - ('from_trovehost', 'TEXT'), - ('from_path', 'TEXT'), - ('running_job', 'INT'), - ('last_run', 'INT'), - ('interval', 'INT'), - ('lorry_timeout', 'INT'), - ('disk_usage', 'INT'), + ('path', 'TEXT PRIMARY KEY', None), + ('text', 'TEXT', None), + ('from_trovehost', 'TEXT', 'from_host'), + ('from_path', 'TEXT', None), + ('running_job', 'INT', None), + ('last_run', 'INT', None), + ('interval', 'INT', None), + ('lorry_timeout', 'INT', None), + ('disk_usage', 'INT', None), ] self.lorries_fields = list(self.initial_lorries_fields) self.lorries_fields.extend([ - ('last_run_exit', 'TEXT'), - ('last_run_error', 'TEXT'), + ('last_run_exit', 'TEXT', None), + ('last_run_error', 'TEXT', None), ]) self.lorries_booleans = [ ] @@ -110,11 +110,16 @@ class StateDB(object): logging.debug('Initialising tables in database') c = self._conn.cursor() + # Note that this creates the *original* schema, which will + # then be updated by the migrations (_perform_any_migrations + # above). Since we did not use yoyo originally, this can't + # be moved to a migration. + # Table for holding the "are we scheduling jobs" value. c.execute('CREATE TABLE running_queue (running INT)') c.execute('INSERT INTO running_queue VALUES (1)') - # Table for known remote Troves. + # Table for known remote Hosts. c.execute( 'CREATE TABLE troves (' @@ -133,7 +138,8 @@ class StateDB(object): # Table for all the known lorries (the "run queue"). fields_sql = ', '.join( - '%s %s' % (name, info) for name, info in self.initial_lorries_fields + '%s %s' % (column, info) + for column, info, key in self.initial_lorries_fields ) c.execute('CREATE TABLE lorries (%s)' % fields_sql) @@ -231,42 +237,45 @@ class StateDB(object): self.get_cursor().execute( 'UPDATE running_queue SET running = ?', str(new_value)) - def get_trove_info(self, trovehost): + def get_host_info(self, host): c = self.get_cursor() c.execute( - 'SELECT protocol, username, password, lorry_interval, ' - 'lorry_timeout, ls_interval, ls_last_run, ' - 'prefixmap, ignore, gitlab_token ' - 'FROM troves WHERE trovehost IS ?', - (trovehost,)) + 'SELECT protocol, username, password, type, type_params, ' + 'lorry_interval, lorry_timeout, ls_interval, ls_last_run, ' + 'prefixmap, ignore ' + 'FROM hosts WHERE host IS ?', + (host,)) row = c.fetchone() if row is None: - raise lorrycontroller.TroveNotFoundError(trovehost) + raise lorrycontroller.HostNotFoundError(host) return { - 'trovehost': trovehost, + 'host': host, 'protocol': row[0], 'username': row[1], 'password': row[2], - 'lorry_interval': row[3], - 'lorry_timeout': row[4], - 'ls_interval': row[5], - 'ls_last_run': row[6], - 'prefixmap': row[7], - 'ignore': row[8], - 'gitlab_token': row[9] + 'type': row[3], + 'type_params': json.loads(row[4]), + 'lorry_interval': row[5], + 'lorry_timeout': row[6], + 'ls_interval': row[7], + 'ls_last_run': row[8], + 'prefixmap': row[9], + 'ignore': row[10], } - def add_trove(self, trovehost=None, protocol=None, username=None, - password=None, lorry_interval=None, - lorry_timeout=None, ls_interval=None, - prefixmap=None, ignore=None, gitlab_token=None): + def add_host(self, host=None, protocol=None, username=None, + password=None, host_type=None, type_params={}, + lorry_interval=None, lorry_timeout=None, ls_interval=None, + prefixmap=None, ignore=None): logging.debug( - 'StateDB.add_trove(%r,%r,%r,%r,%r,%r) called', - trovehost, lorry_interval, lorry_timeout, ls_interval, + 'StateDB.add_host(%r,%r,%r,%r,%r,%r) called', + host, lorry_interval, lorry_timeout, ls_interval, prefixmap, ignore) - assert trovehost is not None + assert host is not None assert protocol is not None + assert host_type is not None + assert isinstance(type_params, dict) assert lorry_interval is not None assert lorry_timeout is not None assert ls_interval is not None @@ -274,53 +283,57 @@ class StateDB(object): assert ignore is not None assert self.in_transaction + type_params = json.dumps(type_params) + try: - self.get_trove_info(trovehost) - except lorrycontroller.TroveNotFoundError: + self.get_host_info(host) + except lorrycontroller.HostNotFoundError: c = self.get_cursor() c.execute( - 'INSERT INTO troves ' - '(trovehost, protocol, username, password, ' + 'INSERT INTO hosts ' + '(host, protocol, username, password, type, type_params, ' 'lorry_interval, lorry_timeout, ' 'ls_interval, ls_last_run, ' - 'prefixmap, ignore, gitlab_token) ' - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - (trovehost, protocol, username, password, + 'prefixmap, ignore) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + (host, protocol, username, password, host_type, type_params, lorry_interval, lorry_timeout, ls_interval, 0, - prefixmap, ignore, gitlab_token)) + prefixmap, ignore)) else: c = self.get_cursor() c.execute( - 'UPDATE troves ' + 'UPDATE hosts ' 'SET lorry_interval=?, lorry_timeout=?, ls_interval=?, ' - 'prefixmap=?, ignore=?, protocol=?, gitlab_token=? ' - 'WHERE trovehost IS ?', + 'prefixmap=?, ignore=?, protocol=?, type_params=? ' + 'WHERE host IS ?', (lorry_interval, lorry_timeout, ls_interval, prefixmap, - ignore, protocol, gitlab_token, trovehost)) + ignore, protocol, type_params, host)) - def remove_trove(self, trovehost): - logging.debug('StateDB.remove_trove(%r) called', trovehost) + def remove_host(self, host): + logging.debug('StateDB.remove_host(%r) called', host) assert self.in_transaction c = self.get_cursor() - c.execute('DELETE FROM troves WHERE trovehost=?', (trovehost,)) + c.execute('DELETE FROM hosts WHERE host=?', (host,)) - def get_troves(self): + def get_hosts(self): c = self.get_cursor() - c.execute('SELECT trovehost FROM troves') + c.execute('SELECT host FROM hosts') return [row[0] for row in c.fetchall()] - def set_trove_ls_last_run(self, trovehost, ls_last_run): + def set_host_ls_last_run(self, host, ls_last_run): logging.debug( - 'StateDB.set_trove_ls_last_run(%r,%r) called', - trovehost, ls_last_run) + 'StateDB.set_host_ls_last_run(%r,%r) called', + host, ls_last_run) assert self.in_transaction c = self.get_cursor() c.execute( - 'UPDATE troves SET ls_last_run=? WHERE trovehost=?', - (ls_last_run, trovehost)) + 'UPDATE hosts SET ls_last_run=? WHERE host=?', + (ls_last_run, host)) def make_lorry_info_from_row(self, row): - result = dict((t[0], row[i]) for i, t in enumerate(self.lorries_fields)) + result = dict( + (key or column, row[i]) + for i, (column, info, key) in enumerate(self.lorries_fields)) for field in self.lorries_booleans: result[field] = bool(result[field]) return result @@ -345,27 +358,27 @@ class StateDB(object): for row in c.execute( 'SELECT path FROM lorries ORDER BY (last_run + interval)')] - def get_lorries_for_trove(self, trovehost): + def get_lorries_for_host(self, host): c = self.get_cursor() c.execute( - 'SELECT path FROM lorries WHERE from_trovehost IS ?', (trovehost,)) + 'SELECT path FROM lorries WHERE from_trovehost IS ?', (host,)) return [row[0] for row in c.fetchall()] - def add_to_lorries(self, path=None, text=None, from_trovehost=None, + def add_to_lorries(self, path=None, text=None, from_host=None, from_path=None, interval=None, timeout=None): logging.debug( 'StateDB.add_to_lorries(' - 'path=%r, text=%r, from_trovehost=%r, interval=%s, ' + 'path=%r, text=%r, from_host=%r, interval=%s, ' 'timeout=%r called', path, text, - from_trovehost, + from_host, interval, timeout) assert path is not None assert text is not None - assert from_trovehost is not None + assert from_host is not None assert from_path is not None assert interval is not None assert timeout is not None @@ -380,7 +393,7 @@ class StateDB(object): '(path, text, from_trovehost, from_path, last_run, interval, ' 'lorry_timeout, running_job) ' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', - (path, text, from_trovehost, from_path, 0, + (path, text, from_host, from_path, 0, interval, timeout, None)) else: c = self.get_cursor() @@ -389,7 +402,7 @@ class StateDB(object): 'SET text=?, from_trovehost=?, from_path=?, interval=?, ' 'lorry_timeout=? ' 'WHERE path IS ?', - (text, from_trovehost, from_path, interval, timeout, path)) + (text, from_host, from_path, interval, timeout, path)) def remove_lorry(self, path): logging.debug('StateDB.remove_lorry(%r) called', path) @@ -397,12 +410,12 @@ class StateDB(object): c = self.get_cursor() c.execute('DELETE FROM lorries WHERE path IS ?', (path,)) - def remove_lorries_for_trovehost(self, trovehost): + def remove_lorries_for_host(self, host): logging.debug( - 'StateDB.remove_lorries_for_trovest(%r) called', trovehost) + 'StateDB.remove_lorries_for_host(%r) called', host) assert self.in_transaction c = self.get_cursor() - c.execute('DELETE FROM lorries WHERE from_trovehost IS ?', (trovehost,)) + c.execute('DELETE FROM lorries WHERE from_trovehost IS ?', (host,)) def set_running_job(self, path, job_id): logging.debug( diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py index 2e6334d..cca8e8a 100644 --- a/lorrycontroller/status.py +++ b/lorrycontroller/status.py @@ -36,7 +36,7 @@ class StatusRenderer(object): 'timestamp': time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(now)), 'run_queue': self.get_run_queue(statedb), - 'troves': self.get_troves(statedb), + 'hosts': self.get_hosts(statedb), 'warning_msg': '', 'max_jobs': self.get_max_jobs(statedb), 'links': True, @@ -148,20 +148,20 @@ class StatusRenderer(object): return ' '.join(result) - def get_troves(self, statedb): - troves = [] - for trovehost in statedb.get_troves(): - trove_info = statedb.get_trove_info(trovehost) + def get_hosts(self, statedb): + hosts = [] + for host in statedb.get_hosts(): + host_info = statedb.get_host_info(host) - trove_info['ls_interval_nice'] = self.format_secs_nicely( - trove_info['ls_interval']) + host_info['ls_interval_nice'] = self.format_secs_nicely( + host_info['ls_interval']) - ls_due = trove_info['ls_last_run'] + trove_info['ls_interval'] + ls_due = host_info['ls_last_run'] + host_info['ls_interval'] now = int(statedb.get_current_time()) - trove_info['ls_due_nice'] = self.format_due_nicely(ls_due, now) + host_info['ls_due_nice'] = self.format_due_nicely(ls_due, now) - troves.append(trove_info) - return troves + hosts.append(host_info) + return hosts def get_max_jobs(self, statedb): max_jobs = statedb.get_max_jobs() |