From 4cce11af4ca0491b3da92321ddc44169909a6c26 Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Thu, 7 May 2020 21:40:01 +0100 Subject: Move Downstream Host type-specific code into modules * Introduce hosts module and DownstreamHost abstract base class * Define a subclass of this for each Downstream Host type * Define a name-type map for these in the lorrycontroller package * Use these classes to replace type-specific code elsewhere Related to #5. --- lorrycontroller/__init__.py | 13 +++++- lorrycontroller/gerrit.py | 29 +++++++++++-- lorrycontroller/gitano.py | 40 ++++++++++++++++++ lorrycontroller/gitlab.py | 70 ++++++++++++++++++++++--------- lorrycontroller/givemejob.py | 98 +++++++++----------------------------------- lorrycontroller/hosts.py | 57 ++++++++++++++++++++++++++ lorrycontroller/local.py | 26 ++++++++++++ 7 files changed, 230 insertions(+), 103 deletions(-) create mode 100644 lorrycontroller/hosts.py create mode 100644 lorrycontroller/local.py (limited to 'lorrycontroller') diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py index 64c4a6f..c00e142 100644 --- a/lorrycontroller/__init__.py +++ b/lorrycontroller/__init__.py @@ -44,8 +44,19 @@ from .gitano import ( new_gitano_command) from .static import StaticFile from .proxy import setup_proxy -from .gerrit import Gerrit from .gitlab import Gitlab +from . import gerrit +from . import gitano +from . import gitlab +from . import local + + +downstream_types = { + 'gerrit': gerrit.GerritDownstream, + 'gitano': gitano.GitanoDownstream, + 'gitlab': gitlab.GitlabDownstream, + 'local': local.LocalDownstream, +} __all__ = locals() diff --git a/lorrycontroller/gerrit.py b/lorrycontroller/gerrit.py index 6ae24e3..6fde905 100644 --- a/lorrycontroller/gerrit.py +++ b/lorrycontroller/gerrit.py @@ -13,11 +13,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import logging import cliapp +from . import hosts -class Gerrit(object): + +class GerritDownstream(hosts.DownstreamHost): '''Run commands on a Gerrit instance. @@ -28,7 +31,12 @@ class Gerrit(object): ''' - def __init__(self, host, user, port=29418): + def __init__(self, app_settings): + # XXX These need to be configurable + host = 'localhost' + port = 29418 + user = 'lorry' + self._ssh_command_args = [ 'ssh', '-oStrictHostKeyChecking=no', '-oBatchMode=yes', '-p%i' % port, '%s@%s' % (user, host)] @@ -39,7 +47,7 @@ class Gerrit(object): out = out.decode('utf-8', errors='replace') return out - def has_project(self, name): + def _has_project(self, name): # There's no 'does this project exist' command in Gerrit 2.9.4; 'list # all projects with this prefix' is as close we can get. @@ -52,5 +60,18 @@ class Gerrit(object): else: return False - def create_project(self, name): + def prepare_repo(self, name, metadata): + '''Create a project in the local Gerrit server. + + The 'lorry' user must have createProject capability in the Gerrit. + + ''' + # TODO: set metadata + + if self._has_project(name): + logging.info('Project %s exists in local Gerrit already.', + name) + return + self._ssh_command(['gerrit', 'create-project', name]) + logging.info('Created %s project in local Gerrit.', name) diff --git a/lorrycontroller/gitano.py b/lorrycontroller/gitano.py index 7d9c436..06039b0 100644 --- a/lorrycontroller/gitano.py +++ b/lorrycontroller/gitano.py @@ -23,6 +23,7 @@ import cliapp import requests import lorrycontroller +from . import hosts class GitanoCommandFailure(Exception): @@ -149,3 +150,42 @@ def new_gitano_command(statedb, trovehost): trove_info['protocol'], trove_info['username'], trove_info['password']) + + +class GitanoDownstream(hosts.DownstreamHost): + def __init__(self, app_settings): + self._gitano = LocalTroveGitanoCommand() + + def prepare_repo(self, repo_path, metadata): + # Create repository on local Trove. If it fails, assume + # it failed because the repository already existed, and + # ignore the failure (but log message). + + try: + self._gitano.create(repo_path) + except GitanoCommandFailure as e: + logging.debug( + 'Ignoring error creating %s on local Trove: %s', + repo_path, e) + else: + logging.info('Created %s on local repo', repo_path) + + try: + local_config = self._gitano.get_gitano_config(repo_path) + if 'head' in metadata \ + and metadata['head'] != local_config['project.head']: + self._gitano.set_gitano_config(repo_path, + 'project.head', + metadata['head']) + if 'description' in metadata \ + and metadata['description'] != \ + local_config['project.description']: + self._gitano.set_gitano_config(repo_path, + 'project.description', + metadata['description']) + except GitanoCommandFailure as e: + logging.error('ERROR: %s' % str(e)) + # FIXME: We need a good way to report these errors to the + # user. However, we probably don't want to fail the + # request, so that's not the way to do this. Needs + # thinking. diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py index 13becfe..90ba192 100644 --- a/lorrycontroller/gitlab.py +++ b/lorrycontroller/gitlab.py @@ -13,7 +13,15 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +''' +Run commands on a GitLab instance. +This uses the python wrapper around the GitLab API. +Use of the API requires the private token of a user with master access +to the targetted group. +''' + +import logging import re import urllib.parse @@ -22,40 +30,57 @@ try: except ImportError: gitlab = None +from . import hosts + class MissingGitlabModuleError(Exception): pass -class Gitlab(object): +def _init_gitlab(host, token): + if gitlab: + url = "http://" + host + return gitlab.Gitlab(url, token) + else: + raise MissingGitlabModuleError('gitlab module missing\n' + '\tpython-gitlab is required with GitLab as the git server') - '''Run commands on a GitLab instance. - This uses the python wrapper around the GitLab API. - Use of the API requires the private token of a user with master access - to the targetted group. +class GitlabDownstream(hosts.DownstreamHost): + @staticmethod + def add_app_settings(app_settings): + app_settings.string( + ['gitlab-private-token'], + 'private token for GitLab API access') - ''' + @staticmethod + def check_app_settings(app_settings): + if not app_settings['gitlab-private-token']: + logging.error('A private token must be provided to create ' + 'repositories on a GitLab instance.') + app_settings.require('gitlab-private-token') - def __init__(self, host, token): - if gitlab: - url = "http://" + host - self.gl = gitlab.Gitlab(url, token) - else: - raise MissingGitlabModuleError('gitlab module missing\n' - '\tpython-gitlab is required with GitLab as the git server') + def __init__(self, app_settings): + # XXX This needs to be configurable + host = 'localhost' - def find_project(self, repo_path): - return self.gl.projects.get(repo_path) + self.gl = _init_gitlab(host, app_settings['gitlab-private-token']) - def has_project(self, repo_path): + def _has_project(self, repo_path): try: - self.find_project(repo_path) + self.gl.projects.get(repo_path) return True except gitlab.GitlabGetError: return False - def create_project(self, repo_path): + def prepare_repo(self, repo_path, metadata): + # TODO: set metadata + + if self._has_project(repo_path): + logging.info('Project %s exists in local GitLab already.', + repo_path) + return + path_comps = repo_path.split('/') if len(path_comps) < 2: @@ -87,6 +112,13 @@ class Gitlab(object): } self.gl.projects.create(project) + logging.info('Created %s project in local GitLab.', repo_path) + + +class Gitlab(object): + def __init__(self, host, token): + self.gl = _init_gitlab(host, token) + def list_projects(self): '''List projects on a GitLab instance.''' @@ -105,7 +137,7 @@ class Gitlab(object): format matching 'http(s)://host/group/project.git'. ''' - project = self.find_project(project_path) + project = self.gl.projects.get(repo_path) if protocol == 'ssh': return project.ssh_url_to_repo diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index 9d4d4d2..85f1818 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -36,9 +36,16 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): now = statedb.get_current_time() for lorry_info in lorry_infos: if self.ready_to_run(lorry_info, now): - self.create_repository(statedb, lorry_info) if lorry_info['from_host']: - self.copy_repository_metadata(statedb, lorry_info) + metadata = self.get_repo_metadata(statedb, + lorry_info) + else: + metadata = {} + downstream_type = lorrycontroller.downstream_types[ + self.app_settings['git-server-type']] + downstream_type(self.app_settings) \ + .prepare_repo(lorry_info['path'], metadata) + self.give_job_to_minion(statedb, lorry_info, now) logging.info( 'Giving job %s to lorry %s to MINION %s:%s', @@ -62,92 +69,23 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): due = lorry_info['last_run'] + lorry_info['interval'] return (lorry_info['running_job'] is None and due <= now) - def create_repository(self, statedb, lorry_info): - api = self.app_settings['git-server-type'] - if api == 'gitano': - self.create_repository_in_local_trove(statedb, lorry_info) - elif api == 'gerrit': - self.create_gerrit_project(statedb, lorry_info) - elif api == 'gitlab': - self.create_gitlab_project(statedb, lorry_info) - elif api == 'local': - pass - - def create_repository_in_local_trove(self, statedb, lorry_info): - # Create repository on local Trove. If it fails, assume - # it failed because the repository already existed, and - # ignore the failure (but log message). - - local = lorrycontroller.LocalTroveGitanoCommand() - try: - local.create(lorry_info['path']) - except lorrycontroller.GitanoCommandFailure as e: - logging.debug( - 'Ignoring error creating %s on local Trove: %s', - lorry_info['path'], e) - else: - logging.info('Created %s on local repo', lorry_info['path']) - - def create_gerrit_project(self, statedb, lorry_info): - '''Create a project in the local Gerrit server. - - The 'lorry' user must have createProject capability in the Gerrit. - - ''' - gerrit = lorrycontroller.Gerrit( - host='localhost', user='lorry') - project_name = lorry_info['path'] - - if gerrit.has_project(project_name): - logging.info('Project %s exists in local Gerrit already.', - project_name) - else: - gerrit.create_project(project_name) - logging.info('Created %s project in local Gerrit.', project_name) - - def create_gitlab_project(self, statedb, lorry_info): - gitlab = lorrycontroller.Gitlab( - 'localhost', self.app_settings['gitlab-private-token']) - project_name = lorry_info['path'] - - if gitlab.has_project(project_name): - logging.info('Project %s exists in local GitLab already.', - project_name) - else: - gitlab.create_project(lorry_info['path']) - logging.info('Created %s project in local GitLab.', project_name) - - def copy_repository_metadata(self, statedb, lorry_info): - '''Copy project.head and project.description to the local Trove.''' + def get_repo_metadata(self, statedb, lorry_info): + '''Get repository head and description.''' assert lorry_info['from_host'] assert lorry_info['from_path'] - if self.app_settings['git-server-type'] != 'gitano': - # FIXME: would be good to have this info in Gerrit too - return + # XXX We don't know whether upstream is Trove + metadata = {} remote = lorrycontroller.new_gitano_command(statedb, lorry_info['from_host']) - local = lorrycontroller.LocalTroveGitanoCommand() try: remote_config = remote.get_gitano_config(lorry_info['from_path']) - local_config = local.get_gitano_config(lorry_info['path']) - - if remote_config['project.head'] != local_config['project.head']: - local.set_gitano_config( - lorry_info['path'], - 'project.head', - remote_config['project.head']) - - if not local_config['project.description']: - desc = '{host}: {desc}'.format( - host=lorry_info['from_host'], - desc=remote_config['project.description']) - local.set_gitano_config( - lorry_info['path'], - 'project.description', - desc) + metadata['head'] = remote_config['project.head'] + metadata['description'] = '{host}: {desc}'.format( + host=lorry_info['from_host'], + desc=remote_config['project.description']) except lorrycontroller.GitanoCommandFailure as e: logging.error('ERROR: %s' % str(e)) # FIXME: We need a good way to report these errors to the @@ -155,6 +93,8 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): # request, so that's not the way to do this. Needs # thinking. + return metadata + def give_job_to_minion(self, statedb, lorry_info, now): path = lorry_info['path'] minion_host = bottle.request.forms.host diff --git a/lorrycontroller/hosts.py b/lorrycontroller/hosts.py new file mode 100644 index 0000000..76eeadf --- /dev/null +++ b/lorrycontroller/hosts.py @@ -0,0 +1,57 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import abc + + +class DownstreamHost(abc.ABC): + @staticmethod + def add_app_settings(app_settings): + '''Add any application settings that are specific to this Downstream + Host type. + ''' + pass + + @staticmethod + def check_app_settings(app_settings): + '''Validate any fields in the application settings that are specific + to this Downstream Host type. + ''' + pass + + @abc.abstractmethod + def __init__(self, app_settings): + '''Construct a Downstream Host connector from the application + settings. + ''' + pass + + @abc.abstractmethod + def prepare_repo(self, repo_path, metadata): + '''Prepare a repository on the Host. If the repository does not exist + and Lorry won't automatically create it, this method must create the + repository. It should also set any given metadata on the repository, + whether or not it already exists. + + repo_path is the path that the repository should appear at + within the Host. + + metadata is a dictionary with the following (optional) keys + defined: + + - head: Name of the default branch (a.k.a. HEAD) + - description: Short string describing the repository + ''' + pass diff --git a/lorrycontroller/local.py b/lorrycontroller/local.py new file mode 100644 index 0000000..101bc98 --- /dev/null +++ b/lorrycontroller/local.py @@ -0,0 +1,26 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from . import hosts + + +class LocalDownstream(hosts.DownstreamHost): + def __init__(self, app_settings): + pass + + def prepare_repo(self, repo_path, metadata): + # Lorry can create the repository directly + # TODO: set metadata + pass -- cgit v1.2.1