Move Downstream Host type-specific code into modules

* Introduce hosts module and DownstreamHost abstract base class
* Define a subclass of this for each Downstream Host type
* Define a name-type map for these in the lorrycontroller package
* Use these classes to replace type-specific code elsewhere

Related to #5.

7 files changed, 230 insertions, 103 deletions

diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py
index 64c4a6f..c00e142 100644
--- a/lorrycontroller/__init__.py
+++ b/lorrycontroller/__init__.py
@@ -44,8 +44,19 @@ from .gitano import (
     new_gitano_command)
 from .static import StaticFile
 from .proxy import setup_proxy
-from .gerrit import Gerrit
 from .gitlab import Gitlab
+from . import gerrit
+from . import gitano
+from . import gitlab
+from . import local
+
+
+downstream_types = {
+    'gerrit': gerrit.GerritDownstream,
+    'gitano': gitano.GitanoDownstream,
+    'gitlab': gitlab.GitlabDownstream,
+    'local':  local.LocalDownstream,
+}
 
 
 __all__ = locals()
diff --git a/lorrycontroller/gerrit.py b/lorrycontroller/gerrit.py
index 6ae24e3..6fde905 100644
--- a/lorrycontroller/gerrit.py
+++ b/lorrycontroller/gerrit.py
@@ -13,11 +13,14 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import logging
 
 import cliapp
 
+from . import hosts
 
-class Gerrit(object):
+
+class GerritDownstream(hosts.DownstreamHost):
 
     '''Run commands on a Gerrit instance.
 
@@ -28,7 +31,12 @@ class Gerrit(object):
 
     '''
 
-    def __init__(self, host, user, port=29418):
+    def __init__(self, app_settings):
+        # XXX These need to be configurable
+        host = 'localhost'
+        port = 29418
+        user = 'lorry'
+
         self._ssh_command_args = [
             'ssh', '-oStrictHostKeyChecking=no', '-oBatchMode=yes', '-p%i' % port,
             '%s@%s' % (user, host)]
@@ -39,7 +47,7 @@ class Gerrit(object):
             out = out.decode('utf-8', errors='replace')
         return out
 
-    def has_project(self, name):
+    def _has_project(self, name):
         # There's no 'does this project exist' command in Gerrit 2.9.4; 'list
         # all projects with this prefix' is as close we can get.
 
@@ -52,5 +60,18 @@ class Gerrit(object):
         else:
             return False
 
-    def create_project(self, name):
+    def prepare_repo(self, name, metadata):
+        '''Create a project in the local Gerrit server.
+
+        The 'lorry' user must have createProject capability in the Gerrit.
+
+        '''
+        # TODO: set metadata
+
+        if self._has_project(name):
+            logging.info('Project %s exists in local Gerrit already.',
+                         name)
+            return
+
         self._ssh_command(['gerrit', 'create-project', name])
+        logging.info('Created %s project in local Gerrit.', name)
diff --git a/lorrycontroller/gitano.py b/lorrycontroller/gitano.py
index 7d9c436..06039b0 100644
--- a/lorrycontroller/gitano.py
+++ b/lorrycontroller/gitano.py
@@ -23,6 +23,7 @@ import cliapp
 import requests
 
 import lorrycontroller
+from . import hosts
 
 
 class GitanoCommandFailure(Exception):
@@ -149,3 +150,42 @@ def new_gitano_command(statedb, trovehost):
         trove_info['protocol'],
         trove_info['username'],
         trove_info['password'])
+
+
+class GitanoDownstream(hosts.DownstreamHost):
+    def __init__(self, app_settings):
+        self._gitano = LocalTroveGitanoCommand()
+
+    def prepare_repo(self, repo_path, metadata):
+        # Create repository on local Trove. If it fails, assume
+        # it failed because the repository already existed, and
+        # ignore the failure (but log message).
+
+        try:
+            self._gitano.create(repo_path)
+        except GitanoCommandFailure as e:
+            logging.debug(
+                'Ignoring error creating %s on local Trove: %s',
+                repo_path, e)
+        else:
+            logging.info('Created %s on local repo', repo_path)
+
+        try:
+            local_config = self._gitano.get_gitano_config(repo_path)
+            if 'head' in metadata \
+               and metadata['head'] != local_config['project.head']:
+                self._gitano.set_gitano_config(repo_path,
+                                               'project.head',
+                                               metadata['head'])
+            if 'description' in metadata \
+               and metadata['description'] != \
+                   local_config['project.description']:
+                self._gitano.set_gitano_config(repo_path,
+                                               'project.description',
+                                               metadata['description'])
+        except GitanoCommandFailure as e:
+            logging.error('ERROR: %s' % str(e))
+            # FIXME: We need a good way to report these errors to the
+            # user. However, we probably don't want to fail the
+            # request, so that's not the way to do this. Needs
+            # thinking.
diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py
index 13becfe..90ba192 100644
--- a/lorrycontroller/gitlab.py
+++ b/lorrycontroller/gitlab.py
@@ -13,7 +13,15 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+'''
+Run commands on a GitLab instance.
 
+This uses the python wrapper around the GitLab API.
+Use of the API requires the private token of a user with master access
+to the targetted group.
+'''
+
+import logging
 import re
 import urllib.parse
 
@@ -22,40 +30,57 @@ try:
 except ImportError:
     gitlab = None
 
+from . import hosts
+
 
 class MissingGitlabModuleError(Exception):
     pass
 
 
-class Gitlab(object):
+def _init_gitlab(host, token):
+    if gitlab:
+        url = "http://" + host
+        return gitlab.Gitlab(url, token)
+    else:
+        raise MissingGitlabModuleError('gitlab module missing\n'
+            '\tpython-gitlab is required with GitLab as the git server')
 
-    '''Run commands on a GitLab instance.
 
-    This uses the python wrapper around the GitLab API.
-    Use of the API requires the private token of a user with master access
-    to the targetted group.
+class GitlabDownstream(hosts.DownstreamHost):
+    @staticmethod
+    def add_app_settings(app_settings):
+        app_settings.string(
+            ['gitlab-private-token'],
+            'private token for GitLab API access')
 
-    '''
+    @staticmethod
+    def check_app_settings(app_settings):
+        if not app_settings['gitlab-private-token']:
+            logging.error('A private token must be provided to create '
+                          'repositories on a GitLab instance.')
+            app_settings.require('gitlab-private-token')
 
-    def __init__(self, host, token):
-        if gitlab:
-            url = "http://" + host
-            self.gl = gitlab.Gitlab(url, token)
-        else:
-            raise MissingGitlabModuleError('gitlab module missing\n'
-                '\tpython-gitlab is required with GitLab as the git server')
+    def __init__(self, app_settings):
+        # XXX This needs to be configurable
+        host = 'localhost'
 
-    def find_project(self, repo_path):
-        return self.gl.projects.get(repo_path)
+        self.gl = _init_gitlab(host, app_settings['gitlab-private-token'])
 
-    def has_project(self, repo_path):
+    def _has_project(self, repo_path):
         try:
-            self.find_project(repo_path)
+            self.gl.projects.get(repo_path)
             return True
         except gitlab.GitlabGetError:
             return False
 
-    def create_project(self, repo_path):
+    def prepare_repo(self, repo_path, metadata):
+        # TODO: set metadata
+
+        if self._has_project(repo_path):
+            logging.info('Project %s exists in local GitLab already.',
+                         repo_path)
+            return
+
         path_comps = repo_path.split('/')
 
         if len(path_comps) < 2:
@@ -87,6 +112,13 @@ class Gitlab(object):
         }
         self.gl.projects.create(project)
 
+        logging.info('Created %s project in local GitLab.', repo_path)
+
+
+class Gitlab(object):
+    def __init__(self, host, token):
+        self.gl = _init_gitlab(host, token)
+
     def list_projects(self):
         '''List projects on a GitLab instance.'''
 
@@ -105,7 +137,7 @@ class Gitlab(object):
         format matching 'http(s)://host/group/project.git'.
         '''
 
-        project = self.find_project(project_path)
+        project = self.gl.projects.get(repo_path)
 
         if protocol == 'ssh':
             return project.ssh_url_to_repo
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py
index 9d4d4d2..85f1818 100644
--- a/lorrycontroller/givemejob.py
+++ b/lorrycontroller/givemejob.py
@@ -36,9 +36,16 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
                 now = statedb.get_current_time()
                 for lorry_info in lorry_infos:
                     if self.ready_to_run(lorry_info, now):
-                        self.create_repository(statedb, lorry_info)
                         if lorry_info['from_host']:
-                            self.copy_repository_metadata(statedb, lorry_info)
+                            metadata = self.get_repo_metadata(statedb,
+                                                              lorry_info)
+                        else:
+                            metadata = {}
+                        downstream_type = lorrycontroller.downstream_types[
+                            self.app_settings['git-server-type']]
+                        downstream_type(self.app_settings) \
+                            .prepare_repo(lorry_info['path'], metadata)
+
                         self.give_job_to_minion(statedb, lorry_info, now)
                         logging.info(
                             'Giving job %s to lorry %s to MINION %s:%s',
@@ -62,92 +69,23 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
         due = lorry_info['last_run'] + lorry_info['interval']
         return (lorry_info['running_job'] is None and due <= now)
 
-    def create_repository(self, statedb, lorry_info):
-        api = self.app_settings['git-server-type']
-        if api == 'gitano':
-            self.create_repository_in_local_trove(statedb, lorry_info)
-        elif api == 'gerrit':
-            self.create_gerrit_project(statedb, lorry_info)
-        elif api == 'gitlab':
-            self.create_gitlab_project(statedb, lorry_info)
-        elif api == 'local':
-            pass
-
-    def create_repository_in_local_trove(self, statedb, lorry_info):
-        # Create repository on local Trove. If it fails, assume
-        # it failed because the repository already existed, and
-        # ignore the failure (but log message).
-
-        local = lorrycontroller.LocalTroveGitanoCommand()
-        try:
-            local.create(lorry_info['path'])
-        except lorrycontroller.GitanoCommandFailure as e:
-            logging.debug(
-                'Ignoring error creating %s on local Trove: %s',
-                lorry_info['path'], e)
-        else:
-            logging.info('Created %s on local repo', lorry_info['path'])
-
-    def create_gerrit_project(self, statedb, lorry_info):
-        '''Create a project in the local Gerrit server.
-
-        The 'lorry' user must have createProject capability in the Gerrit.
-
-        '''
-        gerrit = lorrycontroller.Gerrit(
-            host='localhost', user='lorry')
-        project_name = lorry_info['path']
-
-        if gerrit.has_project(project_name):
-            logging.info('Project %s exists in local Gerrit already.',
-                         project_name)
-        else:
-            gerrit.create_project(project_name)
-            logging.info('Created %s project in local Gerrit.', project_name)
-
-    def create_gitlab_project(self, statedb, lorry_info):
-        gitlab = lorrycontroller.Gitlab(
-            'localhost', self.app_settings['gitlab-private-token'])
-        project_name = lorry_info['path']
-
-        if gitlab.has_project(project_name):
-            logging.info('Project %s exists in local GitLab already.',
-                         project_name)
-        else:
-            gitlab.create_project(lorry_info['path'])
-            logging.info('Created %s project in local GitLab.', project_name)
-
-    def copy_repository_metadata(self, statedb, lorry_info):
-        '''Copy project.head and project.description to the local Trove.'''
+    def get_repo_metadata(self, statedb, lorry_info):
+        '''Get repository head and description.'''
 
         assert lorry_info['from_host']
         assert lorry_info['from_path']
 
-        if self.app_settings['git-server-type'] != 'gitano':
-            # FIXME: would be good to have this info in Gerrit too
-            return
+        # XXX We don't know whether upstream is Trove
 
+        metadata = {}
         remote = lorrycontroller.new_gitano_command(statedb, lorry_info['from_host'])
-        local = lorrycontroller.LocalTroveGitanoCommand()
 
         try:
             remote_config = remote.get_gitano_config(lorry_info['from_path'])
-            local_config = local.get_gitano_config(lorry_info['path'])
-
-            if remote_config['project.head'] != local_config['project.head']:
-                local.set_gitano_config(
-                    lorry_info['path'],
-                    'project.head',
-                    remote_config['project.head'])
-
-            if not local_config['project.description']:
-                desc = '{host}: {desc}'.format(
-                    host=lorry_info['from_host'],
-                    desc=remote_config['project.description'])
-                local.set_gitano_config(
-                    lorry_info['path'],
-                    'project.description',
-                    desc)
+            metadata['head'] = remote_config['project.head']
+            metadata['description'] = '{host}: {desc}'.format(
+                host=lorry_info['from_host'],
+                desc=remote_config['project.description'])
         except lorrycontroller.GitanoCommandFailure as e:
             logging.error('ERROR: %s' % str(e))
             # FIXME: We need a good way to report these errors to the
@@ -155,6 +93,8 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
             # request, so that's not the way to do this. Needs
             # thinking.
 
+        return metadata
+
     def give_job_to_minion(self, statedb, lorry_info, now):
         path = lorry_info['path']
         minion_host = bottle.request.forms.host
diff --git a/lorrycontroller/hosts.py b/lorrycontroller/hosts.py
new file mode 100644
index 0000000..76eeadf
--- /dev/null
+++ b/lorrycontroller/hosts.py
@@ -0,0 +1,57 @@
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import abc
+
+
+class DownstreamHost(abc.ABC):
+    @staticmethod
+    def add_app_settings(app_settings):
+        '''Add any application settings that are specific to this Downstream
+        Host type.
+        '''
+        pass
+
+    @staticmethod
+    def check_app_settings(app_settings):
+        '''Validate any fields in the application settings that are specific
+        to this Downstream Host type.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def __init__(self, app_settings):
+        '''Construct a Downstream Host connector from the application
+        settings.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def prepare_repo(self, repo_path, metadata):
+        '''Prepare a repository on the Host.  If the repository does not exist
+        and Lorry won't automatically create it, this method must create the
+        repository.  It should also set any given metadata on the repository,
+        whether or not it already exists.
+
+        repo_path is the path that the repository should appear at
+        within the Host.
+
+        metadata is a dictionary with the following (optional) keys
+        defined:
+
+        - head: Name of the default branch (a.k.a. HEAD)
+        - description: Short string describing the repository
+        '''
+        pass
diff --git a/lorrycontroller/local.py b/lorrycontroller/local.py
new file mode 100644
index 0000000..101bc98
--- /dev/null
+++ b/lorrycontroller/local.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from . import hosts
+
+
+class LocalDownstream(hosts.DownstreamHost):
+    def __init__(self, app_settings):
+        pass
+
+    def prepare_repo(self, repo_path, metadata):
+        # Lorry can create the repository directly
+        # TODO: set metadata
+        pass