From 4947e78942bf325ed85645876f0352eb3cd24e5e Mon Sep 17 00:00:00 2001 From: Ben Brown Date: Mon, 18 Apr 2016 16:04:52 +0100 Subject: Add support for lorrying from a Gitlab server Change-Id: I2bb0aaf428e331a0bcd5a1e3111d4c7bca4afede --- README | 16 +++++++++++++--- lorrycontroller/gitlab.py | 26 ++++++++++++++++++++++++++ lorrycontroller/lstroves.py | 28 ++++++++++++++++++++-------- lorrycontroller/readconf.py | 18 +++++++++++++++--- lorrycontroller/statedb.py | 29 +++++++++++++++++++---------- 5 files changed, 93 insertions(+), 24 deletions(-) diff --git a/README b/README index c62d024..5fe1d93 100644 --- a/README +++ b/README @@ -64,9 +64,10 @@ The `lorry-controller.conf` file -------------------------------- `lorry-controller.conf` is a JSON file containing a list of maps. Each -map specifies another Trove, or one set of `.lorry` files. Here's an -example that tells LC to mirror the `git.baserock.org` Trove and -anything in the `open-source-lorries/*.lorry` files (if any exist). +map specifies another Trove, a GitLab instance, or one set of `.lorry` +files. Here's an example that tells LC to mirror the `git.baserock.org` +Trove and anything in the `open-source-lorries/*.lorry` files (if any +exist). [ { @@ -130,6 +131,15 @@ specifications: (only). It should be a dictionary with the fields `username` and `password`. +A GitLab specification (map) makes use of the same keys as a Trove, +however it uses an additional mandatory key: + +* `type: gitlab` -- specify it's a GitLab specification. + +* `private-token` -- the GitLab private token for a user with the + minimum permissions of master of any group you may wish to create + repositories under. + A Lorry specification (map) uses the following keys, all of them mandatory: diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py index 5c7e579..659d179 100644 --- a/lorrycontroller/gitlab.py +++ b/lorrycontroller/gitlab.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. from __future__ import absolute_import +import urlparse import itertools try: import gitlab @@ -82,3 +83,28 @@ class Gitlab(object): } self.gl.projects.create(project) + def list_projects(self): + return [x.path_with_namespace for x in self.gl.projects.list()] + + def get_project_url(self, protocol, project_path): + '''Return the clone url for a GitLab project. + + Depending on the protocol specified, will return a suitable clone url. + If 'ssh', a url in the format 'git@host:group/project.git' will be + returned. + If 'http' or 'https', the http_url_to_repo from the GitLab API is split + with urlparse into its constituent parts: the protocol (http by + default), the host, and the path ('group/project.git'). This is then + rejoined, replacing the protocol with what is specified. The resulting + format matching 'http(s)://host/group/project.git'. + ''' + + group, project = self.split_path(project_path) + project = self.find_project(group, project) + + if protocol == 'ssh': + return project.ssh_url_to_repo + elif protocol in ('http', 'https'): + split = urlparse.urlsplit(project.http_url_to_repo) + return urlparse.urlunsplit(( + protocol, split.netloc, split.path, '', '')) diff --git a/lorrycontroller/lstroves.py b/lorrycontroller/lstroves.py index 72515f5..456359c 100644 --- a/lorrycontroller/lstroves.py +++ b/lorrycontroller/lstroves.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2015 Codethink Limited +# Copyright (C) 2014-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,14 +23,14 @@ import bottle import lorrycontroller -class GitanoLsError(Exception): +class ServerLsError(Exception): - def __init__(self, trovehost, output): + def __init__(self, remote_host, output): Exception.__init__( self, 'Failed to get list of git repositories ' - 'on remote host %s:\n%s' % (trovehost, output)) - self.trovehost = trovehost + 'on remote host %s:\n%s' % (remote_host, output)) + self.remote_host = remote_host class TroveRepositoryLister(object): @@ -69,7 +69,13 @@ class TroveRepositoryLister(object): return None def get_real_ls_output(self, statedb, trove_info): - gitano = lorrycontroller.new_gitano_command(statedb, trove_info['trovehost']) + gitlab_token = trove_info.get('gitlab_token') + if gitlab_token: + return lorrycontroller.Gitlab( + trove_info['trovehost'], gitlab_token).list_projects() + + gitano = lorrycontroller.new_gitano_command( + statedb, trove_info['trovehost']) output = gitano.ls() return self.parse_ls_output(output) @@ -149,6 +155,12 @@ class TroveRepositoryLister(object): } def construct_lorry_url(self, trove_info, remote_path): + gitlab_token = trove_info.get('gitlab_token') + if gitlab_token: + return lorrycontroller.Gitlab( + trove_info['trovehost'], gitlab_token).get_project_url( + trove_info['protocol'], remote_path) + vars = dict(trove_info) vars['remote_path'] = remote_path @@ -177,7 +189,7 @@ class ForceLsTrove(lorrycontroller.LorryControllerRoute): trove_info = statedb.get_trove_info(trovehost) try: updated = lister.list_trove_into_statedb(statedb, trove_info) - except GitanoLsError as e: + except ServerLsError as e: raise bottle.abort(500, str(e)) return { 'updated-troves': updated } @@ -199,7 +211,7 @@ class LsTroves(lorrycontroller.LorryControllerRoute): logging.info('Trove %r is due an ls', trove_info['trovehost']) try: lister.list_trove_into_statedb(statedb, trove_info) - except GitanoLsError as e: + except ServerLsError as e: bottle.abort(500, str(e)) return { diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py index a108c41..5323a3f 100644 --- a/lorrycontroller/readconf.py +++ b/lorrycontroller/readconf.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -69,7 +69,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): added = self.add_matching_lorries_to_statedb( statedb, section) lorries_to_remove = lorries_to_remove.difference(added) - elif section['type'] in ('trove', 'troves'): + elif section['type'] in ('trove', 'troves', 'gitlab'): self.add_trove(statedb, section) trovehost = section['trovehost'] if trovehost in troves_to_remove: @@ -286,6 +286,10 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): username = auth.get('username') password = auth.get('password') + gitlab_token = None + if section['type'] == 'gitlab': + gitlab_token = section['private-token'] + statedb.add_trove( trovehost=section['trovehost'], protocol=section['protocol'], @@ -296,7 +300,8 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT), ls_interval=section['ls-interval'], prefixmap=json.dumps(section['prefixmap']), - ignore=json.dumps(section.get('ignore', []))) + ignore=json.dumps(section.get('ignore', [])), + gitlab_token=gitlab_token) class ValidationError(Exception): @@ -320,6 +325,8 @@ class LorryControllerConfValidator(object): self._check_troves_section(section) elif section['type'] == 'lorries': self._check_lorries_section(section) + elif section['type'] == 'gitlab': + self._check_gitlab_section(section) else: raise ValidationError( 'unknown section type %r' % section['type']) @@ -338,6 +345,11 @@ class LorryControllerConfValidator(object): if type(item) is not dict: raise ValidationError('all items must be dicts') + def _check_gitlab_section(self, section): + # gitlab section inherits trove configurations, perform the same checks. + self._check_troves_section(section) + self._check_has_required_fields(section, ['private-token']) + def _check_troves_section(self, section): self._check_has_required_fields( section, diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 7f537f3..27e6fae 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -80,6 +80,13 @@ class StateDB(object): logging.debug('New connection is %r', self._conn) if not existed: self._initialise_tables() + else: + c = self._conn.cursor() + c.execute('PRAGMA table_info(troves)') + columns = (info[1] for info in c.fetchall()) + if 'gitlab_token' not in columns: + c.execute('ALTER TABLE troves ADD COLUMN gitlab_token') + self._conn.commit() def _initialise_tables(self): logging.debug('Initialising tables in database') @@ -106,7 +113,8 @@ class StateDB(object): 'ls_interval INT, ' 'ls_last_run INT, ' 'prefixmap TEXT, ' - 'ignore TEXT ' + 'ignore TEXT, ' + 'gitlab_token TEXT ' ')') # Table for all the known lorries (the "run queue"). @@ -215,7 +223,7 @@ class StateDB(object): c.execute( 'SELECT protocol, username, password, lorry_interval, ' 'lorry_timeout, ls_interval, ls_last_run, ' - 'prefixmap, ignore ' + 'prefixmap, ignore, gitlab_token ' 'FROM troves WHERE trovehost IS ?', (trovehost,)) row = c.fetchone() @@ -232,12 +240,13 @@ class StateDB(object): 'ls_last_run': row[6], 'prefixmap': row[7], 'ignore': row[8], - } + 'gitlab_token': row[9] + } def add_trove(self, trovehost=None, protocol=None, username=None, password=None, lorry_interval=None, lorry_timeout=None, ls_interval=None, - prefixmap=None, ignore=None): + prefixmap=None, ignore=None, gitlab_token=None): logging.debug( 'StateDB.add_trove(%r,%r,%r,%r,%r,%r) called', trovehost, lorry_interval, lorry_timeout, ls_interval, @@ -261,20 +270,20 @@ class StateDB(object): '(trovehost, protocol, username, password, ' 'lorry_interval, lorry_timeout, ' 'ls_interval, ls_last_run, ' - 'prefixmap, ignore) ' - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + 'prefixmap, ignore, gitlab_token) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (trovehost, protocol, username, password, lorry_interval, lorry_timeout, ls_interval, 0, - prefixmap, ignore)) + prefixmap, ignore, gitlab_token)) else: c = self.get_cursor() c.execute( 'UPDATE troves ' 'SET lorry_interval=?, lorry_timeout=?, ls_interval=?, ' - 'prefixmap=?, ignore=?, protocol=? ' + 'prefixmap=?, ignore=?, protocol=?, gitlab_token=? ' 'WHERE trovehost IS ?', (lorry_interval, lorry_timeout, ls_interval, prefixmap, - ignore, protocol, trovehost)) + ignore, protocol, gitlab_token, trovehost)) def remove_trove(self, trovehost): logging.debug('StateDB.remove_trove(%r) called', trovehost) -- cgit v1.2.1