diff options
| -rw-r--r-- | lorrycontroller/lsupstreams.py | 20 | ||||
| -rw-r--r-- | lorrycontroller/migrations/0003-generalise-troves.py | 41 | ||||
| -rw-r--r-- | lorrycontroller/readconf.py | 9 | ||||
| -rw-r--r-- | lorrycontroller/statedb.py | 61 | ||||
| -rw-r--r-- | yarns.webapp/900-implementations.yarn | 2 |
5 files changed, 91 insertions, 42 deletions
diff --git a/lorrycontroller/lsupstreams.py b/lorrycontroller/lsupstreams.py index a64a496..b31a385 100644 --- a/lorrycontroller/lsupstreams.py +++ b/lorrycontroller/lsupstreams.py @@ -69,10 +69,11 @@ class HostRepositoryLister(object): return None def get_real_ls_output(self, statedb, host_info): - gitlab_token = host_info.get('gitlab_token') - if gitlab_token: - return lorrycontroller.Gitlab( - host_info['host'], gitlab_token).list_projects() + if host_info['type'] == 'gitlab': + return lorrycontroller.Gitlab(host_info['host'], + host_info['type_params'] + ['private-token']) \ + .list_projects() gitano = lorrycontroller.new_gitano_command( statedb, host_info['host']) @@ -155,11 +156,12 @@ class HostRepositoryLister(object): } def construct_lorry_url(self, host_info, remote_path): - gitlab_token = host_info.get('gitlab_token') - if gitlab_token: - return lorrycontroller.Gitlab( - host_info['host'], gitlab_token).get_project_url( - host_info['protocol'], remote_path) + if host_info['type'] == 'gitlab': + return lorrycontroller.Gitlab(host_info['host'], + host_info['type_params'] + ['private-token']) \ + .get_project_url(host_info['protocol'], + remote_path) vars = dict(host_info) vars['remote_path'] = remote_path diff --git a/lorrycontroller/migrations/0003-generalise-troves.py b/lorrycontroller/migrations/0003-generalise-troves.py new file mode 100644 index 0000000..bacd2ad --- /dev/null +++ b/lorrycontroller/migrations/0003-generalise-troves.py @@ -0,0 +1,41 @@ +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import yoyo + + +yoyo.step('CREATE TABLE hosts (' + 'host TEXT PRIMARY KEY, ' + 'protocol TEXT, ' + 'username TEXT, ' + 'password TEXT, ' + 'type TEXT NOT NULL, ' + 'type_params TEXT NOT NULL, ' + 'lorry_interval INT, ' + 'lorry_timeout INT, ' + 'ls_interval INT, ' + 'ls_last_run INT, ' + 'prefixmap TEXT, ' + 'ignore TEXT ' + ')') +yoyo.step('INSERT INTO hosts ' + 'SELECT trovehost, protocol, username, password, ' + "CASE WHEN gitlab_token IS NULL THEN 'trove' ELSE 'gitlab' END, " + "CASE WHEN gitlab_token IS NULL THEN '{}' " + "ELSE json_object('private-token', gitlab_token) END, " + 'lorry_interval, lorry_timeout, ls_interval, ls_last_run, prefixmap, ' + 'ignore ' + 'FROM troves') +yoyo.step('DROP TABLE troves') diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py index 09a9f52..b95b9de 100644 --- a/lorrycontroller/readconf.py +++ b/lorrycontroller/readconf.py @@ -290,22 +290,23 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute): username = auth.get('username') password = auth.get('password') - gitlab_token = None + type_params = {} if section['type'] == 'gitlab': - gitlab_token = section['private-token'] + type_params['private-token'] = section['private-token'] statedb.add_host( host=section.get('host') or section['trovehost'], protocol=section['protocol'], username=username, password=password, + host_type=section['type'], + type_params=type_params, lorry_interval=section['interval'], lorry_timeout=section.get( 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT), ls_interval=section['ls-interval'], prefixmap=json.dumps(section['prefixmap']), - ignore=json.dumps(section.get('ignore', [])), - gitlab_token=gitlab_token) + ignore=json.dumps(section.get('ignore', []))) class ValidationError(Exception): diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 122cb50..2dd30f0 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -13,7 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +import json import logging import os import sqlite3 @@ -240,10 +240,10 @@ class StateDB(object): def get_host_info(self, host): c = self.get_cursor() c.execute( - 'SELECT protocol, username, password, lorry_interval, ' - 'lorry_timeout, ls_interval, ls_last_run, ' - 'prefixmap, ignore, gitlab_token ' - 'FROM troves WHERE trovehost IS ?', + 'SELECT protocol, username, password, type, type_params, ' + 'lorry_interval, lorry_timeout, ls_interval, ls_last_run, ' + 'prefixmap, ignore ' + 'FROM hosts WHERE host IS ?', (host,)) row = c.fetchone() if row is None: @@ -253,19 +253,20 @@ class StateDB(object): 'protocol': row[0], 'username': row[1], 'password': row[2], - 'lorry_interval': row[3], - 'lorry_timeout': row[4], - 'ls_interval': row[5], - 'ls_last_run': row[6], - 'prefixmap': row[7], - 'ignore': row[8], - 'gitlab_token': row[9] + 'type': row[3], + 'type_params': json.loads(row[4]), + 'lorry_interval': row[5], + 'lorry_timeout': row[6], + 'ls_interval': row[7], + 'ls_last_run': row[8], + 'prefixmap': row[9], + 'ignore': row[10], } def add_host(self, host=None, protocol=None, username=None, - password=None, lorry_interval=None, - lorry_timeout=None, ls_interval=None, - prefixmap=None, ignore=None, gitlab_token=None): + password=None, host_type=None, type_params={}, + lorry_interval=None, lorry_timeout=None, ls_interval=None, + prefixmap=None, ignore=None): logging.debug( 'StateDB.add_host(%r,%r,%r,%r,%r,%r) called', host, lorry_interval, lorry_timeout, ls_interval, @@ -273,6 +274,8 @@ class StateDB(object): assert host is not None assert protocol is not None + assert host_type is not None + assert isinstance(type_params, dict) assert lorry_interval is not None assert lorry_timeout is not None assert ls_interval is not None @@ -280,39 +283,41 @@ class StateDB(object): assert ignore is not None assert self.in_transaction + type_params = json.dumps(type_params) + try: self.get_host_info(host) except lorrycontroller.HostNotFoundError: c = self.get_cursor() c.execute( - 'INSERT INTO troves ' - '(trovehost, protocol, username, password, ' + 'INSERT INTO hosts ' + '(host, protocol, username, password, type, type_params, ' 'lorry_interval, lorry_timeout, ' 'ls_interval, ls_last_run, ' - 'prefixmap, ignore, gitlab_token) ' - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - (host, protocol, username, password, + 'prefixmap, ignore) ' + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + (host, protocol, username, password, host_type, type_params, lorry_interval, lorry_timeout, ls_interval, 0, - prefixmap, ignore, gitlab_token)) + prefixmap, ignore)) else: c = self.get_cursor() c.execute( - 'UPDATE troves ' + 'UPDATE hosts ' 'SET lorry_interval=?, lorry_timeout=?, ls_interval=?, ' - 'prefixmap=?, ignore=?, protocol=?, gitlab_token=? ' - 'WHERE trovehost IS ?', + 'prefixmap=?, ignore=?, protocol=?, type_params=? ' + 'WHERE host IS ?', (lorry_interval, lorry_timeout, ls_interval, prefixmap, - ignore, protocol, gitlab_token, host)) + ignore, protocol, type_params, host)) def remove_host(self, host): logging.debug('StateDB.remove_host(%r) called', host) assert self.in_transaction c = self.get_cursor() - c.execute('DELETE FROM troves WHERE trovehost=?', (host,)) + c.execute('DELETE FROM hosts WHERE host=?', (host,)) def get_hosts(self): c = self.get_cursor() - c.execute('SELECT trovehost FROM troves') + c.execute('SELECT host FROM hosts') return [row[0] for row in c.fetchall()] def set_host_ls_last_run(self, host, ls_last_run): @@ -322,7 +327,7 @@ class StateDB(object): assert self.in_transaction c = self.get_cursor() c.execute( - 'UPDATE troves SET ls_last_run=? WHERE trovehost=?', + 'UPDATE hosts SET ls_last_run=? WHERE host=?', (ls_last_run, host)) def make_lorry_info_from_row(self, row): diff --git a/yarns.webapp/900-implementations.yarn b/yarns.webapp/900-implementations.yarn index 9a759ba..fc88fd4 100644 --- a/yarns.webapp/900-implementations.yarn +++ b/yarns.webapp/900-implementations.yarn @@ -447,5 +447,5 @@ any rows in them. IMPLEMENTS THEN STATEDB is empty test -s "$DATADIR/webapp.db" - sqlite3 "$DATADIR/webapp.db" 'SELECT * FROM troves;' | stdin_is_empty + sqlite3 "$DATADIR/webapp.db" 'SELECT * FROM hosts;' | stdin_is_empty sqlite3 "$DATADIR/webapp.db" 'SELECT * FROM lorries;' | stdin_is_empty |