diff options
author | Ben Brown <ben.brown@codethink.co.uk> | 2020-08-03 08:17:48 +0000 |
---|---|---|
committer | Ben Brown <ben.brown@codethink.co.uk> | 2020-08-03 08:17:48 +0000 |
commit | ab010fb032b7f75212140ecb7b31e1a96f50a621 (patch) | |
tree | 4381027ca9508d79caa788e41da0ec118cdf91a5 | |
parent | 6e1b4a4363149230dfac57157d894b1fd5528db7 (diff) | |
parent | 68f21f0475e83f839bd8e25de482d1aa60582edd (diff) | |
download | lorry-controller-ab010fb032b7f75212140ecb7b31e1a96f50a621.tar.gz |
Merge branch 'bwh/single-lorry-metadata' into 'master'
Add metadata for single repositories
Closes #15
See merge request CodethinkLabs/lorry/lorry-controller!17
-rw-r--r-- | README.md | 4 | ||||
-rw-r--r-- | lorrycontroller/gitlab.py | 15 | ||||
-rw-r--r-- | lorrycontroller/givemejob.py | 130 |
3 files changed, 124 insertions, 25 deletions
@@ -167,7 +167,9 @@ hours (`h`), and days (`d`), expressed as single-letter codes in upper or lower case. The syntax of `.lorry` files is specified by the Lorry program; see -its documentation for details. +its documentation for details. Lorry Controller supports an +optional `description` field in `.lorry` files that is used to set +the repository description on the Downstream Host. HTTP proxy configuration: `proxy.conf` diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py index 0b5e1c2..266861c 100644 --- a/lorrycontroller/gitlab.py +++ b/lorrycontroller/gitlab.py @@ -76,13 +76,22 @@ class GitlabDownstream(hosts.DownstreamHost): else: logging.info('Project %s exists in local GitLab already.', repo_path) - if 'head' in metadata \ - and project.default_branch != metadata['head']: - project.default_branch = metadata['head'] + if 'description' in metadata \ and project.description != metadata['description']: project.description = metadata['description'] project.save() + + # This will fail if we haven't created the branch yet. + # We'll fix it next time round. + try: + if 'head' in metadata \ + and project.default_branch != metadata['head']: + project.default_branch = metadata['head'] + project.save() + except gitlab.GitlabUpdateError: + pass + return path_comps = repo_path.split('/') diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index 1a0fe35..ee998ea 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -13,10 +13,13 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - +import json import logging +import re +import urllib.parse import bottle +import cliapp import lorrycontroller @@ -65,35 +68,120 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): due = lorry_info['last_run'] + lorry_info['interval'] return (lorry_info['running_job'] is None and due <= now) - def get_repo_metadata(self, statedb, lorry_info): - '''Get repository head and description.''' - - if not lorry_info['from_host']: - return {} - + @staticmethod + def get_upstream_host_repo_metadata(lorry_info): + assert lorry_info['from_host'] assert lorry_info['from_path'] try: host_info = statedb.get_host_info(lorry_info['from_host']) except lorrycontroller.HostNotFoundError: - # XXX We don't know whether upstream is Trove. It should be - # possible to set host type for single repositories. - host_info = { - 'host': lorry_info['from_host'], - 'protocol': 'ssh', - 'username': None, - 'password': None, - 'type': 'trove', - 'type_params': {}, - } - - metadata = lorrycontroller.get_upstream_host(host_info) \ + # XXX Shouldn't happen, but currently the database schema + # does not prevent it + return {} + else: + return lorrycontroller.get_upstream_host(host_info) \ .get_repo_metadata(lorry_info['from_path']) - if 'description' in metadata: + + @staticmethod + def get_single_repo_metadata(lorry_info): + assert not lorry_info['from_host'] + + lorry_dict = json.loads(lorry_info['text']) + _, upstream_config = lorry_dict.popitem() + upstream_type = upstream_config['type'] + + # Get the repository URL + url = None + try: + url = upstream_config['url'].strip() + except KeyError: + if upstream_type == 'bzr': + try: + url = upstream_config['branches']['trunk'].strip() + except KeyError: + pass + + # Extract the host-name and repo path + host_name, repo_path = None, None + if url: + # Handle pseudo-URLs + if upstream_type == 'bzr': + if url.startswith('lp:'): + host_name = 'launchpad.net' + repo_path = url[3:] + elif upstream_type == 'cvs': + # :pserver:user@host:/path, user@host:/path, etc. + match = re.match(r'^(?::[^:@/]+:)?(?:[^:@/]+@)?([^:@/]+):/', + url) + if match: + host_name = match.group(1) + repo_path = url[match.end():].rstrip('/') + elif upstream_type == 'git': + # user@host:path, host:path. Path must not start with + # '//' as that indicates a real URL. + match = re.match(r'^(?:[^:@/]+@)?([^:@/]+):(?!//)', url) + if match: + host_name = match.group(1) + repo_path = url[match.end():].strip('/') + + # Default to parsing as a real URL + if not host_name: + try: + url_obj = urllib.parse.urlparse(url) + except ValueError: + pass + else: + host_name = url_obj.hostname + repo_path = url_obj.path.strip('/') + + metadata = {} + + # Determine the default branch + if upstream_type == 'bzr': + # Default in Bazaar is 'trunk' and we don't remap it + metadata['head'] = 'trunk' + elif upstream_type == 'git': + if url: + # Query the remote to find its default + try: + output = cliapp.runcmd(['git', 'ls-remote', '--symref', + '--', url, 'HEAD']) \ + .decode('utf-8', errors='replace') + match = re.match(r'^ref: refs/heads/([^\s]+)\tHEAD\n', + output) + if match: + metadata['head'] = match.group(1) + except cliapp.AppException: + pass + else: + # We currently produce 'master' for all other types + metadata['head'] = 'master' + + # Use description from .lorry file, or repository name + try: + metadata['description'] = upstream_config['description'] + except KeyError: + if repo_path: + metadata['description'] = repo_path + + return host_name, metadata + + def get_repo_metadata(self, statedb, lorry_info): + '''Get repository head and description.''' + + host_name = lorry_info['from_host'] + if host_name: + metadata = self.get_upstream_host_repo_metadata(lorry_info) + else: + host_name, metadata = self.get_single_repo_metadata(lorry_info) + + if host_name and 'description' in metadata: # Prepend Upstream Host name metadata['description'] = '{host}: {desc}'.format( - host=lorry_info['from_host'], + host=host_name, desc=metadata['description']) + return metadata def give_job_to_minion(self, statedb, lorry_info, now): |