diff options
-rw-r--r-- | .gitlab-ci.yml | 9 | ||||
-rwxr-xr-x | lorry-controller-minion | 16 | ||||
-rwxr-xr-x | lorry-controller-remove-old-jobs | 22 | ||||
-rwxr-xr-x | lorry-controller-webapp | 7 | ||||
-rw-r--r-- | lorrycontroller/__init__.py | 50 | ||||
-rw-r--r-- | lorrycontroller/gitano.py | 10 | ||||
-rw-r--r-- | lorrycontroller/gitlab.py | 12 | ||||
-rw-r--r-- | lorrycontroller/lstroves.py | 4 | ||||
-rw-r--r-- | lorrycontroller/proxy.py | 6 | ||||
-rw-r--r-- | lorrycontroller/readconf.py | 4 | ||||
-rw-r--r-- | lorrycontroller/showlorry.py | 10 | ||||
-rw-r--r-- | lorrycontroller/status.py | 6 | ||||
-rw-r--r-- | setup.py | 4 | ||||
-rw-r--r-- | templates/status.tpl | 2 | ||||
-rwxr-xr-x | test-wait-for-port | 6 | ||||
-rw-r--r-- | yarns.webapp/900-implementations.yarn | 40 | ||||
-rw-r--r-- | yarns.webapp/yarn.sh | 3 | ||||
-rw-r--r-- | yarns.webapp/yarnlib.py | 4 |
18 files changed, 109 insertions, 106 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28c664c..f2e6051 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,12 +2,15 @@ image: debian:stretch before_script: - apt-get update -y - - apt-get install -y -qq python-dev python-pip + - apt-get install -y -qq python3-dev python3-pip # Deps for running tests - apt-get install -y -qq cmdtest curl git # Deps for lorry-controller - - apt-get install -y -qq python-bottle python-flup python-requests - - pip install yoyo-migrations + - apt-get install -y -qq python3-bottle python3-requests + - pip3 install flup + - pip3 install yoyo-migrations + - pip3 install pyyaml + - pip3 install https://gitlab.com/trovekube/cliapp/-/archive/cliapp-1.20180812.1/cliapp-cliapp-1.20180812.1.tar.gz run-check: script: - sh check diff --git a/lorry-controller-minion b/lorry-controller-minion index 459130e..6e2affd 100755 --- a/lorry-controller-minion +++ b/lorry-controller-minion @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2014-2019 Codethink Limited # @@ -17,7 +17,7 @@ import fcntl -import httplib +import http.client import json import logging import os @@ -28,7 +28,7 @@ import signal import subprocess import tempfile import time -import urllib +import urllib.parse import cliapp @@ -115,7 +115,7 @@ class MINION(cliapp.Application): logging.debug('Requesting job from WEBAPP (%s:%s)', host, port) - params = urllib.urlencode({ + params = urllib.parse.urlencode({ 'host': platform.node(), 'pid': os.getpid(), }) @@ -159,7 +159,7 @@ class MINION(cliapp.Application): job_spec['path']) fd, self.temp_lorry_filename = tempfile.mkstemp() - os.write(fd, job_spec['text']) + os.write(fd, job_spec['text'].encode('utf-8')) os.close(fd) @@ -237,7 +237,7 @@ class MINION(cliapp.Application): else: disk_usage = self.get_lorry_disk_usage(job_spec) - params = urllib.urlencode({ + params = urllib.parse.urlencode({ 'job_id': job_spec['job_id'], 'exit': 'no' if exit is None else exit, 'stdout': stdout, @@ -262,7 +262,7 @@ class MINION(cliapp.Application): host = self.settings['webapp-host'] port = int(self.settings['webapp-port']) timeout = self.settings['webapp-timeout'] - conn = httplib.HTTPConnection(host, port=port, timeout=timeout) + conn = http.client.HTTPConnection(host, port=port, timeout=timeout) headers = {} if body: @@ -274,7 +274,7 @@ class MINION(cliapp.Application): response_body = response.read() conn.close() - if response.status != httplib.OK: + if response.status != http.client.OK: raise WEBAPPError(response.status, response.reason, response_body) return response_body diff --git a/lorry-controller-remove-old-jobs b/lorry-controller-remove-old-jobs index 2a54a30..fcd8b38 100755 --- a/lorry-controller-remove-old-jobs +++ b/lorry-controller-remove-old-jobs @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,7 @@ import json import logging import time -import urllib2 -import urlparse +import urllib.request, urllib.error, urllib.parse import contextlib import cliapp @@ -78,12 +77,12 @@ class OldJobRemover(cliapp.Application): def list_jobs(self): data = self.get('/1.0/list-jobs') - obj = json.loads(data) + obj = json.loads(data.decode('utf-8')) return obj['job_ids'] def get(self, path): url = self.make_url(path) - with contextlib.closing(urllib2.urlopen(url)) as f: + with urllib.request.urlopen(url) as f: return f.read() def make_url(self, path): @@ -93,14 +92,14 @@ class OldJobRemover(cliapp.Application): query = None fragment = None parts = (scheme, netloc, path, query, fragment) - return urlparse.urlunsplit(parts) + return urllib.parse.urlunsplit(parts) def get_job_infos(self, job_ids): job_infos = [] for job_id in job_ids: try: job_infos.append(self.get_job_info(job_id)) - except urllib2.HTTPError as e: + except urllib.error.HTTPError as e: logging.warning( 'Trouble getting job info for job %s: %s' % (job_id, str(e))) @@ -108,7 +107,7 @@ class OldJobRemover(cliapp.Application): def get_job_info(self, job_id): data = self.get('/1.0/job/%s' % job_id) - obj = json.loads(data) + obj = json.loads(data.decode('utf-8')) exit_code = obj['exit'] if obj['job_ended']: exit_timestamp = self.parse_timestamp(obj['job_ended']) @@ -144,9 +143,8 @@ class OldJobRemover(cliapp.Application): def post(self, path, data): url = self.make_url(path) - f = urllib2.urlopen(url, data) - result = f.read() - f.close() + with urllib.request.urlopen(url, data.encode('utf-8')) as f: + result = f.read() OldJobRemover().run() diff --git a/lorry-controller-webapp b/lorry-controller-webapp index 18b28a9..c73333a 100755 --- a/lorry-controller-webapp +++ b/lorry-controller-webapp @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (C) 2014-2017 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -235,7 +235,8 @@ class WEBAPP(cliapp.Application): ''' - def __init__(self, (host, port), *args, **kwargs): + def __init__(self, server_address, *args, **kwargs): + (host, port) = server_address wsgiref.simple_server.WSGIServer.__init__( self, (host, 0), *args, **kwargs) with open(server_port_file, 'w') as f: diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py index 72696fa..c5bf0ad 100644 --- a/lorrycontroller/__init__.py +++ b/lorrycontroller/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,38 +14,38 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from statedb import ( +from .statedb import ( StateDB, LorryNotFoundError, WrongNumberLorriesRunningJob, TroveNotFoundError) -from route import LorryControllerRoute -from readconf import ReadConfiguration -from status import Status, StatusHTML, StatusRenderer -from listqueue import ListQueue -from showlorry import ShowLorry, ShowLorryHTML -from startstopqueue import StartQueue, StopQueue -from givemejob import GiveMeJob -from jobupdate import JobUpdate -from listrunningjobs import ListRunningJobs -from movetopbottom import MoveToTop, MoveToBottom -from stopjob import StopJob -from listjobs import ListAllJobs, ListAllJobsHTML -from showjob import ShowJob, ShowJobHTML, JobShower -from removeghostjobs import RemoveGhostJobs -from removejob import RemoveJob -from lstroves import LsTroves, ForceLsTrove -from pretendtime import PretendTime -from maxjobs import GetMaxJobs, SetMaxJobs -from gitano import ( +from .route import LorryControllerRoute +from .readconf import ReadConfiguration +from .status import Status, StatusHTML, StatusRenderer +from .listqueue import ListQueue +from .showlorry import ShowLorry, ShowLorryHTML +from .startstopqueue import StartQueue, StopQueue +from .givemejob import GiveMeJob +from .jobupdate import JobUpdate +from .listrunningjobs import ListRunningJobs +from .movetopbottom import MoveToTop, MoveToBottom +from .stopjob import StopJob +from .listjobs import ListAllJobs, ListAllJobsHTML +from .showjob import ShowJob, ShowJobHTML, JobShower +from .removeghostjobs import RemoveGhostJobs +from .removejob import RemoveJob +from .lstroves import LsTroves, ForceLsTrove +from .pretendtime import PretendTime +from .maxjobs import GetMaxJobs, SetMaxJobs +from .gitano import ( GitanoCommand, LocalTroveGitanoCommand, GitanoCommandFailure, new_gitano_command) -from static import StaticFile -from proxy import setup_proxy -from gerrit import Gerrit -from gitlab import Gitlab +from .static import StaticFile +from .proxy import setup_proxy +from .gerrit import Gerrit +from .gitlab import Gitlab __all__ = locals() diff --git a/lorrycontroller/gitano.py b/lorrycontroller/gitano.py index c0cca05..d0d1a0c 100644 --- a/lorrycontroller/gitano.py +++ b/lorrycontroller/gitano.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,8 +17,8 @@ import collections import logging import re -import urllib2 -import urlparse +import urllib.request, urllib.error, urllib.parse +import urllib.parse import cliapp import requests @@ -104,8 +104,8 @@ class GitanoCommand(object): return stdout def _http_command(self, gitano_args): - quoted_args = urllib2.quote(' '.join(gitano_args)) - url = urlparse.urlunsplit(( + quoted_args = urllib.parse.quote(' '.join(gitano_args)) + url = urllib.parse.urlunsplit(( self.protocol, self.trovehost, '/gitano-command.cgi', diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py index a426b0d..6938cae 100644 --- a/lorrycontroller/gitlab.py +++ b/lorrycontroller/gitlab.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016 Codethink Limited +# Copyright (C) 2016-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,9 +13,9 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from __future__ import absolute_import + import re -import urlparse +import urllib.parse import itertools try: import gitlab @@ -46,7 +46,7 @@ class Gitlab(object): '\tpython-gitlab is required with GitLab as the git server') def first(self, predicate, iterable): - return next(itertools.ifilter(predicate, iterable)) + return next(filter(predicate, iterable)) def split_and_unslashify_path(self, path): group, project = path.split('/', 1) @@ -137,6 +137,6 @@ class Gitlab(object): if protocol == 'ssh': return project.ssh_url_to_repo elif protocol in ('http', 'https'): - split = urlparse.urlsplit(project.http_url_to_repo) - return urlparse.urlunsplit(( + split = urllib.parse.urlsplit(project.http_url_to_repo) + return urllib.parse.urlunsplit(( protocol, split.netloc, split.path, '', '')) diff --git a/lorrycontroller/lstroves.py b/lorrycontroller/lstroves.py index 456359c..34648cb 100644 --- a/lorrycontroller/lstroves.py +++ b/lorrycontroller/lstroves.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -126,7 +126,7 @@ class TroveRepositoryLister(object): def update_lorries_for_trove(self, statedb, trove_info, repo_map): trovehost = trove_info['trovehost'] - for remote_path, local_path in repo_map.items(): + for remote_path, local_path in list(repo_map.items()): lorry = self.construct_lorry(trove_info, local_path, remote_path) statedb.add_to_lorries( path=local_path, diff --git a/lorrycontroller/proxy.py b/lorrycontroller/proxy.py index b9e75b8..5eea4f1 100644 --- a/lorrycontroller/proxy.py +++ b/lorrycontroller/proxy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,13 +16,13 @@ import json import os -import urllib +import urllib.request, urllib.parse, urllib.error def build_proxy_url(protocol, proxy_config): """Build a proxy URL from data in our proxy configuration format.""" - hostname = urllib.quote(proxy_config['hostname']) + hostname = urllib.parse.quote(proxy_config['hostname']) url = '%s:%s' % (hostname, proxy_config['port']) if 'username' not in proxy_config: diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py index 162e116..a8949c1 100644 --- a/lorrycontroller/readconf.py +++ b/lorrycontroller/readconf.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -394,7 +394,7 @@ class LorryControllerConfValidator(object): def _check_is_list_of_strings(self, section, field): obj = section[field] if not isinstance(obj, list) or not all( - isinstance(s, basestring) for s in obj): + isinstance(s, str) for s in obj): raise ValidationError( '%s field in %r must be a list of strings' % (field, section)) diff --git a/lorrycontroller/showlorry.py b/lorrycontroller/showlorry.py index d54073a..b553b9c 100644 --- a/lorrycontroller/showlorry.py +++ b/lorrycontroller/showlorry.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,7 @@ import json import logging import time -import urlparse +import urllib.parse import bottle @@ -65,7 +65,7 @@ class ShowLorryHTML(ShowLorryBase, lorrycontroller.LorryControllerRoute): renderer = lorrycontroller.StatusRenderer() shower = lorrycontroller.JobShower() - lorry_obj = json.loads(lorry_info['text']).values()[0] + lorry_obj = list(json.loads(lorry_info['text']).values())[0] lorry_info['url'] = lorry_obj['url'] lorry_info['interval_nice'] = renderer.format_secs_nicely( @@ -85,9 +85,9 @@ class ShowLorryHTML(ShowLorryBase, lorrycontroller.LorryControllerRoute): timestamp = time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(now)) - parts = urlparse.urlparse(bottle.request.url) + parts = urllib.parse.urlparse(bottle.request.url) host, port = parts.netloc.split(':', 1) - http_server_root = urlparse.urlunparse( + http_server_root = urllib.parse.urlunparse( (parts.scheme, host, '', '', '', '')) return bottle.template( diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py index 9d65c4e..2e6334d 100644 --- a/lorrycontroller/status.py +++ b/lorrycontroller/status.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2017 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -61,7 +61,7 @@ class StatusRenderer(object): try: temp_filename = self.temp_filename_in_same_dir_as(filename) - with open(temp_filename, 'w') as f: + with open(temp_filename, 'wb') as f: f.write(html.encode("UTF-8")) os.rename(temp_filename, filename) @@ -85,7 +85,7 @@ class StatusRenderer(object): def temp_filename_in_same_dir_as(self, filename): dirname = os.path.dirname(filename) fd, temp_filename = tempfile.mkstemp(dir=dirname) - os.fchmod(fd, 0644) + os.fchmod(fd, 0o644) os.close(fd) return temp_filename @@ -1,6 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # -# Copyright (C) 2012-2016 Codethink Limited +# Copyright (C) 2012-2019 Codethink Limited from distutils.core import setup diff --git a/templates/status.tpl b/templates/status.tpl index 939f77d..46fe034 100644 --- a/templates/status.tpl +++ b/templates/status.tpl @@ -110,7 +110,7 @@ </tr> % for i, spec in enumerate(run_queue): % obj = json.loads(spec['text']) -% name = obj.keys()[0] +% name = list(obj)[0] % fields = obj[name] <tr> <td>{{i+1}}</td> diff --git a/test-wait-for-port b/test-wait-for-port index 22e07be..9f6a726 100755 --- a/test-wait-for-port +++ b/test-wait-for-port @@ -1,6 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,7 +28,7 @@ host = sys.argv[1] port = int(sys.argv[2]) while True: - print "Trying %s port %s" % (host, port) + print("Trying %s port %s" % (host, port)) s = socket.socket() try: s.connect((host, port)) diff --git a/yarns.webapp/900-implementations.yarn b/yarns.webapp/900-implementations.yarn index 06411a7..245cd73 100644 --- a/yarns.webapp/900-implementations.yarn +++ b/yarns.webapp/900-implementations.yarn @@ -86,7 +86,7 @@ most of the configuration. IMPLEMENTS GIVEN (\S+) in (\S+) adds lorries (\S+) using prefix (\S+) cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import yarnlib @@ -112,7 +112,7 @@ most of the configuration. IMPLEMENTS GIVEN (\S+) in (\S+) adds trove (\S+) cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import yarnlib @@ -138,7 +138,7 @@ file. IMPLEMENTS GIVEN (\S+) in (\S+) has (\S+) set to (.+) for everything cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import json import yarnlib @@ -159,7 +159,7 @@ Set a specific field for a `troves` section. IMPLEMENTS GIVEN (\S+) in (\S+) sets (\S+) to (.+) for trove (\S+) cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import json import yarnlib @@ -182,7 +182,7 @@ Remove a specified field for a `troves` section IMPLEMENTS GIVEN (\S+) in (\S+) removes field (\S+) from trove (\S+) cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import yarnlib @@ -204,7 +204,7 @@ file. Note that the Trove must already be in the configuration file. IMPLEMENTS GIVEN (\S+) in (\S+) has prefixmap (\S+):(\S+) for (\S+) cd "$SRCDIR"/yarns.webapp - python -c ' + python3 -c ' import os import yarnlib @@ -242,7 +242,7 @@ Control the ls listing of a remote Trove. echo "{}" > "$filename" fi cat "$filename" - python -c ' + python3 -c ' import json, os, sys MATCH_2 = os.environ["MATCH_2"] filename = sys.argv[1] @@ -262,7 +262,7 @@ Remove a repository from the fake remote Trove. echo "{}" > "$filename" fi cat "$filename" - python -c ' + python3 -c ' import json, os, sys MATCH_2 = os.environ["MATCH_2"] filename = sys.argv[1] @@ -334,7 +334,7 @@ value is expresssed as a JSON value in the step. IMPLEMENTS THEN response has (\S+) set to (.+) cat "$DATADIR/response.body" - python -c ' + python3 -c ' import json, os, sys data = json.load(sys.stdin) key = os.environ["MATCH_1"] @@ -353,22 +353,22 @@ we may need to look at a list of dicts, as below. IMPLEMENTS THEN response has (\S+) item (\d+) field (\S+) set to (\S+) cat "$DATADIR/response.body" - python -c ' + python3 -c ' import json, os, sys data = json.load(sys.stdin) - print "data:", repr(data) + print("data:", repr(data)) items = os.environ["MATCH_1"] - print "items:", repr(items) + print("items:", repr(items)) item = int(os.environ["MATCH_2"]) - print "item:", repr(item) + print("item:", repr(item)) field = os.environ["MATCH_3"] - print "field:", repr(field) - print "match3:", repr(os.environ["MATCH_4"]) + print("field:", repr(field)) + print("match3:", repr(os.environ["MATCH_4"])) expected = json.loads(os.environ["MATCH_4"]) - print "expected:", repr(expected) - print "data[items]:", repr(data[items]) - print "data[items][item]:", repr(data[items][item]) - print "data[items][item][field]:", repr(data[items][item][field]) + print("expected:", repr(expected)) + print("data[items]:", repr(data[items])) + print("data[items][item]:", repr(data[items][item])) + print("data[items][item][field]:", repr(data[items][item][field])) value = data[items][item][field] if value != expected: sys.stderr.write( @@ -384,7 +384,7 @@ value, but we do care that it is there. IMPLEMENTS THEN response has (\S+) set cat "$DATADIR/response.body" - python -c ' + python3 -c ' import json, os, sys data = json.load(sys.stdin) key = os.environ["MATCH_1"] diff --git a/yarns.webapp/yarn.sh b/yarns.webapp/yarn.sh index 2a9081d..260fd74 100644 --- a/yarns.webapp/yarn.sh +++ b/yarns.webapp/yarn.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,6 +40,7 @@ add_to_config_file() then printf '[config]\n' > "$1" fi + sed -i "/^$2/d" "$1" printf '%s = %s\n' "$2" "$3" >> "$1" } diff --git a/yarns.webapp/yarnlib.py b/yarns.webapp/yarnlib.py index 89f87d9..4582649 100644 --- a/yarns.webapp/yarnlib.py +++ b/yarns.webapp/yarnlib.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 Codethink Limited +# Copyright (C) 2015-2019 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ def matches(): matches = {} prefix = "MATCH_" - for key, value in os.environ.iteritems(): + for key, value in os.environ.items(): if key.startswith(prefix): stripped_key = key[len(prefix):] if stripped_key.isdigit() and stripped_key != "0": |