diff options
Diffstat (limited to 'distbuild')
| -rw-r--r-- | distbuild/__init__.py | 52 | ||||
| -rw-r--r-- | distbuild/artifact_reference.py | 4 | ||||
| -rw-r--r-- | distbuild/build_controller.py | 17 | ||||
| -rw-r--r-- | distbuild/crashpoint_tests.py | 2 | ||||
| -rw-r--r-- | distbuild/helper_router.py | 2 | ||||
| -rw-r--r-- | distbuild/idgen.py | 2 | ||||
| -rw-r--r-- | distbuild/initiator.py | 2 | ||||
| -rw-r--r-- | distbuild/initiator_connection.py | 2 | ||||
| -rw-r--r-- | distbuild/jm.py | 6 | ||||
| -rw-r--r-- | distbuild/protocol.py | 2 | ||||
| -rw-r--r-- | distbuild/route_map.py | 2 | ||||
| -rw-r--r-- | distbuild/sockbuf.py | 6 | ||||
| -rw-r--r-- | distbuild/socketsrc.py | 2 | ||||
| -rw-r--r-- | distbuild/sockserv.py | 4 | ||||
| -rw-r--r-- | distbuild/worker_build_scheduler.py | 29 |
15 files changed, 74 insertions, 60 deletions
diff --git a/distbuild/__init__.py b/distbuild/__init__.py index bc5bc9b5..5f95e071 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -15,34 +15,34 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. -from stringbuffer import StringBuffer -from sm import StateMachine -from eventsrc import EventSource -from socketsrc import (SocketError, NewConnection, ListeningSocketEventSource, +from .stringbuffer import StringBuffer +from .sm import StateMachine +from .eventsrc import EventSource +from .socketsrc import (SocketError, NewConnection, ListeningSocketEventSource, SocketReadable, SocketWriteable, SocketEventSource, set_nonblocking) -from sockbuf import (SocketBufferNewData, SocketBufferEof, +from .sockbuf import (SocketBufferNewData, SocketBufferEof, SocketBufferClosed, SocketBuffer) -from mainloop import MainLoop -from sockserv import ListenServer -from jm import JsonMachine, JsonNewMessage, JsonEof, JsonError +from .mainloop import MainLoop +from .sockserv import ListenServer +from .jm import JsonMachine, JsonNewMessage, JsonEof, JsonError -from artifact_reference import (encode_artifact, +from .artifact_reference import (encode_artifact, encode_artifact_reference, decode_artifact_reference) -from idgen import IdentifierGenerator -from route_map import RouteMap -from timer_event_source import TimerEventSource, Timer -from proxy_event_source import ProxyEventSource -from json_router import JsonRouter -from helper_router import (HelperRouter, HelperRequest, HelperOutput, +from .idgen import IdentifierGenerator +from .route_map import RouteMap +from .timer_event_source import TimerEventSource, Timer +from .proxy_event_source import ProxyEventSource +from .json_router import JsonRouter +from .helper_router import (HelperRouter, HelperRequest, HelperOutput, HelperResult) -from initiator_connection import (InitiatorConnection, InitiatorDisconnect, +from .initiator_connection import (InitiatorConnection, InitiatorDisconnect, CancelRequest) -from connection_machine import (ConnectionMachine, InitiatorConnectionMachine, +from .connection_machine import (ConnectionMachine, InitiatorConnectionMachine, Reconnect, StopConnecting) -from worker_build_scheduler import (WorkerBuildQueuer, - WorkerConnection, +from .worker_build_scheduler import (WorkerBuildQueuer, + WorkerConnection, WorkerBuildRequest, WorkerCancelPending, WorkerBuildOutput, @@ -53,21 +53,21 @@ from worker_build_scheduler import (WorkerBuildQueuer, WorkerBuildFailed, WorkerBuildStepStarted) -from build_controller import (BuildController, BuildFailed, BuildProgress, +from .build_controller import (BuildController, BuildFailed, BuildProgress, BuildStepStarted, BuildStepAlreadyStarted, BuildOutput, BuildStepFinished, BuildStepFailed, BuildFinished, BuildCancel, BuildStarted, GraphingStarted, GraphingFinished, CacheState, build_step_name, map_build_graph) -from initiator import (Initiator, InitiatorStart, InitiatorCommand) -from protocol import message +from .initiator import (Initiator, InitiatorStart, InitiatorCommand) +from .protocol import message -from crashpoint import (crash_point, add_crash_condition, add_crash_conditions, - clear_crash_conditions) +from .crashpoint import (crash_point, add_crash_condition, + add_crash_conditions, clear_crash_conditions) -from distbuild_socket import create_socket +from .distbuild_socket import create_socket -from subprocess_eventsrc import (FileReadable, FileWriteable, +from .subprocess_eventsrc import (FileReadable, FileWriteable, SubprocessEventSource) __all__ = locals() diff --git a/distbuild/artifact_reference.py b/distbuild/artifact_reference.py index 633ed749..6e251144 100644 --- a/distbuild/artifact_reference.py +++ b/distbuild/artifact_reference.py @@ -179,14 +179,14 @@ def decode_artifact_reference(encoded): artifacts = {} # decode artifacts - for basename, artifact_dict in encoded_artifacts.iteritems(): + for basename, artifact_dict in encoded_artifacts.items(): artifact_dict.update(encoded_sources[artifact_dict['cache_key']]) artifact = ArtifactReference(basename, artifact_dict) artifact.root_filename = content['root-filename'] artifacts[basename] = artifact # add dependencies - for basename, a_dict in encoded_artifacts.iteritems(): + for basename, a_dict in encoded_artifacts.items(): artifact = artifacts[basename] artifact.dependencies = [artifacts.get(dep) for dep in artifact.dependencies] diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index bfd910b2..9762a691 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -16,14 +16,21 @@ import logging -import httplib import traceback -import urllib -import urlparse import json +import sys import distbuild +if sys.version_info[0] == 2: + import httplib + from urllib import quote as url_quote + import urlparse +else: + import http.client as httplib + import urllib.parse as urlparse + from urllib.parse import quote as url_quote + # Artifact build states. These are used to loosely track the state of the # remote cache. @@ -905,13 +912,13 @@ class BuildController(distbuild.StateMachine): (c.cache_key, c.kind, c.name)) - urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + urls.append('%s?filename=%s' % (baseurl, url_quote(name))) if not self._components: name = ('%s.%s.%s' % (self._artifact.cache_key, self._artifact.kind, self._artifact.name)) - urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) + urls.append('%s?filename=%s' % (baseurl, url_quote(name))) finished = BuildFinished(self._request['id'], urls) self.mainloop.queue_event(BuildController, finished) diff --git a/distbuild/crashpoint_tests.py b/distbuild/crashpoint_tests.py index be073a96..50efd3fb 100644 --- a/distbuild/crashpoint_tests.py +++ b/distbuild/crashpoint_tests.py @@ -17,7 +17,7 @@ import unittest -import crashpoint +import distbuild.crashpoint as crashpoint class CrashConditionTests(unittest.TestCase): diff --git a/distbuild/helper_router.py b/distbuild/helper_router.py index 22096b81..a6647abe 100644 --- a/distbuild/helper_router.py +++ b/distbuild/helper_router.py @@ -167,7 +167,7 @@ class HelperRouter(distbuild.StateMachine): self._pending_helpers.remove(event_source) # Re-queue any requests running on the hlper that just quit. - for request_id in self._running_requests.keys(): + for request_id in list(self._running_requests.keys()): request, helper = self._running_requests[request_id] if event_source == helper: del self._running_requests[request_id] diff --git a/distbuild/idgen.py b/distbuild/idgen.py index ef5684d7..ac908e91 100644 --- a/distbuild/idgen.py +++ b/distbuild/idgen.py @@ -25,7 +25,7 @@ class IdentifierGenerator(object): def __init__(self, series): self._series = series self._counter = 0 - + def next(self): self._counter += 1 return '%s-%d' % (self._series, self._counter) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index 84a1f27f..853ed2e9 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -295,7 +295,7 @@ class Initiator(distbuild.StateMachine): # makes it easier to tell whether a build was aborted due to a bug or # dropped connection, or if the user cancelled with CTRL+C / SIGINT. - for f in self._step_outputs.itervalues(): + for f in self._step_outputs.values(): self._write_status_to_build_log(f, 'Initiator cancelled') f.close() diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index d48ad214..daa1b2d4 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -167,7 +167,7 @@ class InitiatorConnection(distbuild.StateMachine): self._log_send(msg) def _handle_build_request(self, event): - new_id = self._idgen.next() + new_id = next(self._idgen) self.our_ids.add(new_id) self._route_map.add(event.msg['id'], new_id) event.msg['id'] = new_id diff --git a/distbuild/jm.py b/distbuild/jm.py index d1e5fab7..3223a8a9 100644 --- a/distbuild/jm.py +++ b/distbuild/jm.py @@ -23,9 +23,9 @@ import socket import sys import yaml -from sm import StateMachine -from stringbuffer import StringBuffer -from sockbuf import (SocketBuffer, SocketBufferNewData, +from .sm import StateMachine +from .stringbuffer import StringBuffer +from .sockbuf import (SocketBuffer, SocketBufferNewData, SocketBufferEof, SocketError) diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 44552ae1..4ed963f3 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -133,7 +133,7 @@ def _validate(message_type, **kwargs): required_fields = _required_fields[message_type] optional_fields = _optional_fields.get(message_type, []) - known_types = _required_fields.keys() + known_types = list(_required_fields.keys()) assert message_type in known_types for name in required_fields: diff --git a/distbuild/route_map.py b/distbuild/route_map.py index 0d482e24..a3af92fd 100644 --- a/distbuild/route_map.py +++ b/distbuild/route_map.py @@ -53,7 +53,7 @@ class RouteMap(object): return self._routes[outgoing_id] def get_outgoing_ids(self, incoming_id): - return [o for (o, i) in self._routes.iteritems() if i == incoming_id] + return [o for (o, i) in self._routes.items() if i == incoming_id] def remove(self, outgoing_id): del self._routes[outgoing_id] diff --git a/distbuild/sockbuf.py b/distbuild/sockbuf.py index be17a8f4..05816014 100644 --- a/distbuild/sockbuf.py +++ b/distbuild/sockbuf.py @@ -45,10 +45,10 @@ been emptied. ''' -from socketsrc import (SocketError, SocketReadable, SocketWriteable, +from .socketsrc import (SocketError, SocketReadable, SocketWriteable, SocketEventSource) -from sm import StateMachine -from stringbuffer import StringBuffer +from .sm import StateMachine +from .stringbuffer import StringBuffer class SocketBufferNewData(object): diff --git a/distbuild/socketsrc.py b/distbuild/socketsrc.py index 8207c35c..6bd88278 100644 --- a/distbuild/socketsrc.py +++ b/distbuild/socketsrc.py @@ -22,7 +22,7 @@ import socket import distbuild -from eventsrc import EventSource +from .eventsrc import EventSource def set_nonblocking(handle): diff --git a/distbuild/sockserv.py b/distbuild/sockserv.py index 998cfb11..5d7c222a 100644 --- a/distbuild/sockserv.py +++ b/distbuild/sockserv.py @@ -17,8 +17,8 @@ import logging -from sm import StateMachine -from socketsrc import NewConnection, SocketError, ListeningSocketEventSource +from .sm import StateMachine +from .socketsrc import NewConnection, SocketError, ListeningSocketEventSource class ListenServer(StateMachine): diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index e5548ad4..b0705b5b 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -16,15 +16,22 @@ import collections -import httplib import logging import socket -import urllib -import urlparse +import sys import distbuild +if sys.version_info[0] == 2: + import httplib + from urllib import quote as url_quote + import urlparse +else: + import http.client as httplib + import urllib.parse as urlparse + from urllib.parse import quote as url_quote + class WorkerBuildRequest(object): def __init__(self, artifact, initiator_id): @@ -169,7 +176,7 @@ class JobQueue(object): return True def __iter__(self): - return self._jobs.itervalues() + return iter(self._jobs.values()) def remove_jobs(self, jobs): for job in jobs: @@ -186,7 +193,7 @@ class JobQueue(object): def __repr__(self): items = [] - for job in self._jobs.itervalues(): + for job in self._jobs.values(): items.append( '%s (%s)' % (job.artifact.basename(), job.describe_state())) return str(items) @@ -400,9 +407,9 @@ class WorkerBuildQueuer(distbuild.StateMachine): # the _available_workers list. But anything can happen in space! So we # take care to remove all GiveJob messages in the list that came from # the disconnected worker, not the first. - self._available_workers = filter( - lambda worker_msg: worker_msg.who != worker, - self._available_workers) + self._available_workers = [worker_msg + for worker_msg in self._available_workers + if worker_msg.who != worker] class WorkerConnection(distbuild.StateMachine): @@ -648,7 +655,7 @@ class WorkerConnection(distbuild.StateMachine): if kind == 'stratum': suffixes.append(filename + '.meta') - suffixes = [urllib.quote(x) for x in suffixes] + suffixes = [url_quote(x) for x in suffixes] suffixes = ','.join(suffixes) worker_host = self._conn.getpeername()[0] @@ -656,9 +663,9 @@ class WorkerConnection(distbuild.StateMachine): url = urlparse.urljoin( self._writeable_cache_server, '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % - (urllib.quote(worker_host), + (url_quote(worker_host), self._worker_cache_server_port, - urllib.quote(job.artifact.cache_key), + url_quote(job.artifact.cache_key), suffixes)) msg = distbuild.message( |