diff options
40 files changed, 630 insertions, 771 deletions
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 54b229e..c8c660e 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -15,7 +15,6 @@ jobs: strategy: matrix: py: - - "2.7" - "3.5" - "3.6" - "3.7" @@ -56,11 +55,6 @@ jobs: name: Validate coverage steps: - uses: actions/checkout@v2 - - name: Setup python 2.7 - uses: actions/setup-python@v2 - with: - python-version: 2.7 - architecture: x64 - name: Setup python 3.8 uses: actions/setup-python@v2 with: @@ -68,7 +62,7 @@ jobs: architecture: x64 - run: pip install tox - - run: tox -e py38,py27,coverage + - run: tox -e py38,coverage docs: runs-on: ubuntu-latest name: Build the documentation diff --git a/CHANGES.txt b/CHANGES.txt index 5550995..f4d1acc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,8 @@ +2.0.0 (unreleased) +------------------ + +- Drop Python 2.7 support + 1.4.4 (2020-06-01) ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index 7540038..b41b4db 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,5 +17,6 @@ include .coveragerc .flake8 include tox.ini rtd.txt exclude TODO.txt +prune docs/_build recursive-exclude * __pycache__ *.py[cod] @@ -16,10 +16,11 @@ Waitress :target: https://webchat.freenode.net/?channels=pyramid :alt: IRC Freenode -Waitress is meant to be a production-quality pure-Python WSGI server with very -acceptable performance. It has no dependencies except ones which live in the -Python standard library. It runs on CPython on Unix and Windows under Python -2.7+ and Python 3.5+. It is also known to run on PyPy 1.6.0+ on UNIX. It -supports HTTP/1.0 and HTTP/1.1. +Waitress is a production-quality pure-Python WSGI server with very acceptable +performance. It has no dependencies except ones which live in the Python +standard library. It runs on CPython on Unix and Windows under Python 3.5+. It +is also known to run on PyPy (version 3.5 compatible) on UNIX. It supports +HTTP/1.0 and HTTP/1.1. -For more information, see the "docs" directory of the Waitress package or visit https://docs.pylonsproject.org/projects/waitress/en/latest/ +For more information, see the "docs" directory of the Waitress package or visit +https://docs.pylonsproject.org/projects/waitress/en/latest/ diff --git a/pyproject.toml b/pyproject.toml index 7f50ece..b68b905 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,25 @@ requires = ["setuptools >= 41"] build-backend = "setuptools.build_meta" [tool.black] -py36 = false +target-version = ['py35', 'py36', 'py37', 'py38'] exclude = ''' /( \.git | .tox )/ ''' + + # This next section only exists for people that have their editors +# automatically call isort, black already sorts entries on its own when run. +[tool.isort] +profile = "black" +multi_line_output = 3 +src_paths = ["src", "tests"] +skip_glob = ["docs/*"] +include_trailing_comma = true +force_grid_wrap = false +combine_as_imports = true +line_length = 88 +force_sort_within_sections = true +default_section = "THIRDPARTY" +known_first_party = "waitress" @@ -1,6 +1,6 @@ [metadata] name = waitress -version = 1.4.4 +version = 2.0.0dev0 description = Waitress WSGI server long_description = file: README.rst, CHANGES.txt long_description_content_type = text/x-rst @@ -12,8 +12,6 @@ classifiers = Intended Audience :: Developers License :: OSI Approved :: Zope Public License Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 @@ -39,7 +37,7 @@ maintainer_email = pylons-discuss@googlegroups.com package_dir= =src packages=find: -python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.* +python_requires = >=3.5.0 [options.entry_points] paste.server_runner = @@ -61,13 +59,10 @@ docs = docutils pylons-sphinx-themes>=1.0.9 -[bdist_wheel] -universal = 1 - [tool:pytest] python_files = test_*.py # For the benefit of test_wasyncore.py python_classes = Test_* testpaths = tests -addopts = -W always --cov +addopts = --cov -W always diff --git a/src/waitress/__init__.py b/src/waitress/__init__.py index e6e5911..bbb99da 100644 --- a/src/waitress/__init__.py +++ b/src/waitress/__init__.py @@ -1,6 +1,7 @@ -from waitress.server import create_server import logging +from waitress.server import create_server + def serve(app, **kw): _server = kw.pop("_server", create_server) # test shim diff --git a/src/waitress/adjustments.py b/src/waitress/adjustments.py index 93439ea..145ac86 100644 --- a/src/waitress/adjustments.py +++ b/src/waitress/adjustments.py @@ -17,13 +17,8 @@ import getopt import socket import warnings +from .compat import HAS_IPV6, WIN from .proxy_headers import PROXY_HEADERS -from .compat import ( - PY2, - WIN, - string_types, - HAS_IPV6, -) truthy = frozenset(("t", "true", "y", "yes", "on", "1")) @@ -52,7 +47,7 @@ def asoctal(s): def aslist_cronly(value): - if isinstance(value, string_types): + if isinstance(value, str): value = filter(None, [x.strip() for x in value.splitlines()]) return list(value) @@ -100,11 +95,11 @@ class _int_marker(int): pass -class _bool_marker(object): +class _bool_marker: pass -class Adjustments(object): +class Adjustments: """This class contains tunable parameters. """ @@ -346,7 +341,7 @@ class Adjustments(object): else: (host, port) = (i, str(self.port)) - if WIN and PY2: # pragma: no cover + if WIN: # pragma: no cover try: # Try turning the port into an integer port = int(port) diff --git a/src/waitress/buffers.py b/src/waitress/buffers.py index 04f6b42..0086fe8 100644 --- a/src/waitress/buffers.py +++ b/src/waitress/buffers.py @@ -22,7 +22,7 @@ COPY_BYTES = 1 << 18 # 256K STRBUF_LIMIT = 8192 -class FileBasedBuffer(object): +class FileBasedBuffer: remain = 0 @@ -187,7 +187,7 @@ class ReadOnlyFileBasedBuffer(FileBasedBuffer): raise NotImplementedError -class OverflowableBuffer(object): +class OverflowableBuffer: """ This buffer implementation has four stages: - No data diff --git a/src/waitress/channel.py b/src/waitress/channel.py index bc9a2bb..d756b96 100644 --- a/src/waitress/channel.py +++ b/src/waitress/channel.py @@ -16,18 +16,9 @@ import threading import time import traceback -from waitress.buffers import ( - OverflowableBuffer, - ReadOnlyFileBasedBuffer, -) - +from waitress.buffers import OverflowableBuffer, ReadOnlyFileBasedBuffer from waitress.parser import HTTPRequestParser - -from waitress.task import ( - ErrorTask, - WSGITask, -) - +from waitress.task import ErrorTask, WSGITask from waitress.utilities import InternalServerError from . import wasyncore @@ -37,7 +28,7 @@ class ClientDisconnected(Exception): """ Raised when attempting to write to a closed socket.""" -class HTTPChannel(wasyncore.dispatcher, object): +class HTTPChannel(wasyncore.dispatcher): """ Setting self.requests = [somerequest] prevents more requests from being received until the out buffers have been flushed. @@ -85,16 +76,20 @@ class HTTPChannel(wasyncore.dispatcher, object): # if there's data in the out buffer or we've been instructed to close # the channel (possibly by our server maintenance logic), run # handle_write + return self.total_outbufs_len or self.will_close or self.close_when_flushed def handle_write(self): # Precondition: there's data in the out buffer to be sent, or # there's a pending will_close request + if not self.connected: # we dont want to close the channel twice + return # try to flush any pending output + if not self.requests: # 1. There are no running tasks, so we don't need to try to lock # the outbuf before sending @@ -116,11 +111,11 @@ class HTTPChannel(wasyncore.dispatcher, object): if flush: try: flush() - except socket.error: + except OSError: if self.adj.log_socket_errors: self.logger.exception("Socket error") self.will_close = True - except Exception: + except Exception: # pragma: nocover self.logger.exception("Unexpected exception when flushing") self.will_close = True @@ -134,19 +129,29 @@ class HTTPChannel(wasyncore.dispatcher, object): def readable(self): # We might want to create a new task. We can only do this if: # 1. We're not already about to close the connection. - # 2. There's no already currently running task(s). - # 3. There's no data in the output buffer that needs to be sent + # 2. We're not waiting to flush remaining data before closing the + # connection + # 3. There's no already currently running task(s). + # 4. There's no data in the output buffer that needs to be sent # before we potentially create a new task. - return not (self.will_close or self.requests or self.total_outbufs_len) + + return not ( + self.will_close + or self.close_when_flushed + or self.requests + or self.total_outbufs_len + ) def handle_read(self): try: data = self.recv(self.adj.recv_bytes) - except socket.error: + except OSError: if self.adj.log_socket_errors: self.logger.exception("Socket error") self.handle_close() + return + if data: self.last_activity = time.time() self.received(data) @@ -167,9 +172,11 @@ class HTTPChannel(wasyncore.dispatcher, object): if request is None: request = self.parser_class(self.adj) n = request.received(data) + if request.expect_continue and request.headers_finished: # guaranteed by parser to be a 1.1 request request.expect_continue = False + if not self.sent_continue: # there's no current task, so we don't need to try to # lock the outbuf to append to it. @@ -181,14 +188,17 @@ class HTTPChannel(wasyncore.dispatcher, object): self.sent_continue = True self._flush_some() request.completed = False + if request.completed: # The request (with the body) is ready to use. self.request = None + if not request.empty: requests.append(request) request = None else: self.request = request + if n >= len(data): break data = data[n:] @@ -202,6 +212,7 @@ class HTTPChannel(wasyncore.dispatcher, object): def _flush_some_if_lockable(self): # Since our task may be appending to the outbuf, we try to acquire # the lock, but we don't block if we can't. + if self.outbuf_lock.acquire(False): try: self._flush_some() @@ -222,9 +233,11 @@ class HTTPChannel(wasyncore.dispatcher, object): # use outbuf.__len__ rather than len(outbuf) FBO of not getting # OverflowError on 32-bit Python outbuflen = outbuf.__len__() + while outbuflen > 0: chunk = outbuf.get(self.sendbuf_len) num_sent = self.send(chunk) + if num_sent: outbuf.skip(num_sent, True) outbuflen -= num_sent @@ -233,9 +246,11 @@ class HTTPChannel(wasyncore.dispatcher, object): else: # failed to write anything, break out entirely dobreak = True + break else: # self.outbufs[-1] must always be a writable outbuf + if len(self.outbufs) > 1: toclose = self.outbufs.pop(0) try: @@ -251,6 +266,7 @@ class HTTPChannel(wasyncore.dispatcher, object): if sent: self.last_activity = time.time() + return True return False @@ -285,6 +301,7 @@ class HTTPChannel(wasyncore.dispatcher, object): fd = self._fileno # next line sets this to None wasyncore.dispatcher.del_channel(self, map) ac = self.server.active_channels + if fd in ac: del ac[fd] @@ -297,14 +314,17 @@ class HTTPChannel(wasyncore.dispatcher, object): # if the socket is closed then interrupt the task so that it # can cleanup possibly before the app_iter is exhausted raise ClientDisconnected + if data: # the async mainloop might be popping data off outbuf; we can # block here waiting for it because we're in a task thread with self.outbuf_lock: self._flush_outbufs_below_high_watermark() + if not self.connected: raise ClientDisconnected num_bytes = len(data) + if data.__class__ is ReadOnlyFileBasedBuffer: # they used wsgi.file_wrapper self.outbufs.append(data) @@ -321,13 +341,17 @@ class HTTPChannel(wasyncore.dispatcher, object): self.outbufs[-1].append(data) self.current_outbuf_count += num_bytes self.total_outbufs_len += num_bytes + if self.total_outbufs_len >= self.adj.send_bytes: self.server.pull_trigger() + return num_bytes + return 0 def _flush_outbufs_below_high_watermark(self): # check first to avoid locking if possible + if self.total_outbufs_len > self.adj.outbuf_high_watermark: with self.outbuf_lock: while ( @@ -342,6 +366,7 @@ class HTTPChannel(wasyncore.dispatcher, object): with self.task_lock: while self.requests: request = self.requests[0] + if request.error: task = self.error_task_class(self, request) else: @@ -357,6 +382,7 @@ class HTTPChannel(wasyncore.dispatcher, object): self.logger.exception( "Exception while serving %s" % task.request.path ) + if not task.wrote_header: if self.adj.expose_tracebacks: body = traceback.format_exc() @@ -385,8 +411,10 @@ class HTTPChannel(wasyncore.dispatcher, object): task.close_on_finish = True # we cannot allow self.requests to drop to empty til # here; otherwise the mainloop gets confused + if task.close_on_finish: self.close_when_flushed = True + for request in self.requests: request.close() self.requests = [] @@ -398,6 +426,7 @@ class HTTPChannel(wasyncore.dispatcher, object): # that we need to account for, otherwise it'd be better # to do this check at the start of the request instead of # at the end to account for consecutive service() calls + if len(self.requests) > 1: self._flush_outbufs_below_high_watermark() @@ -406,6 +435,7 @@ class HTTPChannel(wasyncore.dispatcher, object): # outbufs across requests which can cause outbufs to # not be deallocated regularly when a connection is open # for a long time + if self.current_outbuf_count > 0: self.current_outbuf_count = self.adj.outbuf_high_watermark diff --git a/src/waitress/compat.py b/src/waitress/compat.py index fe72a76..67543b9 100644 --- a/src/waitress/compat.py +++ b/src/waitress/compat.py @@ -1,135 +1,15 @@ -import os -import sys -import types import platform -import warnings - -try: - import urlparse -except ImportError: # pragma: no cover - from urllib import parse as urlparse - -try: - import fcntl -except ImportError: # pragma: no cover - fcntl = None # windows - -# True if we are running on Python 3. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -# True if we are running on Windows -WIN = platform.system() == "Windows" - -if PY3: # pragma: no cover - string_types = (str,) - integer_types = (int,) - class_types = (type,) - text_type = str - binary_type = bytes - long = int -else: - string_types = (basestring,) - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - long = long - -if PY3: # pragma: no cover - from urllib.parse import unquote_to_bytes - - def unquote_bytes_to_wsgi(bytestring): - return unquote_to_bytes(bytestring).decode("latin-1") - - -else: - from urlparse import unquote as unquote_to_bytes - - def unquote_bytes_to_wsgi(bytestring): - return unquote_to_bytes(bytestring) - - -def text_(s, encoding="latin-1", errors="strict"): - """ If ``s`` is an instance of ``binary_type``, return - ``s.decode(encoding, errors)``, otherwise return ``s``""" - if isinstance(s, binary_type): - return s.decode(encoding, errors) - return s # pragma: no cover - - -if PY3: # pragma: no cover - - def tostr(s): - if isinstance(s, text_type): - s = s.encode("latin-1") - return str(s, "latin-1", "strict") - - def tobytes(s): - return bytes(s, "latin-1") - - -else: - tostr = str - - def tobytes(s): - return s - - -if PY3: # pragma: no cover - import builtins - - exec_ = getattr(builtins, "exec") - - def reraise(tp, value, tb=None): - if value is None: - value = tp - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - - del builtins - -else: # pragma: no cover - - def exec_(code, globs=None, locs=None): - """Execute code in a namespace.""" - if globs is None: - frame = sys._getframe(1) - globs = frame.f_globals - if locs is None: - locs = frame.f_locals - del frame - elif locs is None: - locs = globs - exec("""exec code in globs, locs""") - - exec_( - """def reraise(tp, value, tb=None): - raise tp, value, tb -""" - ) - -try: - from StringIO import StringIO as NativeIO -except ImportError: # pragma: no cover - from io import StringIO as NativeIO - -try: - import httplib -except ImportError: # pragma: no cover - from http import client as httplib - -try: - MAXINT = sys.maxint -except AttributeError: # pragma: no cover - MAXINT = sys.maxsize - # Fix for issue reported in https://github.com/Pylons/waitress/issues/138, # Python on Windows may not define IPPROTO_IPV6 in socket. import socket +import sys +import warnings +# True if we are running on Windows +WIN = platform.system() == "Windows" + +MAXINT = sys.maxsize HAS_IPV6 = socket.has_ipv6 if hasattr(socket, "IPPROTO_IPV6") and hasattr(socket, "IPV6_V6ONLY"): @@ -147,33 +27,3 @@ else: # pragma: no cover RuntimeWarning, ) HAS_IPV6 = False - - -def set_nonblocking(fd): # pragma: no cover - if PY3 and sys.version_info[1] >= 5: - os.set_blocking(fd, False) - elif fcntl is None: - raise RuntimeError("no fcntl module present") - else: - flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) - - -if PY3: - ResourceWarning = ResourceWarning -else: - ResourceWarning = UserWarning - - -def qualname(cls): - if PY3: - return cls.__qualname__ - return cls.__name__ - - -try: - import thread -except ImportError: - # py3 - import _thread as thread diff --git a/src/waitress/parser.py b/src/waitress/parser.py index 765fe59..3b99921 100644 --- a/src/waitress/parser.py +++ b/src/waitress/parser.py @@ -16,11 +16,12 @@ This server uses asyncore to accept connections and do initial processing but threads to do work. """ -import re from io import BytesIO +import re +from urllib import parse +from urllib.parse import unquote_to_bytes from waitress.buffers import OverflowableBuffer -from waitress.compat import tostr, unquote_bytes_to_wsgi, urlparse from waitress.receiver import ChunkedReceiver, FixedStreamReceiver from waitress.utilities import ( BadRequest, @@ -29,9 +30,14 @@ from waitress.utilities import ( ServerNotImplemented, find_double_newline, ) + from .rfc7230 import HEADER_FIELD +def unquote_bytes_to_wsgi(bytestring): + return unquote_to_bytes(bytestring).decode("latin-1") + + class ParsingError(Exception): pass @@ -40,7 +46,7 @@ class TransferEncodingNotImplemented(Exception): pass -class HTTPRequestParser(object): +class HTTPRequestParser: """A structure that collects the HTTP request. Once the stream is completed, the instance is passed to @@ -79,11 +85,13 @@ class HTTPRequestParser(object): bytes consumed. Sets the completed flag once both the header and the body have been received. """ + if self.completed: return 0 # Can't consume any more. datalen = len(data) br = self.body_rcv + if br is None: # In header. max_header = self.adj.max_request_header_size @@ -105,12 +113,14 @@ class HTTPRequestParser(object): # If the first line + headers is over the max length, we return a # RequestHeaderFieldsTooLarge error rather than continuing to # attempt to parse the headers. + if self.header_bytes_received >= max_header: self.parse_header(b"GET / HTTP/1.0\r\n") self.error = RequestHeaderFieldsTooLarge( "exceeds max_header of %s" % max_header ) self.completed = True + return consumed if index >= 0: @@ -194,6 +204,7 @@ class HTTPRequestParser(object): first line of the request). """ index = header_plus.find(b"\r\n") + if index >= 0: first_line = header_plus[:index].rstrip() header = header_plus[index + 2 :] @@ -208,6 +219,7 @@ class HTTPRequestParser(object): lines = get_header_lines(header) headers = self.headers + for line in lines: header = HEADER_FIELD.match(line) @@ -218,25 +230,26 @@ class HTTPRequestParser(object): if b"_" in key: # TODO(xistence): Should we drop this request instead? + continue # Only strip off whitespace that is considered valid whitespace by # RFC7230, don't strip the rest value = value.strip(b" \t") - key1 = tostr(key.upper().replace(b"-", b"_")) + key1 = key.upper().replace(b"-", b"_").decode("latin-1") # If a header already exists, we append subsequent values # separated by a comma. Applications already need to handle # the comma separated values, as HTTP front ends might do # the concatenation for you (behavior specified in RFC2616). try: - headers[key1] += tostr(b", " + value) + headers[key1] += (b", " + value).decode("latin-1") except KeyError: - headers[key1] = tostr(value) + headers[key1] = value.decode("latin-1") # command, uri, version will be bytes command, uri, version = crack_first_line(first_line) - version = tostr(version) - command = tostr(command) + version = version.decode("latin-1") + command = command.decode("latin-1") self.command = command self.version = version ( @@ -279,6 +292,7 @@ class HTTPRequestParser(object): # Note: the identity transfer-coding was removed in RFC7230: # https://tools.ietf.org/html/rfc7230#appendix-A.2 and is thus # not supported + if encoding not in {"chunked"}: raise TransferEncodingNotImplemented( "Transfer-Encoding requested is not supported." @@ -295,6 +309,7 @@ class HTTPRequestParser(object): expect = headers.get("EXPECT", "").lower() self.expect_continue = expect == "100-continue" + if connection.lower() == "close": self.connection_close = True @@ -305,12 +320,14 @@ class HTTPRequestParser(object): raise ParsingError("Content-Length is invalid") self.content_length = cl + if cl > 0: buf = OverflowableBuffer(self.adj.inbuf_overflow) self.body_rcv = FixedStreamReceiver(cl, buf) def get_body_stream(self): body_rcv = self.body_rcv + if body_rcv is not None: return body_rcv.getfile() else: @@ -318,6 +335,7 @@ class HTTPRequestParser(object): def close(self): body_rcv = self.body_rcv + if body_rcv is not None: body_rcv.getbuf().close() @@ -345,16 +363,16 @@ def split_uri(uri): path, query = path.split(b"?", 1) else: try: - scheme, netloc, path, query, fragment = urlparse.urlsplit(uri) + scheme, netloc, path, query, fragment = parse.urlsplit(uri) except UnicodeError: raise ParsingError("Bad URI") return ( - tostr(scheme), - tostr(netloc), + scheme.decode("latin-1"), + netloc.decode("latin-1"), unquote_bytes_to_wsgi(path), - tostr(query), - tostr(fragment), + query.decode("latin-1"), + fragment.decode("latin-1"), ) @@ -364,20 +382,24 @@ def get_header_lines(header): """ r = [] lines = header.split(b"\r\n") + for line in lines: if not line: continue if b"\r" in line or b"\n" in line: - raise ParsingError('Bare CR or LF found in header line "%s"' % tostr(line)) + raise ParsingError( + 'Bare CR or LF found in header line "%s"' % str(line, "latin-1") + ) if line.startswith((b" ", b"\t")): if not r: # https://corte.si/posts/code/pathod/pythonservers/index.html - raise ParsingError('Malformed header line "%s"' % tostr(line)) + raise ParsingError('Malformed header line "%s"' % str(line, "latin-1")) r[-1] += line else: r.append(line) + return r @@ -390,6 +412,7 @@ first_line_re = re.compile( def crack_first_line(line): m = first_line_re.match(line) + if m is not None and m.end() == len(line): if m.group(3): version = m.group(5) @@ -406,9 +429,11 @@ def crack_first_line(line): # unsuspecting souls from sending lowercase HTTP methods to waitress # and having the request complete, while servers like nginx drop the # request onto the floor. + if method != method.upper(): - raise ParsingError('Malformed HTTP method "%s"' % tostr(method)) + raise ParsingError('Malformed HTTP method "%s"' % str(method, "latin-1")) uri = m.group(2) + return method, uri, version else: return b"", b"", b"" diff --git a/src/waitress/proxy_headers.py b/src/waitress/proxy_headers.py index 1df8b8e..13cb2ed 100644 --- a/src/waitress/proxy_headers.py +++ b/src/waitress/proxy_headers.py @@ -1,7 +1,6 @@ from collections import namedtuple -from .utilities import logger, undquote, BadRequest - +from .utilities import BadRequest, logger, undquote PROXY_HEADERS = frozenset( { @@ -22,7 +21,7 @@ class MalformedProxyHeader(Exception): self.header = header self.reason = reason self.value = value - super(MalformedProxyHeader, self).__init__(header, reason, value) + super().__init__(header, reason, value) def proxy_headers_middleware( @@ -53,7 +52,7 @@ def proxy_headers_middleware( ex.reason, ex.value, ) - error = BadRequest('Header "{0}" malformed.'.format(ex.header)) + error = BadRequest('Header "{}" malformed.'.format(ex.header)) return error.wsgi_response(environ, start_response) # Clear out the untrusted proxy headers diff --git a/src/waitress/receiver.py b/src/waitress/receiver.py index 5d1568d..8785280 100644 --- a/src/waitress/receiver.py +++ b/src/waitress/receiver.py @@ -17,7 +17,7 @@ from waitress.utilities import BadRequest, find_double_newline -class FixedStreamReceiver(object): +class FixedStreamReceiver: # See IStreamConsumer completed = False @@ -59,7 +59,7 @@ class FixedStreamReceiver(object): return self.buf -class ChunkedReceiver(object): +class ChunkedReceiver: chunk_remainder = 0 validate_chunk_end = False diff --git a/src/waitress/rfc7230.py b/src/waitress/rfc7230.py index cd33c90..9b25fbd 100644 --- a/src/waitress/rfc7230.py +++ b/src/waitress/rfc7230.py @@ -5,8 +5,6 @@ needed to properly parse HTTP messages. import re -from .compat import tobytes - WS = "[ \t]" OWS = WS + "{0,}?" RWS = WS + "{1,}?" @@ -46,7 +44,7 @@ FIELD_CONTENT = FIELD_VCHAR + "+(?:[ \t]+" + FIELD_VCHAR + "+)*" FIELD_VALUE = "(?:" + FIELD_CONTENT + ")?" HEADER_FIELD = re.compile( - tobytes( + ( "^(?P<name>" + TOKEN + "):" + OWS + "(?P<value>" + FIELD_VALUE + ")" + OWS + "$" - ) + ).encode("latin-1") ) diff --git a/src/waitress/runner.py b/src/waitress/runner.py index 2495084..4fb3e6b 100644 --- a/src/waitress/runner.py +++ b/src/waitress/runner.py @@ -14,7 +14,6 @@ """Command line runner. """ -from __future__ import print_function, unicode_literals import getopt import os @@ -191,7 +190,7 @@ RUNNER_PATTERN = re.compile( def match(obj_name): matches = RUNNER_PATTERN.match(obj_name) if not matches: - raise ValueError("Malformed application '{0}'".format(obj_name)) + raise ValueError("Malformed application '{}'".format(obj_name)) return matches.group("module"), matches.group("object") @@ -216,7 +215,7 @@ def resolve(module_name, object_name): def show_help(stream, name, error=None): # pragma: no cover if error is not None: - print("Error: {0}\n".format(error), file=stream) + print("Error: {}\n".format(error), file=stream) print(HELP.format(name), file=stream) @@ -224,7 +223,7 @@ def show_exception(stream): exc_type, exc_value = sys.exc_info()[:2] args = getattr(exc_value, "args", None) print( - ("There was an exception ({0}) importing your module.\n").format( + ("There was an exception ({}) importing your module.\n").format( exc_type.__name__, ), file=stream, @@ -232,7 +231,7 @@ def show_exception(stream): if args: print("It had these arguments: ", file=stream) for idx, arg in enumerate(args, start=1): - print("{0}. {1}\n".format(idx, arg), file=stream) + print("{}. {}\n".format(idx, arg), file=stream) else: print("It had no arguments.", file=stream) @@ -269,11 +268,11 @@ def run(argv=sys.argv, _serve=serve): try: app = resolve(module, obj_name) except ImportError: - show_help(sys.stderr, name, "Bad module '{0}'".format(module)) + show_help(sys.stderr, name, "Bad module '{}'".format(module)) show_exception(sys.stderr) return 1 except AttributeError: - show_help(sys.stderr, name, "Bad object name '{0}'".format(obj_name)) + show_help(sys.stderr, name, "Bad object name '{}'".format(obj_name)) show_exception(sys.stderr) return 1 if kw["call"]: diff --git a/src/waitress/server.py b/src/waitress/server.py index ae56699..06bb957 100644 --- a/src/waitress/server.py +++ b/src/waitress/server.py @@ -20,13 +20,10 @@ import time from waitress import trigger from waitress.adjustments import Adjustments from waitress.channel import HTTPChannel +from waitress.compat import IPPROTO_IPV6, IPV6_V6ONLY from waitress.task import ThreadedTaskDispatcher from waitress.utilities import cleanup_unix_socket -from waitress.compat import ( - IPPROTO_IPV6, - IPV6_V6ONLY, -) from . import wasyncore from .proxy_headers import proxy_headers_middleware @@ -137,7 +134,7 @@ def create_server( # This class is only ever used if we have multiple listen sockets. It allows # the serve() API to call .run() which starts the wasyncore loop, and catches # SystemExit/KeyboardInterrupt so that it can atempt to cleanly shut down. -class MultiSocketServer(object): +class MultiSocketServer: asyncore = wasyncore # test shim def __init__( @@ -172,7 +169,7 @@ class MultiSocketServer(object): wasyncore.close_all(self.map) -class BaseWSGIServer(wasyncore.dispatcher, object): +class BaseWSGIServer(wasyncore.dispatcher): channel_class = HTTPChannel next_channel_cleanup = 0 @@ -260,7 +257,7 @@ class BaseWSGIServer(wasyncore.dispatcher, object): if server_name == "0.0.0.0" or server_name == "::": try: return str(self.socketmod.gethostname()) - except (socket.error, UnicodeDecodeError): # pragma: no cover + except (OSError, UnicodeDecodeError): # pragma: no cover # We also deal with UnicodeDecodeError in case of Windows with # non-ascii hostname return "localhost" @@ -268,7 +265,7 @@ class BaseWSGIServer(wasyncore.dispatcher, object): # Now let's try and convert the IP address to a proper hostname try: server_name = self.socketmod.gethostbyaddr(server_name)[0] - except (socket.error, UnicodeDecodeError): # pragma: no cover + except (OSError, UnicodeDecodeError): # pragma: no cover # We also deal with UnicodeDecodeError in case of Windows with # non-ascii hostname pass @@ -312,7 +309,7 @@ class BaseWSGIServer(wasyncore.dispatcher, object): if v is None: return conn, addr = v - except socket.error: + except OSError: # Linux: On rare occasions we get a bogus socket back from # accept. socketmodule.c:makesockaddr complains that the # address family is unknown. We don't want the whole server @@ -405,7 +402,7 @@ if hasattr(socket, "AF_UNIX"): if sockinfo is None: sockinfo = (socket.AF_UNIX, socket.SOCK_STREAM, None, None) - super(UnixWSGIServer, self).__init__( + super().__init__( application, map=map, _start=_start, @@ -413,7 +410,7 @@ if hasattr(socket, "AF_UNIX"): dispatcher=dispatcher, adj=adj, sockinfo=sockinfo, - **kw + **kw, ) def bind_server_socket(self): diff --git a/src/waitress/task.py b/src/waitress/task.py index 6350e34..1bcc540 100644 --- a/src/waitress/task.py +++ b/src/waitress/task.py @@ -12,14 +12,13 @@ # ############################################################################## +from collections import deque import socket import sys import threading import time -from collections import deque from .buffers import ReadOnlyFileBasedBuffer -from .compat import reraise, tobytes from .utilities import build_http_date, logger, queue_logger rename_headers = { # or keep them without the HTTP_ prefix added @@ -41,7 +40,7 @@ hop_by_hop = frozenset( ) -class ThreadedTaskDispatcher(object): +class ThreadedTaskDispatcher: """A Task Dispatcher that creates a thread for each task. """ @@ -141,7 +140,7 @@ class ThreadedTaskDispatcher(object): return False -class Task(object): +class Task: close_on_finish = False status = "200 OK" wrote_header = False @@ -166,16 +165,13 @@ class Task(object): def service(self): try: - try: - self.start() - self.execute() - self.finish() - except socket.error: - self.close_on_finish = True - if self.channel.adj.log_socket_errors: - raise - finally: - pass + self.start() + self.execute() + self.finish() + except OSError: + self.close_on_finish = True + if self.channel.adj.log_socket_errors: + raise @property def has_body(self): @@ -281,7 +277,7 @@ class Task(object): lines = [first_line] + next_lines res = "%s\r\n\r\n" % "\r\n".join(lines) - return tobytes(res) + return res.encode("latin-1") def remove_content_length_header(self): response_headers = [] @@ -317,7 +313,7 @@ class Task(object): cl = self.content_length if self.chunked_response: # use chunked encoding response - towrite = tobytes(hex(len(data))[2:].upper()) + b"\r\n" + towrite = hex(len(data))[2:].upper().encode("latin-1") + b"\r\n" towrite += data + b"\r\n" elif cl is not None: towrite = data[: cl - self.content_bytes_written] @@ -361,7 +357,7 @@ class ErrorTask(Task): self.response_headers.append(("Connection", "close")) self.close_on_finish = True self.content_length = len(body) - self.write(tobytes(body)) + self.write(body.encode("latin-1")) class WSGITask(Task): @@ -385,7 +381,7 @@ class WSGITask(Task): # 1. "service" method in task.py # 2. "service" method in channel.py # 3. "handler_thread" method in task.py - reraise(exc_info[0], exc_info[1], exc_info[2]) + raise exc_info[1] else: # As per WSGI spec existing headers must be cleared self.response_headers = [] diff --git a/src/waitress/trigger.py b/src/waitress/trigger.py index 6a57c12..24c4d0d 100644 --- a/src/waitress/trigger.py +++ b/src/waitress/trigger.py @@ -12,9 +12,9 @@ # ############################################################################## +import errno import os import socket -import errno import threading from . import wasyncore @@ -50,7 +50,7 @@ from . import wasyncore # the main thread is trying to remove some] -class _triggerbase(object): +class _triggerbase: """OS-independent base class for OS-dependent trigger class.""" kind = None # subclass must set to "pipe" or "loopback"; used by repr @@ -98,7 +98,7 @@ class _triggerbase(object): def handle_read(self): try: self.recv(8192) - except (OSError, socket.error): + except OSError: return with self.lock: for thunk in self.thunks: @@ -173,7 +173,7 @@ else: # pragma: no cover try: w.connect(connect_address) break # success - except socket.error as detail: + except OSError as detail: if detail[0] != errno.WSAEADDRINUSE: # "Address already in use" is the only error # I've seen on two WinXP Pro SP2 boxes, under diff --git a/src/waitress/utilities.py b/src/waitress/utilities.py index 556bed2..3caaa33 100644 --- a/src/waitress/utilities.py +++ b/src/waitress/utilities.py @@ -273,7 +273,7 @@ def cleanup_unix_socket(path): pass -class Error(object): +class Error: code = 500 reason = "Internal Server Error" diff --git a/src/waitress/wasyncore.py b/src/waitress/wasyncore.py index 09bcafa..9a68c51 100644 --- a/src/waitress/wasyncore.py +++ b/src/waitress/wasyncore.py @@ -51,33 +51,31 @@ in the stdlib will be dropped soon. It is neither a copy of the 2.7 asyncore nor the 3.X asyncore; it is a version compatible with either 2.7 or 3.X. """ -from . import compat -from . import utilities - -import logging -import select -import socket -import sys -import time -import warnings - -import os from errno import ( + EAGAIN, EALREADY, - EINPROGRESS, - EWOULDBLOCK, + EBADF, + ECONNABORTED, ECONNRESET, + EINPROGRESS, + EINTR, EINVAL, - ENOTCONN, - ESHUTDOWN, EISCONN, - EBADF, - ECONNABORTED, + ENOTCONN, EPIPE, - EAGAIN, - EINTR, + ESHUTDOWN, + EWOULDBLOCK, errorcode, ) +import logging +import os +import select +import socket +import sys +import time +import warnings + +from . import compat, utilities _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF}) @@ -138,7 +136,7 @@ def readwrite(obj, flags): obj.handle_expt_event() if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() - except socket.error as e: + except OSError as e: if e.args[0] not in _DISCONNECTED: obj.handle_error() else: @@ -172,7 +170,7 @@ def poll(timeout=0.0, map=None): try: r, w, e = select.select(r, w, e, timeout) - except select.error as err: + except OSError as err: if err.args[0] != EINTR: raise else: @@ -218,7 +216,7 @@ def poll2(timeout=0.0, map=None): try: r = pollster.poll(timeout) - except select.error as err: + except OSError as err: if err.args[0] != EINTR: raise r = [] @@ -305,7 +303,7 @@ class dispatcher: # passed be connected. try: self.addr = sock.getpeername() - except socket.error as err: + except OSError as err: if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. @@ -320,7 +318,7 @@ class dispatcher: self.socket = None def __repr__(self): - status = [self.__class__.__module__ + "." + compat.qualname(self.__class__)] + status = [self.__class__.__module__ + "." + self.__class__.__qualname__] if self.accepting and self.addr: status.append("listening") elif self.connected: @@ -368,7 +366,7 @@ class dispatcher: socket.SO_REUSEADDR, self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1, ) - except socket.error: + except OSError: pass # ================================================== @@ -412,7 +410,7 @@ class dispatcher: self.addr = address self.handle_connect_event() else: - raise socket.error(err, errorcode[err]) + raise OSError(err, errorcode[err]) def accept(self): # XXX can return either an address pair or None @@ -420,7 +418,7 @@ class dispatcher: conn, addr = self.socket.accept() except TypeError: return None - except socket.error as why: + except OSError as why: if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: @@ -432,7 +430,7 @@ class dispatcher: try: result = self.socket.send(data) return result - except socket.error as why: + except OSError as why: if why.args[0] == EWOULDBLOCK: return 0 elif why.args[0] in _DISCONNECTED: @@ -451,7 +449,7 @@ class dispatcher: return b"" else: return data - except socket.error as why: + except OSError as why: # winsock sometimes raises ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() @@ -467,7 +465,7 @@ class dispatcher: if self.socket is not None: try: self.socket.close() - except socket.error as why: + except OSError as why: if why.args[0] not in (ENOTCONN, EBADF): raise @@ -501,7 +499,7 @@ class dispatcher: def handle_connect_event(self): err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: - raise socket.error(err, _strerror(err)) + raise OSError(err, _strerror(err)) self.handle_connect() self.connected = True self.connecting = False @@ -608,7 +606,7 @@ def close_all(map=None, ignore_all=False): for x in list(map.values()): # list() FBO py3 try: x.close() - except socket.error as x: + except OSError as x: if x.args[0] == EBADF: pass elif not ignore_all: @@ -646,7 +644,7 @@ if os.name == "posix": def __del__(self): if self.fd >= 0: - warnings.warn("unclosed file %r" % self, compat.ResourceWarning) + warnings.warn("unclosed file %r" % self, ResourceWarning) self.close() def recv(self, *args): @@ -685,7 +683,7 @@ if os.name == "posix": pass self.set_file(fd) # set it to non-blocking mode - compat.set_nonblocking(fd) + os.set_blocking(fd, False) def set_file(self, fd): self.socket = file_wrapper(fd) diff --git a/tests/fixtureapps/filewrapper.py b/tests/fixtureapps/filewrapper.py index 63df5a6..40b7685 100644 --- a/tests/fixtureapps/filewrapper.py +++ b/tests/fixtureapps/filewrapper.py @@ -5,7 +5,7 @@ here = os.path.dirname(os.path.abspath(__file__)) fn = os.path.join(here, "groundhog1.jpg") -class KindaFilelike(object): # pragma: no cover +class KindaFilelike: # pragma: no cover def __init__(self, bytes): self.bytes = bytes diff --git a/tests/fixtureapps/getline.py b/tests/fixtureapps/getline.py index 5e0ad3a..bb5b39c 100644 --- a/tests/fixtureapps/getline.py +++ b/tests/fixtureapps/getline.py @@ -2,9 +2,9 @@ import sys if __name__ == "__main__": try: - from urllib.request import urlopen, URLError + from urllib.request import URLError, urlopen except ImportError: - from urllib2 import urlopen, URLError + from urllib2 import URLError, urlopen url = sys.argv[1] headers = {"Content-Type": "text/plain; charset=utf-8"} diff --git a/tests/fixtureapps/nocl.py b/tests/fixtureapps/nocl.py index f82bba0..c95a4f5 100644 --- a/tests/fixtureapps/nocl.py +++ b/tests/fixtureapps/nocl.py @@ -6,8 +6,7 @@ def chunks(l, n): # pragma: no cover def gen(body): # pragma: no cover - for chunk in chunks(body, 10): - yield chunk + yield from chunks(body, 10) def app(environ, start_response): # pragma: no cover diff --git a/tests/test_adjustments.py b/tests/test_adjustments.py index 303c1aa..420ee4c 100644 --- a/tests/test_adjustments.py +++ b/tests/test_adjustments.py @@ -1,16 +1,9 @@ -import sys import socket +import sys +import unittest import warnings -from waitress.compat import ( - PY2, - WIN, -) - -if sys.version_info[:2] == (2, 6): # pragma: no cover - import unittest2 as unittest -else: # pragma: no cover - import unittest +from waitress.compat import WIN class Test_asbool(unittest.TestCase): @@ -60,10 +53,12 @@ class Test_as_socket_list(unittest.TestCase): socket.socket(socket.AF_INET, socket.SOCK_STREAM), socket.socket(socket.AF_INET6, socket.SOCK_STREAM), ] + if hasattr(socket, "AF_UNIX"): sockets.append(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) new_sockets = as_socket_list(sockets) self.assertEqual(sockets, new_sockets) + for sock in sockets: sock.close() @@ -77,6 +72,7 @@ class Test_as_socket_list(unittest.TestCase): ] new_sockets = as_socket_list(sockets) self.assertEqual(new_sockets, [sockets[0], sockets[1]]) + for sock in [sock for sock in sockets if isinstance(sock, socket.socket)]: sock.close() @@ -99,6 +95,7 @@ class TestAdjustments(unittest.TestCase): return True except socket.gaierror as e: # Check to see what the error is + if e.errno == socket.EAI_ADDRFAMILY: return False else: @@ -220,11 +217,12 @@ class TestAdjustments(unittest.TestCase): self.assertRaises(ValueError, self._makeOne, listen="127.0.0.1:test") def test_service_port(self): - if WIN and PY2: # pragma: no cover - # On Windows and Python 2 this is broken, so we raise a ValueError + if WIN: # pragma: no cover + # On Windows this is broken, so we raise a ValueError self.assertRaises( ValueError, self._makeOne, listen="127.0.0.1:http", ) + return inst = self._makeOne(listen="127.0.0.1:http 0.0.0.0:https") @@ -406,6 +404,9 @@ class TestCLI(unittest.TestCase): return Adjustments.parse_args(argv) + def assertDictContainsSubset(self, subset, dictionary): + self.assertTrue(set(subset.items()) <= set(dictionary.items())) + def test_noargs(self): opts, args = self.parse([]) self.assertDictEqual(opts, {"call": False, "help": False}) diff --git a/tests/test_buffers.py b/tests/test_buffers.py index a1330ac..01cdc2d 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -1,5 +1,5 @@ -import unittest import io +import unittest class TestFileBasedBuffer(unittest.TestCase): @@ -413,7 +413,7 @@ class TestOverflowableBuffer(unittest.TestCase): def test_prune_with_buf(self): inst = self._makeOne() - class Buf(object): + class Buf: def prune(self): self.pruned = True @@ -477,7 +477,7 @@ class TestOverflowableBuffer(unittest.TestCase): self.buffers_to_close.remove(inst) def test_close_withbuf(self): - class Buffer(object): + class Buffer: def close(self): self.closed = True @@ -489,7 +489,7 @@ class TestOverflowableBuffer(unittest.TestCase): self.buffers_to_close.remove(inst) -class KindaFilelike(object): +class KindaFilelike: def __init__(self, bytes, close=None, tellresults=None): self.bytes = bytes self.tellresults = tellresults @@ -506,7 +506,7 @@ class Filelike(KindaFilelike): return v -class DummyBuffer(object): +class DummyBuffer: def __init__(self, length=0): self.length = length diff --git a/tests/test_channel.py b/tests/test_channel.py index 14ef5a0..df3d450 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -1,5 +1,5 @@ -import unittest import io +import unittest class TestHTTPChannel(unittest.TestCase): @@ -29,13 +29,13 @@ class TestHTTPChannel(unittest.TestCase): inst, _, map = self._makeOneWithMap() - class DummyBuffer(object): + class DummyBuffer: chunks = [] def append(self, data): self.chunks.append(data) - class DummyData(object): + class DummyData: def __len__(self): return MAXINT @@ -195,7 +195,7 @@ class TestHTTPChannel(unittest.TestCase): inst.will_close = False def recv(b): - raise socket.error + raise OSError inst.recv = recv inst.last_activity = 0 @@ -270,7 +270,7 @@ class TestHTTPChannel(unittest.TestCase): class Lock(DummyLock): def wait(self): inst.total_outbufs_len = 0 - super(Lock, self).wait() + super().wait() inst.outbuf_lock = Lock() wrote = inst.write_soon(b"xyz") @@ -367,7 +367,7 @@ class TestHTTPChannel(unittest.TestCase): inst, sock, map = self._makeOneWithMap() - class DummyHugeOutbuffer(object): + class DummyHugeOutbuffer: def __init__(self): self.length = MAXINT + 1 @@ -705,7 +705,7 @@ class TestHTTPChannel(unittest.TestCase): self.assertEqual(inst.requests, []) -class DummySock(object): +class DummySock: blocking = False closed = False @@ -732,7 +732,7 @@ class DummySock(object): return len(data) -class DummyLock(object): +class DummyLock: notified = False def __init__(self, acquirable=True): @@ -759,7 +759,7 @@ class DummyLock(object): pass -class DummyBuffer(object): +class DummyBuffer: closed = False def __init__(self, data, toraise=None): @@ -783,7 +783,7 @@ class DummyBuffer(object): self.closed = True -class DummyAdjustments(object): +class DummyAdjustments: outbuf_overflow = 1048576 outbuf_high_watermark = 1048576 inbuf_overflow = 512000 @@ -798,7 +798,7 @@ class DummyAdjustments(object): max_request_header_size = 10000 -class DummyServer(object): +class DummyServer: trigger_pulled = False adj = DummyAdjustments() @@ -813,7 +813,7 @@ class DummyServer(object): self.trigger_pulled = True -class DummyParser(object): +class DummyParser: version = 1 data = None completed = True @@ -831,7 +831,7 @@ class DummyParser(object): return len(data) -class DummyRequest(object): +class DummyRequest: error = None path = "/" version = "1.0" @@ -844,7 +844,7 @@ class DummyRequest(object): self.closed = True -class DummyLogger(object): +class DummyLogger: def __init__(self): self.exceptions = [] self.infos = [] @@ -857,13 +857,13 @@ class DummyLogger(object): self.exceptions.append(msg) -class DummyError(object): +class DummyError: code = "431" reason = "Bleh" body = "My body" -class DummyTaskClass(object): +class DummyTaskClass: wrote_header = True close_on_finish = False serviced = False diff --git a/tests/test_compat.py b/tests/test_compat.py deleted file mode 100644 index 37c2193..0000000 --- a/tests/test_compat.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -import unittest - - -class Test_unquote_bytes_to_wsgi(unittest.TestCase): - def _callFUT(self, v): - from waitress.compat import unquote_bytes_to_wsgi - - return unquote_bytes_to_wsgi(v) - - def test_highorder(self): - from waitress.compat import PY3 - - val = b"/a%C5%9B" - result = self._callFUT(val) - if PY3: # pragma: no cover - # PEP 3333 urlunquoted-latin1-decoded-bytes - self.assertEqual(result, "/aÅ\x9b") - else: # pragma: no cover - # sanity - self.assertEqual(result, b"/a\xc5\x9b") diff --git a/tests/test_functional.py b/tests/test_functional.py index e894497..c99876d 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -1,4 +1,5 @@ import errno +from http import client as httplib import logging import multiprocessing import os @@ -9,8 +10,8 @@ import subprocess import sys import time import unittest + from waitress import server -from waitress.compat import httplib, tobytes from waitress.utilities import cleanup_unix_socket dn = os.path.dirname @@ -54,14 +55,15 @@ class FixtureTcpWSGIServer(server.TcpWSGIServer): def __init__(self, application, queue, **kw): # pragma: no cover # Coverage doesn't see this as it's ran in a separate process. kw["port"] = 0 # Bind to any available port. - super(FixtureTcpWSGIServer, self).__init__(application, **kw) + super().__init__(application, **kw) host, port = self.socket.getsockname() + if os.name == "nt": host = "127.0.0.1" queue.put((host, port)) -class SubprocessTests(object): +class SubprocessTests: # For nose: all tests may be ran in separate processes. _multiprocess_can_split_ = True @@ -98,9 +100,9 @@ class SubprocessTests(object): def assertline(self, line, status, reason, version): v, s, r = (x.strip() for x in line.split(None, 2)) - self.assertEqual(s, tobytes(status)) - self.assertEqual(r, tobytes(reason)) - self.assertEqual(v, tobytes(version)) + self.assertEqual(s, status.encode("latin-1")) + self.assertEqual(r, reason.encode("latin-1")) + self.assertEqual(v, version.encode("latin-1")) def create_socket(self): return socket.socket(self.server.family, socket.SOCK_STREAM) @@ -142,9 +144,11 @@ class SleepyThreadTests(TcpTests, unittest.TestCase): ) r, w = os.pipe() procs = [] + for cmd in cmds: procs.append(subprocess.Popen(cmd, stdout=w)) time.sleep(3) + for proc in procs: if proc.returncode is not None: # pragma: no cover proc.terminate() @@ -158,7 +162,7 @@ class SleepyThreadTests(TcpTests, unittest.TestCase): self.assertEqual(result, b"notsleepy returnedsleepy returned") -class EchoTests(object): +class EchoTests: def setUp(self): from tests.fixtureapps import echo @@ -177,11 +181,11 @@ class EchoTests(object): from tests.fixtureapps import echo line, headers, body = read_http(fp) + return line, headers, echo.parse_response(body) def test_date_and_server(self): - to_send = "GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -192,8 +196,7 @@ class EchoTests(object): def test_bad_host_header(self): # https://corte.si/posts/code/pathod/pythonservers/index.html - to_send = "GET / HTTP/1.0\r\n Host: 0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET / HTTP/1.0\r\n Host: 0\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -203,9 +206,8 @@ class EchoTests(object): self.assertTrue(headers.get("date")) def test_send_with_body(self): - to_send = "GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n" - to_send += "hello" - to_send = tobytes(to_send) + to_send = b"GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n" + to_send += b"hello" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -215,8 +217,7 @@ class EchoTests(object): self.assertEqual(echo.body, b"hello") def test_send_empty_body(self): - to_send = "GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET / HTTP/1.0\r\nContent-Length: 0\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -227,6 +228,7 @@ class EchoTests(object): def test_multiple_requests_with_body(self): orig_sock = self.sock + for x in range(3): self.sock = self.create_socket() self.test_send_with_body() @@ -235,6 +237,7 @@ class EchoTests(object): def test_multiple_requests_without_body(self): orig_sock = self.sock + for x in range(3): self.sock = self.create_socket() self.test_send_empty_body() @@ -242,13 +245,13 @@ class EchoTests(object): self.sock = orig_sock def test_without_crlf(self): - data = "Echo\r\nthis\r\nplease" - s = tobytes( - "GET / HTTP/1.0\r\n" - "Connection: close\r\n" - "Content-Length: %d\r\n" - "\r\n" - "%s" % (len(data), data) + data = b"Echo\r\nthis\r\nplease" + s = ( + b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"%s" % (len(data), data) ) self.connect() self.sock.send(s) @@ -257,40 +260,42 @@ class EchoTests(object): self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(int(echo.content_length), len(data)) self.assertEqual(len(echo.body), len(data)) - self.assertEqual(echo.body, tobytes(data)) + self.assertEqual(echo.body, (data)) def test_large_body(self): # 1024 characters. - body = "This string has 32 characters.\r\n" * 32 - s = tobytes( - "GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(body), body) - ) + body = b"This string has 32 characters.\r\n" * 32 + s = b"GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(body), body) self.connect() self.sock.send(s) fp = self.sock.makefile("rb", 0) line, headers, echo = self._read_echo(fp) self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(echo.content_length, "1024") - self.assertEqual(echo.body, tobytes(body)) + self.assertEqual(echo.body, body) def test_many_clients(self): conns = [] + for n in range(50): h = self.make_http_connection() h.request("GET", "/", headers={"Accept": "text/plain"}) conns.append(h) responses = [] + for h in conns: response = h.getresponse() self.assertEqual(response.status, 200) responses.append(response) + for response in responses: response.read() + for h in conns: h.close() def test_chunking_request_without_content(self): - header = tobytes("GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n") + header = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" self.connect() self.sock.send(header) self.sock.send(b"0\r\n\r\n") @@ -305,10 +310,11 @@ class EchoTests(object): control_line = b"20;\r\n" # 20 hex = 32 dec s = b"This string has 32 characters.\r\n" expected = s * 12 - header = tobytes("GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n") + header = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" self.connect() self.sock.send(header) fp = self.sock.makefile("rb", 0) + for n in range(12): self.sock.send(control_line) self.sock.send(s) @@ -321,13 +327,12 @@ class EchoTests(object): self.assertFalse("transfer-encoding" in headers) def test_broken_chunked_encoding(self): - control_line = "20;\r\n" # 20 hex = 32 dec - s = "This string has 32 characters.\r\n" - to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" - to_send += control_line + s + "\r\n" + control_line = b"20;\r\n" # 20 hex = 32 dec + s = b"This string has 32 characters.\r\n" + to_send = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + to_send += control_line + s + b"\r\n" # garbage in input - to_send += "garbage\r\n" - to_send = tobytes(to_send) + to_send += b"garbage\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -346,13 +351,12 @@ class EchoTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_broken_chunked_encoding_missing_chunk_end(self): - control_line = "20;\r\n" # 20 hex = 32 dec - s = "This string has 32 characters.\r\n" - to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + control_line = b"20;\r\n" # 20 hex = 32 dec + s = b"This string has 32 characters.\r\n" + to_send = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" to_send += control_line + s # garbage in input - to_send += "garbage" - to_send = tobytes(to_send) + to_send += b"garbage" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -373,10 +377,8 @@ class EchoTests(object): def test_keepalive_http_10(self): # Handling of Keep-Alive within HTTP 1.0 - data = "Default: Don't keep me alive" - s = tobytes( - "GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data) - ) + data = b"Default: Don't keep me alive" + s = b"GET / HTTP/1.0\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data) self.connect() self.sock.send(s) response = httplib.HTTPResponse(self.sock) @@ -391,13 +393,13 @@ class EchoTests(object): # If header Connection: Keep-Alive is explicitly sent, # we want to keept the connection open, we also need to return # the corresponding header - data = "Keep me alive" - s = tobytes( - "GET / HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: %d\r\n" - "\r\n" - "%s" % (len(data), data) + data = b"Keep me alive" + s = ( + b"GET / HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"%s" % (len(data), data) ) self.connect() self.sock.send(s) @@ -411,10 +413,8 @@ class EchoTests(object): # Handling of Keep-Alive within HTTP 1.1 # All connections are kept alive, unless stated otherwise - data = "Default: Keep me alive" - s = tobytes( - "GET / HTTP/1.1\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data) - ) + data = b"Default: Keep me alive" + s = b"GET / HTTP/1.1\r\nContent-Length: %d\r\n\r\n%s" % (len(data), data) self.connect() self.sock.send(s) response = httplib.HTTPResponse(self.sock) @@ -424,13 +424,13 @@ class EchoTests(object): def test_keepalive_http11_explicit(self): # Explicitly set keep-alive - data = "Default: Keep me alive" - s = tobytes( - "GET / HTTP/1.1\r\n" - "Connection: keep-alive\r\n" - "Content-Length: %d\r\n" - "\r\n" - "%s" % (len(data), data) + data = b"Default: Keep me alive" + s = ( + b"GET / HTTP/1.1\r\n" + b"Connection: keep-alive\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"%s" % (len(data), data) ) self.connect() self.sock.send(s) @@ -441,13 +441,13 @@ class EchoTests(object): def test_keepalive_http11_connclose(self): # specifying Connection: close explicitly - data = "Don't keep me alive" - s = tobytes( - "GET / HTTP/1.1\r\n" - "Connection: close\r\n" - "Content-Length: %d\r\n" - "\r\n" - "%s" % (len(data), data) + data = b"Don't keep me alive" + s = ( + b"GET / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"%s" % (len(data), data) ) self.connect() self.sock.send(s) @@ -458,14 +458,13 @@ class EchoTests(object): def test_proxy_headers(self): to_send = ( - "GET / HTTP/1.0\r\n" - "Content-Length: 0\r\n" - "Host: www.google.com:8080\r\n" - "X-Forwarded-For: 192.168.1.1\r\n" - "X-Forwarded-Proto: https\r\n" - "X-Forwarded-Port: 5000\r\n\r\n" + b"GET / HTTP/1.0\r\n" + b"Content-Length: 0\r\n" + b"Host: www.google.com:8080\r\n" + b"X-Forwarded-For: 192.168.1.1\r\n" + b"X-Forwarded-Proto: https\r\n" + b"X-Forwarded-Port: 5000\r\n\r\n" ) - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -480,7 +479,7 @@ class EchoTests(object): self.assertEqual(echo.remote_host, "192.168.1.1") -class PipeliningTests(object): +class PipeliningTests: def setUp(self): from tests.fixtureapps import echo @@ -491,27 +490,30 @@ class PipeliningTests(object): def test_pipelining(self): s = ( - "GET / HTTP/1.0\r\n" - "Connection: %s\r\n" - "Content-Length: %d\r\n" - "\r\n" - "%s" + b"GET / HTTP/1.0\r\n" + b"Connection: %s\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"%s" ) to_send = b"" count = 25 + for n in range(count): - body = "Response #%d\r\n" % (n + 1) + body = b"Response #%d\r\n" % (n + 1) + if n + 1 < count: - conn = "keep-alive" + conn = b"keep-alive" else: - conn = "close" - to_send += tobytes(s % (conn, len(body), body)) + conn = b"close" + to_send += s % (conn, len(body), body) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) + for n in range(count): - expect_body = tobytes("Response #%d\r\n" % (n + 1)) + expect_body = b"Response #%d\r\n" % (n + 1) line = fp.readline() # status line version, status, reason = (x.strip() for x in line.split(None, 2)) headers = parse_headers(fp) @@ -522,7 +524,7 @@ class PipeliningTests(object): self.assertEqual(response_body, expect_body) -class ExpectContinueTests(object): +class ExpectContinueTests: def setUp(self): from tests.fixtureapps import echo @@ -533,14 +535,14 @@ class ExpectContinueTests(object): def test_expect_continue(self): # specifying Connection: close explicitly - data = "I have expectations" - to_send = tobytes( - "GET / HTTP/1.1\r\n" - "Connection: close\r\n" - "Content-Length: %d\r\n" - "Expect: 100-continue\r\n" - "\r\n" - "%s" % (len(data), data) + data = b"I have expectations" + to_send = ( + b"GET / HTTP/1.1\r\n" + b"Connection: close\r\n" + b"Content-Length: %d\r\n" + b"Expect: 100-continue\r\n" + b"\r\n" + b"%s" % (len(data), data) ) self.connect() self.sock.send(to_send) @@ -558,10 +560,10 @@ class ExpectContinueTests(object): response_body = fp.read(length) self.assertEqual(int(status), 200) self.assertEqual(length, len(response_body)) - self.assertEqual(response_body, tobytes(data)) + self.assertEqual(response_body, data) -class BadContentLengthTests(object): +class BadContentLengthTests: def setUp(self): from tests.fixtureapps import badcl @@ -573,11 +575,11 @@ class BadContentLengthTests(object): def test_short_body(self): # check to see if server closes connection when body is too short # for cl header - to_send = tobytes( - "GET /short_body HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /short_body HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -590,7 +592,7 @@ class BadContentLengthTests(object): self.assertEqual(int(status), 200) self.assertNotEqual(content_length, len(response_body)) self.assertEqual(len(response_body), content_length - 1) - self.assertEqual(response_body, tobytes("abcdefghi")) + self.assertEqual(response_body, b"abcdefghi") # remote closed connection (despite keepalive header); not sure why # first send succeeds self.send_check_error(to_send) @@ -599,11 +601,11 @@ class BadContentLengthTests(object): def test_long_body(self): # check server doesnt close connection when body is too short # for cl header - to_send = tobytes( - "GET /long_body HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /long_body HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -615,7 +617,7 @@ class BadContentLengthTests(object): response_body = fp.read(content_length) self.assertEqual(int(status), 200) self.assertEqual(content_length, len(response_body)) - self.assertEqual(response_body, tobytes("abcdefgh")) + self.assertEqual(response_body, b"abcdefgh") # remote does not close connection (keepalive header) self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -627,7 +629,7 @@ class BadContentLengthTests(object): self.assertEqual(int(status), 200) -class NoContentLengthTests(object): +class NoContentLengthTests: def setUp(self): from tests.fixtureapps import nocl @@ -637,14 +639,13 @@ class NoContentLengthTests(object): self.stop_subprocess() def test_http10_generator(self): - body = string.ascii_letters + body = string.ascii_letters.encode("latin-1") to_send = ( - "GET / HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: %d\r\n\r\n" % len(body) + b"GET / HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: %d\r\n\r\n" % len(body) ) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -652,21 +653,20 @@ class NoContentLengthTests(object): self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(headers.get("content-length"), None) self.assertEqual(headers.get("connection"), "close") - self.assertEqual(response_body, tobytes(body)) + self.assertEqual(response_body, body) # remote closed connection (despite keepalive header), because # generators cannot have a content-length divined self.send_check_error(to_send) self.assertRaises(ConnectionClosed, read_http, fp) def test_http10_list(self): - body = string.ascii_letters + body = string.ascii_letters.encode("latin-1") to_send = ( - "GET /list HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: %d\r\n\r\n" % len(body) + b"GET /list HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: %d\r\n\r\n" % len(body) ) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -674,7 +674,7 @@ class NoContentLengthTests(object): self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(headers["content-length"], str(len(body))) self.assertEqual(headers.get("connection"), "Keep-Alive") - self.assertEqual(response_body, tobytes(body)) + self.assertEqual(response_body, body) # remote keeps connection open because it divined the content length # from a length-1 list self.sock.send(to_send) @@ -682,14 +682,13 @@ class NoContentLengthTests(object): self.assertline(line, "200", "OK", "HTTP/1.0") def test_http10_listlentwo(self): - body = string.ascii_letters + body = string.ascii_letters.encode("latin-1") to_send = ( - "GET /list_lentwo HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: %d\r\n\r\n" % len(body) + b"GET /list_lentwo HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: %d\r\n\r\n" % len(body) ) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -697,7 +696,7 @@ class NoContentLengthTests(object): self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(headers.get("content-length"), None) self.assertEqual(headers.get("connection"), "close") - self.assertEqual(response_body, tobytes(body)) + self.assertEqual(response_body, body) # remote closed connection (despite keepalive header), because # lists of length > 1 cannot have their content length divined self.send_check_error(to_send) @@ -705,18 +704,20 @@ class NoContentLengthTests(object): def test_http11_generator(self): body = string.ascii_letters - to_send = "GET / HTTP/1.1\r\nContent-Length: %s\r\n\r\n" % len(body) + body = body.encode("latin-1") + to_send = b"GET / HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") line, headers, response_body = read_http(fp) self.assertline(line, "200", "OK", "HTTP/1.1") expected = b"" + for chunk in chunks(body, 10): - expected += tobytes( - "%s\r\n%s\r\n" % (str(hex(len(chunk))[2:].upper()), chunk) + expected += b"%s\r\n%s\r\n" % ( + hex(len(chunk))[2:].upper().encode("latin-1"), + chunk, ) expected += b"0\r\n\r\n" self.assertEqual(response_body, expected) @@ -725,17 +726,16 @@ class NoContentLengthTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_http11_list(self): - body = string.ascii_letters - to_send = "GET /list HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) + body = string.ascii_letters.encode("latin-1") + to_send = b"GET /list HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) line, headers, response_body = read_http(fp) self.assertline(line, "200", "OK", "HTTP/1.1") self.assertEqual(headers["content-length"], str(len(body))) - self.assertEqual(response_body, tobytes(body)) + self.assertEqual(response_body, body) # remote keeps connection open because it divined the content length # from a length-1 list self.sock.send(to_send) @@ -743,19 +743,20 @@ class NoContentLengthTests(object): self.assertline(line, "200", "OK", "HTTP/1.1") def test_http11_listlentwo(self): - body = string.ascii_letters - to_send = "GET /list_lentwo HTTP/1.1\r\nContent-Length: %s\r\n\r\n" % len(body) + body = string.ascii_letters.encode("latin-1") + to_send = b"GET /list_lentwo HTTP/1.1\r\nContent-Length: %d\r\n\r\n" % len(body) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") line, headers, response_body = read_http(fp) self.assertline(line, "200", "OK", "HTTP/1.1") expected = b"" - for chunk in (body[0], body[1:]): - expected += tobytes( - "%s\r\n%s\r\n" % (str(hex(len(chunk))[2:].upper()), chunk) + + for chunk in (body[:1], body[1:]): + expected += b"%s\r\n%s\r\n" % ( + (hex(len(chunk))[2:].upper().encode("latin-1")), + chunk, ) expected += b"0\r\n\r\n" self.assertEqual(response_body, expected) @@ -764,7 +765,7 @@ class NoContentLengthTests(object): self.assertRaises(ConnectionClosed, read_http, fp) -class WriteCallbackTests(object): +class WriteCallbackTests: def setUp(self): from tests.fixtureapps import writecb @@ -776,11 +777,11 @@ class WriteCallbackTests(object): def test_short_body(self): # check to see if server closes connection when body is too short # for cl header - to_send = tobytes( - "GET /short_body HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /short_body HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -792,7 +793,7 @@ class WriteCallbackTests(object): self.assertEqual(cl, 9) self.assertNotEqual(cl, len(response_body)) self.assertEqual(len(response_body), cl - 1) - self.assertEqual(response_body, tobytes("abcdefgh")) + self.assertEqual(response_body, b"abcdefgh") # remote closed connection (despite keepalive header) self.send_check_error(to_send) self.assertRaises(ConnectionClosed, read_http, fp) @@ -800,11 +801,11 @@ class WriteCallbackTests(object): def test_long_body(self): # check server doesnt close connection when body is too long # for cl header - to_send = tobytes( - "GET /long_body HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /long_body HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -813,7 +814,7 @@ class WriteCallbackTests(object): content_length = int(headers.get("content-length")) or None self.assertEqual(content_length, 9) self.assertEqual(content_length, len(response_body)) - self.assertEqual(response_body, tobytes("abcdefghi")) + self.assertEqual(response_body, b"abcdefghi") # remote does not close connection (keepalive header) self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -823,11 +824,11 @@ class WriteCallbackTests(object): def test_equal_body(self): # check server doesnt close connection when body is equal to # cl header - to_send = tobytes( - "GET /equal_body HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /equal_body HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -837,7 +838,7 @@ class WriteCallbackTests(object): self.assertEqual(content_length, 9) self.assertline(line, "200", "OK", "HTTP/1.0") self.assertEqual(content_length, len(response_body)) - self.assertEqual(response_body, tobytes("abcdefghi")) + self.assertEqual(response_body, b"abcdefghi") # remote does not close connection (keepalive header) self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -846,11 +847,11 @@ class WriteCallbackTests(object): def test_no_content_length(self): # wtf happens when there's no content-length - to_send = tobytes( - "GET /no_content_length HTTP/1.0\r\n" - "Connection: Keep-Alive\r\n" - "Content-Length: 0\r\n" - "\r\n" + to_send = ( + b"GET /no_content_length HTTP/1.0\r\n" + b"Connection: Keep-Alive\r\n" + b"Content-Length: 0\r\n" + b"\r\n" ) self.connect() self.sock.send(to_send) @@ -859,13 +860,13 @@ class WriteCallbackTests(object): line, headers, response_body = read_http(fp) content_length = headers.get("content-length") self.assertEqual(content_length, None) - self.assertEqual(response_body, tobytes("abcdefghi")) + self.assertEqual(response_body, b"abcdefghi") # remote closed connection (despite keepalive header) self.send_check_error(to_send) self.assertRaises(ConnectionClosed, read_http, fp) -class TooLargeTests(object): +class TooLargeTests: toobig = 1050 @@ -880,10 +881,9 @@ class TooLargeTests(object): self.stop_subprocess() def test_request_body_too_large_with_wrong_cl_http10(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.0\r\nContent-Length: 5\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") @@ -899,12 +899,11 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_wrong_cl_http10_keepalive(self): - body = "a" * self.toobig + body = b"a" * self.toobig to_send = ( - "GET / HTTP/1.0\r\nContent-Length: 5\r\nConnection: Keep-Alive\r\n\r\n" + b"GET / HTTP/1.0\r\nContent-Length: 5\r\nConnection: Keep-Alive\r\n\r\n" ) to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") @@ -922,10 +921,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_no_cl_http10(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.0\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.0\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -938,10 +936,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_no_cl_http10_keepalive(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -961,10 +958,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_wrong_cl_http11(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") @@ -983,10 +979,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_wrong_cl_http11_connclose(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.1\r\nContent-Length: 5\r\nConnection: close\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.1\r\nContent-Length: 5\r\nConnection: close\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1000,10 +995,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_no_cl_http11(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.1\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.1\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb") @@ -1025,10 +1019,9 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_with_no_cl_http11_connclose(self): - body = "a" * self.toobig - to_send = "GET / HTTP/1.1\r\nConnection: close\r\n\r\n" + body = b"a" * self.toobig + to_send = b"GET / HTTP/1.1\r\nConnection: close\r\n\r\n" to_send += body - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1042,12 +1035,11 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_request_body_too_large_chunked_encoding(self): - control_line = "20;\r\n" # 20 hex = 32 dec - s = "This string has 32 characters.\r\n" - to_send = "GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" + control_line = b"20;\r\n" # 20 hex = 32 dec + s = b"This string has 32 characters.\r\n" + to_send = b"GET / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n" repeat = control_line + s to_send += repeat * ((self.toobig // len(repeat)) + 1) - to_send = tobytes(to_send) self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1062,7 +1054,7 @@ class TooLargeTests(object): self.assertRaises(ConnectionClosed, read_http, fp) -class InternalServerErrorTests(object): +class InternalServerErrorTests: def setUp(self): from tests.fixtureapps import error @@ -1072,8 +1064,7 @@ class InternalServerErrorTests(object): self.stop_subprocess() def test_before_start_response_http_10(self): - to_send = "GET /before_start_response HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /before_start_response HTTP/1.0\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1088,8 +1079,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_before_start_response_http_11(self): - to_send = "GET /before_start_response HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /before_start_response HTTP/1.1\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1107,9 +1097,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_before_start_response_http_11_close(self): - to_send = tobytes( - "GET /before_start_response HTTP/1.1\r\nConnection: close\r\n\r\n" - ) + to_send = b"GET /before_start_response HTTP/1.1\r\nConnection: close\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1128,8 +1116,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_after_start_response_http10(self): - to_send = "GET /after_start_response HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /after_start_response HTTP/1.0\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1148,8 +1135,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_after_start_response_http11(self): - to_send = "GET /after_start_response HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /after_start_response HTTP/1.1\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1167,9 +1153,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_after_start_response_http11_close(self): - to_send = tobytes( - "GET /after_start_response HTTP/1.1\r\nConnection: close\r\n\r\n" - ) + to_send = b"GET /after_start_response HTTP/1.1\r\nConnection: close\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1188,8 +1172,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_after_write_cb(self): - to_send = "GET /after_write_cb HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /after_write_cb HTTP/1.1\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1201,8 +1184,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_in_generator(self): - to_send = "GET /in_generator HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /in_generator HTTP/1.1\r\n\r\n" self.connect() self.sock.send(to_send) fp = self.sock.makefile("rb", 0) @@ -1214,7 +1196,7 @@ class InternalServerErrorTests(object): self.assertRaises(ConnectionClosed, read_http, fp) -class FileWrapperTests(object): +class FileWrapperTests: def setUp(self): from tests.fixtureapps import filewrapper @@ -1224,8 +1206,7 @@ class FileWrapperTests(object): self.stop_subprocess() def test_filelike_http11(self): - to_send = "GET /filelike HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike HTTP/1.1\r\n\r\n" self.connect() @@ -1242,8 +1223,7 @@ class FileWrapperTests(object): # connection has not been closed def test_filelike_nocl_http11(self): - to_send = "GET /filelike_nocl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike_nocl HTTP/1.1\r\n\r\n" self.connect() @@ -1260,8 +1240,7 @@ class FileWrapperTests(object): # connection has not been closed def test_filelike_shortcl_http11(self): - to_send = "GET /filelike_shortcl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike_shortcl HTTP/1.1\r\n\r\n" self.connect() @@ -1279,8 +1258,7 @@ class FileWrapperTests(object): # connection has not been closed def test_filelike_longcl_http11(self): - to_send = "GET /filelike_longcl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike_longcl HTTP/1.1\r\n\r\n" self.connect() @@ -1297,8 +1275,7 @@ class FileWrapperTests(object): # connection has not been closed def test_notfilelike_http11(self): - to_send = "GET /notfilelike HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike HTTP/1.1\r\n\r\n" self.connect() @@ -1315,8 +1292,7 @@ class FileWrapperTests(object): # connection has not been closed def test_notfilelike_iobase_http11(self): - to_send = "GET /notfilelike_iobase HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike_iobase HTTP/1.1\r\n\r\n" self.connect() @@ -1333,8 +1309,7 @@ class FileWrapperTests(object): # connection has not been closed def test_notfilelike_nocl_http11(self): - to_send = "GET /notfilelike_nocl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike_nocl HTTP/1.1\r\n\r\n" self.connect() @@ -1350,8 +1325,7 @@ class FileWrapperTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_notfilelike_shortcl_http11(self): - to_send = "GET /notfilelike_shortcl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike_shortcl HTTP/1.1\r\n\r\n" self.connect() @@ -1369,8 +1343,7 @@ class FileWrapperTests(object): # connection has not been closed def test_notfilelike_longcl_http11(self): - to_send = "GET /notfilelike_longcl HTTP/1.1\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike_longcl HTTP/1.1\r\n\r\n" self.connect() @@ -1388,8 +1361,7 @@ class FileWrapperTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_filelike_http10(self): - to_send = "GET /filelike HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike HTTP/1.0\r\n\r\n" self.connect() @@ -1407,8 +1379,7 @@ class FileWrapperTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_filelike_nocl_http10(self): - to_send = "GET /filelike_nocl HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /filelike_nocl HTTP/1.0\r\n\r\n" self.connect() @@ -1426,8 +1397,7 @@ class FileWrapperTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_notfilelike_http10(self): - to_send = "GET /notfilelike HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike HTTP/1.0\r\n\r\n" self.connect() @@ -1445,8 +1415,7 @@ class FileWrapperTests(object): self.assertRaises(ConnectionClosed, read_http, fp) def test_notfilelike_nocl_http10(self): - to_send = "GET /notfilelike_nocl HTTP/1.0\r\n\r\n" - to_send = tobytes(to_send) + to_send = b"GET /notfilelike_nocl HTTP/1.0\r\n\r\n" self.connect() @@ -1512,7 +1481,7 @@ if hasattr(socket, "AF_UNIX"): # Coverage doesn't see this as it's ran in a separate process. # To permit parallel testing, use a PID-dependent socket. kw["unix_socket"] = "/tmp/waitress.test-%d.sock" % os.getpid() - super(FixtureUnixWSGIServer, self).__init__(application, **kw) + super().__init__(application, **kw) queue.put(self.socket.getsockname()) class UnixTests(SubprocessTests): @@ -1523,7 +1492,7 @@ if hasattr(socket, "AF_UNIX"): return UnixHTTPConnection(self.bound_to) def stop_subprocess(self): - super(UnixTests, self).stop_subprocess() + super().stop_subprocess() cleanup_unix_socket(self.bound_to) def send_check_error(self, to_send): @@ -1531,8 +1500,9 @@ if hasattr(socket, "AF_UNIX"): # 'Broken pipe' error when the socket it closed. try: self.sock.send(to_send) - except socket.error as exc: - self.assertEqual(get_errno(exc), errno.EPIPE) + except OSError as exc: + valid_errors = {errno.EPIPE, errno.ENOTCONN} + self.assertIn(get_errno(exc), valid_errors) class UnixEchoTests(EchoTests, UnixTests, unittest.TestCase): pass @@ -1570,13 +1540,16 @@ def parse_headers(fp): """Parses only RFC2822 headers from a file pointer. """ headers = {} + while True: line = fp.readline() + if line in (b"\r\n", b"\n", b""): break line = line.decode("iso-8859-1") name, value = line.strip().split(":", 1) headers[name.lower().strip()] = value.lower().strip() + return headers @@ -1602,25 +1575,31 @@ class ConnectionClosed(Exception): def read_http(fp): # pragma: no cover try: response_line = fp.readline() - except socket.error as exc: + except OSError as exc: fp.close() # errno 104 is ENOTRECOVERABLE, In WinSock 10054 is ECONNRESET + if get_errno(exc) in (errno.ECONNABORTED, errno.ECONNRESET, 104, 10054): raise ConnectionClosed raise + if not response_line: raise ConnectionClosed header_lines = [] + while True: line = fp.readline() + if line in (b"\r\n", b"\r\n", b""): break else: header_lines.append(line) headers = dict() + for x in header_lines: x = x.strip() + if not x: continue key, value = x.split(b": ", 1) @@ -1633,8 +1612,10 @@ def read_http(fp): # pragma: no cover num = int(headers["content-length"]) body = b"" left = num + while left > 0: data = fp.read(left) + if not data: break body += data @@ -1670,5 +1651,6 @@ def get_errno(exc): # pragma: no cover def chunks(l, n): """ Yield successive n-sized chunks from l. """ + for i in range(0, len(l), n): yield l[i : i + n] diff --git a/tests/test_init.py b/tests/test_init.py index f9b91d7..c824c21 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -31,7 +31,7 @@ class Test_serve_paste(unittest.TestCase): self.assertEqual(server.ran, True) -class DummyServerFactory(object): +class DummyServerFactory: ran = False def __call__(self, app, **kw): @@ -44,7 +44,7 @@ class DummyServerFactory(object): self.ran = True -class DummyAdj(object): +class DummyAdj: verbose = False def __init__(self, kw): diff --git a/tests/test_parser.py b/tests/test_parser.py index 91837c7..eace4af 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -15,13 +15,26 @@ """ import unittest -from waitress.compat import text_, tobytes +from waitress.adjustments import Adjustments +from waitress.parser import ( + HTTPRequestParser, + ParsingError, + TransferEncodingNotImplemented, + crack_first_line, + get_header_lines, + split_uri, + unquote_bytes_to_wsgi, +) +from waitress.utilities import ( + BadRequest, + RequestEntityTooLarge, + RequestHeaderFieldsTooLarge, + ServerNotImplemented, +) class TestHTTPRequestParser(unittest.TestCase): def setUp(self): - from waitress.parser import HTTPRequestParser - from waitress.adjustments import Adjustments my_adj = Adjustments() self.parser = HTTPRequestParser(my_adj) @@ -45,8 +58,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers, {}) def test_received_bad_host_header(self): - from waitress.utilities import BadRequest - data = b"HTTP/1.0 GET /foobar\r\n Host: foo\r\n\r\n" result = self.parser.received(data) self.assertEqual(result, 36) @@ -54,8 +65,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.error.__class__, BadRequest) def test_received_bad_transfer_encoding(self): - from waitress.utilities import ServerNotImplemented - data = ( b"GET /foobar HTTP/1.1\r\n" b"Transfer-Encoding: foo\r\n" @@ -89,7 +98,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(result, 0) def test_received_cl_too_large(self): - from waitress.utilities import RequestEntityTooLarge self.parser.adj.max_request_body_size = 2 data = b"GET /foobar HTTP/8.4\r\nContent-Length: 10\r\n\r\n" @@ -99,7 +107,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(isinstance(self.parser.error, RequestEntityTooLarge)) def test_received_headers_too_large(self): - from waitress.utilities import RequestHeaderFieldsTooLarge self.parser.adj.max_request_header_size = 2 data = b"GET /foobar HTTP/8.4\r\nX-Foo: 1\r\n\r\n" @@ -109,8 +116,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(isinstance(self.parser.error, RequestHeaderFieldsTooLarge)) def test_received_body_too_large(self): - from waitress.utilities import RequestEntityTooLarge - self.parser.adj.max_request_body_size = 2 data = ( b"GET /foobar HTTP/1.1\r\n" @@ -129,8 +134,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(isinstance(self.parser.error, RequestEntityTooLarge)) def test_received_error_from_parser(self): - from waitress.utilities import BadRequest - data = ( b"GET /foobar HTTP/1.1\r\n" b"Transfer-Encoding: chunked\r\n" @@ -171,8 +174,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers["FOO"], "bar") def test_parse_header_no_cr_in_headerplus(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4" try: @@ -183,8 +184,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_bad_content_length(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\r\ncontent-length: abc\r\n" try: @@ -195,8 +194,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_multiple_content_length(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\r\ncontent-length: 10\r\ncontent-length: 20\r\n" try: @@ -213,8 +210,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.body_rcv.__class__.__name__, "ChunkedReceiver") def test_parse_header_transfer_encoding_invalid(self): - from waitress.parser import TransferEncodingNotImplemented - data = b"GET /foobar HTTP/1.1\r\ntransfer-encoding: gzip\r\n" try: @@ -225,7 +220,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_transfer_encoding_invalid_multiple(self): - from waitress.parser import TransferEncodingNotImplemented data = b"GET /foobar HTTP/1.1\r\ntransfer-encoding: gzip\r\ntransfer-encoding: chunked\r\n" @@ -237,8 +231,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_transfer_encoding_invalid_whitespace(self): - from waitress.parser import TransferEncodingNotImplemented - data = b"GET /foobar HTTP/1.1\r\nTransfer-Encoding:\x85chunked\r\n" try: @@ -249,8 +241,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_transfer_encoding_invalid_unicode(self): - from waitress.parser import TransferEncodingNotImplemented - # This is the binary encoding for the UTF-8 character # https://www.compart.com/en/unicode/U+212A "unicode character "K"" # which if waitress were to accidentally do the wrong thing get @@ -286,8 +276,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.parser.close() # doesn't raise def test_parse_header_lf_only(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\nfoo: bar" try: @@ -298,8 +286,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_cr_only(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\rfoo: bar" try: self.parser.parse_header(data) @@ -309,8 +295,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_extra_lf_in_header(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\r\nfoo: \nbar\r\n" try: self.parser.parse_header(data) @@ -320,8 +304,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_extra_lf_in_first_line(self): - from waitress.parser import ParsingError - data = b"GET /foobar\n HTTP/8.4\r\n" try: self.parser.parse_header(data) @@ -331,8 +313,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_invalid_whitespace(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/8.4\r\nfoo : bar\r\n" try: self.parser.parse_header(data) @@ -342,8 +322,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_invalid_whitespace_vtab(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo:\x0bbar\r\n" try: self.parser.parse_header(data) @@ -353,8 +331,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_invalid_no_colon(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar\r\nnotvalid\r\n" try: self.parser.parse_header(data) @@ -364,8 +340,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_invalid_folding_spacing(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar\r\n\t\x0bbaz\r\n" try: self.parser.parse_header(data) @@ -375,8 +349,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_invalid_chars(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar\r\nfoo: \x0bbaz\r\n" try: self.parser.parse_header(data) @@ -386,8 +358,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_empty(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar\r\nempty:\r\n" self.parser.parse_header(data) @@ -397,8 +367,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers["FOO"], "bar") def test_parse_header_multiple_values(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar, whatever, more, please, yes\r\n" self.parser.parse_header(data) @@ -406,8 +374,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers["FOO"], "bar, whatever, more, please, yes") def test_parse_header_multiple_values_header_folded(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar, whatever,\r\n more, please, yes\r\n" self.parser.parse_header(data) @@ -415,8 +381,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers["FOO"], "bar, whatever, more, please, yes") def test_parse_header_multiple_values_header_folded_multiple(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar, whatever,\r\n more\r\nfoo: please, yes\r\n" self.parser.parse_header(data) @@ -425,8 +389,6 @@ class TestHTTPRequestParser(unittest.TestCase): def test_parse_header_multiple_values_extra_space(self): # Tests errata from: https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: abrowser/0.001 (C O M M E N T)\r\n" self.parser.parse_header(data) @@ -434,8 +396,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertEqual(self.parser.headers["FOO"], "abrowser/0.001 (C O M M E N T)") def test_parse_header_invalid_backtrack_bad(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\nfoo: bar\r\nfoo: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\x10\r\n" try: self.parser.parse_header(data) @@ -445,8 +405,6 @@ class TestHTTPRequestParser(unittest.TestCase): self.assertTrue(False) def test_parse_header_short_values(self): - from waitress.parser import ParsingError - data = b"GET /foobar HTTP/1.1\r\none: 1\r\ntwo: 22\r\n" self.parser.parse_header(data) @@ -458,8 +416,6 @@ class TestHTTPRequestParser(unittest.TestCase): class Test_split_uri(unittest.TestCase): def _callFUT(self, uri): - from waitress.parser import split_uri - ( self.proxy_scheme, self.proxy_netloc, @@ -499,7 +455,6 @@ class Test_split_uri(unittest.TestCase): def test_split_uri_unicode_error_raises_parsing_error(self): # See https://github.com/Pylons/waitress/issues/64 - from waitress.parser import ParsingError # Either pass or throw a ParsingError, just don't throw another type of # exception as that will cause the connection to close badly: @@ -535,8 +490,6 @@ class Test_split_uri(unittest.TestCase): class Test_get_header_lines(unittest.TestCase): def _callFUT(self, data): - from waitress.parser import get_header_lines - return get_header_lines(data) def test_get_header_lines(self): @@ -561,15 +514,11 @@ class Test_get_header_lines(unittest.TestCase): def test_get_header_lines_malformed(self): # https://corte.si/posts/code/pathod/pythonservers/index.html - from waitress.parser import ParsingError - self.assertRaises(ParsingError, self._callFUT, b" Host: localhost\r\n\r\n") class Test_crack_first_line(unittest.TestCase): def _callFUT(self, line): - from waitress.parser import crack_first_line - return crack_first_line(line) def test_crack_first_line_matchok(self): @@ -577,8 +526,6 @@ class Test_crack_first_line(unittest.TestCase): self.assertEqual(result, (b"GET", b"/", b"1.0")) def test_crack_first_line_lowercase_method(self): - from waitress.parser import ParsingError - self.assertRaises(ParsingError, self._callFUT, b"get / HTTP/1.0") def test_crack_first_line_nomatch(self): @@ -595,9 +542,6 @@ class Test_crack_first_line(unittest.TestCase): class TestHTTPRequestParserIntegration(unittest.TestCase): def setUp(self): - from waitress.parser import HTTPRequestParser - from waitress.adjustments import Adjustments - my_adj = Adjustments() self.parser = HTTPRequestParser(my_adj) @@ -657,8 +601,8 @@ class TestHTTPRequestParserIntegration(unittest.TestCase): ) # path should be utf-8 encoded self.assertEqual( - tobytes(parser.path).decode("utf-8"), - text_(b"/foo/a++/\xc3\xa4=&a:int", "utf-8"), + parser.path.encode("latin-1").decode("utf-8"), + b"/foo/a++/\xc3\xa4=&a:int".decode("utf-8"), ) self.assertEqual( parser.query, "d=b+%2B%2F%3D%26b%3Aint&c+%2B%2F%3D%26c%3Aint=6" @@ -721,7 +665,19 @@ class TestHTTPRequestParserIntegration(unittest.TestCase): self.assertEqual(self.parser.headers, {"CONTENT_LENGTH": "6",}) -class DummyBodyStream(object): +class Test_unquote_bytes_to_wsgi(unittest.TestCase): + def _callFUT(self, v): + + return unquote_bytes_to_wsgi(v) + + def test_highorder(self): + val = b"/a%C5%9B" + result = self._callFUT(val) + # PEP 3333 urlunquoted-latin1-decoded-bytes + self.assertEqual(result, "/aÅ\x9b") + + +class DummyBodyStream: def getfile(self): return self diff --git a/tests/test_proxy_headers.py b/tests/test_proxy_headers.py index 15b4a08..e6f0ed6 100644 --- a/tests/test_proxy_headers.py +++ b/tests/test_proxy_headers.py @@ -1,7 +1,5 @@ import unittest -from waitress.compat import tobytes - class TestProxyHeadersMiddleware(unittest.TestCase): def _makeOne(self, app, **kw): @@ -18,7 +16,7 @@ class TestProxyHeadersMiddleware(unittest.TestCase): response.headers = response_headers response.steps = list(app(environ, start_response)) - response.body = b"".join(tobytes(s) for s in response.steps) + response.body = b"".join(s.encode("latin-1") for s in response.steps) return response def test_get_environment_values_w_scheme_override_untrusted(self): @@ -681,7 +679,7 @@ class TestProxyHeadersMiddleware(unittest.TestCase): self.assertIn(b'Header "X-Forwarded-Host" malformed', response.body) -class DummyLogger(object): +class DummyLogger: def __init__(self): self.logged = [] @@ -689,14 +687,14 @@ class DummyLogger(object): self.logged.append(msg % args) -class DummyApp(object): +class DummyApp: def __call__(self, environ, start_response): self.environ = environ start_response("200 OK", [("Content-Type", "text/plain")]) yield "hello" -class DummyResponse(object): +class DummyResponse: status = None headers = None body = None diff --git a/tests/test_receiver.py b/tests/test_receiver.py index b4910bb..f55aa68 100644 --- a/tests/test_receiver.py +++ b/tests/test_receiver.py @@ -226,7 +226,7 @@ class TestChunkedReceiver(unittest.TestCase): self.assertEqual(inst.error, None) -class DummyBuffer(object): +class DummyBuffer: def __init__(self, data=None): if data is None: data = [] diff --git a/tests/test_runner.py b/tests/test_runner.py index e53018b..4cf6f6f 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -12,17 +12,17 @@ from waitress import runner class Test_match(unittest.TestCase): def test_empty(self): - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, "^Malformed application ''$", runner.match, "" ) def test_module_only(self): - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, r"^Malformed application 'foo\.bar'$", runner.match, "foo.bar" ) def test_bad_module(self): - self.assertRaisesRegexp( + self.assertRaisesRegex( ValueError, r"^Malformed application 'foo#bar:barney'$", runner.match, @@ -42,7 +42,7 @@ class Test_resolve(unittest.TestCase): ) def test_nonexistent_function(self): - self.assertRaisesRegexp( + self.assertRaisesRegex( AttributeError, r"has no attribute 'nonexistent_function'", runner.resolve, @@ -57,7 +57,7 @@ class Test_resolve(unittest.TestCase): def test_complex_happy_path(self): # Ensure we can recursively resolve object attributes if necessary. - self.assertEquals(runner.resolve("os.path", "exists.__name__"), "exists") + self.assertEqual(runner.resolve("os.path", "exists.__name__"), "exists") class Test_run(unittest.TestCase): @@ -65,7 +65,7 @@ class Test_run(unittest.TestCase): argv = ["waitress-serve"] + argv with capture() as captured: self.assertEqual(runner.run(argv=argv), code) - self.assertRegexpMatches(captured.getvalue(), regex) + self.assertRegex(captured.getvalue(), regex) captured.close() def test_bad(self): @@ -119,7 +119,7 @@ class Test_run(unittest.TestCase): ) def test_simple_call(self): - import tests.fixtureapps.runner as _apps + from tests.fixtureapps import runner as _apps def check_server(app, **kw): self.assertIs(app, _apps.app) @@ -133,7 +133,7 @@ class Test_run(unittest.TestCase): self.assertEqual(runner.run(argv=argv, _serve=check_server), 0) def test_returned_app(self): - import tests.fixtureapps.runner as _apps + from tests.fixtureapps import runner as _apps def check_server(app, **kw): self.assertIs(app, _apps.app) @@ -162,7 +162,7 @@ class Test_helper(unittest.TestCase): raise ImportError("My reason") except ImportError: self.assertEqual(show_exception(sys.stderr), None) - self.assertRegexpMatches(captured.getvalue(), regex) + self.assertRegex(captured.getvalue(), regex) captured.close() regex = ( @@ -175,15 +175,15 @@ class Test_helper(unittest.TestCase): raise ImportError except ImportError: self.assertEqual(show_exception(sys.stderr), None) - self.assertRegexpMatches(captured.getvalue(), regex) + self.assertRegex(captured.getvalue(), regex) captured.close() @contextlib.contextmanager def capture(): - from waitress.compat import NativeIO + from io import StringIO - fd = NativeIO() + fd = StringIO() sys.stdout = fd sys.stderr = fd yield fd diff --git a/tests/test_server.py b/tests/test_server.py index 9134fb8..05f6b4e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -240,7 +240,7 @@ class TestWSGIServer(unittest.TestCase): inst.adj = DummyAdj def foo(): - raise socket.error + raise OSError inst.accept = foo inst.logger = DummyLogger() @@ -263,7 +263,7 @@ class TestWSGIServer(unittest.TestCase): def test_maintenance(self): inst = self._makeOneWithMap() - class DummyChannel(object): + class DummyChannel: requests = [] zombie = DummyChannel() @@ -274,8 +274,8 @@ class TestWSGIServer(unittest.TestCase): self.assertEqual(zombie.will_close, True) def test_backward_compatibility(self): - from waitress.server import WSGIServer, TcpWSGIServer from waitress.adjustments import Adjustments + from waitress.server import TcpWSGIServer, WSGIServer self.assertTrue(WSGIServer is TcpWSGIServer) self.inst = WSGIServer(None, _start=False, port=1234) @@ -411,8 +411,8 @@ if hasattr(socket, "AF_UNIX"): def test_create_with_unix_socket(self): from waitress.server import ( - MultiSocketServer, BaseWSGIServer, + MultiSocketServer, TcpWSGIServer, UnixWSGIServer, ) @@ -479,7 +479,7 @@ class DummySock(socket.socket): pass -class DummyTaskDispatcher(object): +class DummyTaskDispatcher: def __init__(self): self.tasks = [] @@ -490,7 +490,7 @@ class DummyTaskDispatcher(object): self.was_shutdown = True -class DummyTask(object): +class DummyTask: serviced = False start_response_called = False wrote_header = False @@ -512,12 +512,12 @@ class DummyAdj: channel_timeout = 300 -class DummyAsyncore(object): +class DummyAsyncore: def loop(self, timeout=30.0, use_poll=False, map=None, count=None): raise SystemExit -class DummyTrigger(object): +class DummyTrigger: def pull_trigger(self): self.pulled = True @@ -525,7 +525,7 @@ class DummyTrigger(object): pass -class DummyLogger(object): +class DummyLogger: def __init__(self): self.logged = [] diff --git a/tests/test_task.py b/tests/test_task.py index 6466823..0965bf5 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,5 +1,5 @@ -import unittest import io +import unittest class TestThreadedTaskDispatcher(unittest.TestCase): @@ -15,7 +15,7 @@ class TestThreadedTaskDispatcher(unittest.TestCase): class BadDummyTask(DummyTask): def service(self): - super(BadDummyTask, self).service() + super().service() inst.stop_count += 1 raise Exception @@ -400,7 +400,7 @@ class TestWSGITask(unittest.TestCase): inst = self._makeOne() def execute(): - raise socket.error + raise OSError inst.execute = execute self.assertRaises(socket.error, inst.service) @@ -922,7 +922,7 @@ class TestErrorTask(unittest.TestCase): self.assertEqual(lines[8], b"(generated by waitress)") -class DummyTask(object): +class DummyTask: serviced = False cancelled = False @@ -933,7 +933,7 @@ class DummyTask(object): self.cancelled = True -class DummyAdj(object): +class DummyAdj: log_socket_errors = True ident = "waitress" host = "127.0.0.1" @@ -941,7 +941,7 @@ class DummyAdj(object): url_prefix = "" -class DummyServer(object): +class DummyServer: server_name = "localhost" effective_port = 80 @@ -949,7 +949,7 @@ class DummyServer(object): self.adj = DummyAdj() -class DummyChannel(object): +class DummyChannel: closed_when_done = False adj = DummyAdj() creation_time = 0 @@ -970,7 +970,7 @@ class DummyChannel(object): return len(data) -class DummyParser(object): +class DummyParser: version = "1.0" command = "GET" path = "/" @@ -990,7 +990,7 @@ def filter_lines(s): return list(filter(None, s.split(b"\r\n"))) -class DummyLogger(object): +class DummyLogger: def __init__(self): self.logged = [] diff --git a/tests/test_trigger.py b/tests/test_trigger.py index af740f6..265679a 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,6 +1,6 @@ -import unittest import os import sys +import unittest if not sys.platform.startswith("win"): diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 15cd24f..ea08477 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -39,16 +39,17 @@ class Test_parse_http_date(unittest.TestCase): class Test_build_http_date(unittest.TestCase): def test_rountdrip(self): - from waitress.utilities import build_http_date, parse_http_date from time import time + from waitress.utilities import build_http_date, parse_http_date + t = int(time()) self.assertEqual(t, parse_http_date(build_http_date(t))) class Test_unpack_rfc850(unittest.TestCase): def _callFUT(self, val): - from waitress.utilities import unpack_rfc850, rfc850_reg + from waitress.utilities import rfc850_reg, unpack_rfc850 return unpack_rfc850(rfc850_reg.match(val.lower())) @@ -60,7 +61,7 @@ class Test_unpack_rfc850(unittest.TestCase): class Test_unpack_rfc_822(unittest.TestCase): def _callFUT(self, val): - from waitress.utilities import unpack_rfc822, rfc822_reg + from waitress.utilities import rfc822_reg, unpack_rfc822 return unpack_rfc822(rfc822_reg.match(val.lower())) diff --git a/tests/test_wasyncore.py b/tests/test_wasyncore.py index 9c23509..970e993 100644 --- a/tests/test_wasyncore.py +++ b/tests/test_wasyncore.py @@ -1,21 +1,21 @@ -from waitress import wasyncore as asyncore -from waitress import compat +import _thread as thread import contextlib +import errno import functools import gc -import unittest -import select +from io import BytesIO import os -import socket -import sys -import time -import errno import re +import select +import socket import struct +import sys import threading +import time +import unittest import warnings -from io import BytesIO +from waitress import compat, wasyncore as asyncore TIMEOUT = 3 HAS_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") @@ -24,6 +24,7 @@ HOSTv4 = "127.0.0.1" HOSTv6 = "::1" # Filename used for testing + if os.name == "java": # pragma: no cover # Jython disallows @ in module names TESTFN = "$test" @@ -33,7 +34,7 @@ else: TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid()) -class DummyLogger(object): # pragma: no cover +class DummyLogger: # pragma: no cover def __init__(self): self.messages = [] @@ -41,7 +42,7 @@ class DummyLogger(object): # pragma: no cover self.messages.append((severity, message)) -class WarningsRecorder(object): # pragma: no cover +class WarningsRecorder: # pragma: no cover """Convenience wrapper for the warnings list returned on entry to the warnings.catch_warnings() context manager. """ @@ -67,6 +68,7 @@ def _filterwarnings(filters, quiet=False): # pragma: no cover # in order to re-raise the warnings. frame = sys._getframe(2) registry = frame.f_globals.get("__warningregistry__") + if registry: registry.clear() with warnings.catch_warnings(record=True) as w: @@ -78,19 +80,25 @@ def _filterwarnings(filters, quiet=False): # pragma: no cover # Filter the recorded warnings reraise = list(w) missing = [] + for msg, cat in filters: seen = False + for w in reraise[:]: warning = w.message # Filter out the matching messages + if re.match(msg, str(warning), re.I) and issubclass(warning.__class__, cat): seen = True reraise.remove(w) + if not seen and not quiet: # This filter caught nothing missing.append((msg, cat.__name__)) + if reraise: raise AssertionError("unhandled warning %s" % reraise[0]) + if missing: raise AssertionError("filter (%r, %s) did not catch any warning" % missing[0]) @@ -111,11 +119,14 @@ def check_warnings(*filters, **kwargs): # pragma: no cover check_warnings(("", Warning), quiet=True) """ quiet = kwargs.get("quiet") + if not filters: filters = (("", Warning),) # Preserve backward compatibility + if quiet is None: quiet = True + return _filterwarnings(filters, quiet) @@ -130,6 +141,7 @@ def gc_collect(): # pragma: no cover objects to disappear. """ gc.collect() + if sys.platform.startswith("java"): time.sleep(0.1) gc.collect() @@ -137,7 +149,7 @@ def gc_collect(): # pragma: no cover def threading_setup(): # pragma: no cover - return (compat.thread._count(), None) + return (thread._count(), None) def threading_cleanup(*original_values): # pragma: no cover @@ -146,7 +158,8 @@ def threading_cleanup(*original_values): # pragma: no cover _MAX_COUNT = 100 for count in range(_MAX_COUNT): - values = (compat.thread._count(), None) + values = (thread._count(), None) + if values == original_values: break @@ -186,6 +199,7 @@ def join_thread(thread, timeout=30.0): # pragma: no cover after timeout seconds. """ thread.join(timeout) + if thread.is_alive(): msg = "failed to join the thread in %.1f seconds" % timeout raise AssertionError(msg) @@ -213,6 +227,7 @@ def bind_port(sock, host=HOST): # pragma: no cover "tests should never set the SO_REUSEADDR " "socket option on TCP/IP sockets!" ) + if hasattr(socket, "SO_REUSEPORT"): try: if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: @@ -225,11 +240,13 @@ def bind_port(sock, host=HOST): # pragma: no cover # thus defining SO_REUSEPORT but this process is running # under an older kernel that does not support SO_REUSEPORT. pass + if hasattr(socket, "SO_EXCLUSIVEADDRUSE"): sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) sock.bind((host, 0)) port = sock.getsockname()[1] + return port @@ -303,13 +320,16 @@ def capture_server(evt, buf, serv): # pragma no cover else: n = 200 start = time.time() + while n > 0 and time.time() - start < 3.0: r, w, e = select.select([conn], [], [], 0.1) + if r: n -= 1 data = conn.recv(10) # keep everything except for the newline terminator buf.write(data.replace(b"\n", b"")) + if b"\n" in data: break time.sleep(0.01) @@ -332,6 +352,7 @@ def bind_unix_socket(sock, addr): # pragma: no cover def bind_af_aware(sock, addr): """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: # Make sure the path doesn't exist. unlink(addr) @@ -346,6 +367,7 @@ if sys.platform.startswith("win"): # pragma: no cover # Perform the operation func(pathname) # Now setup the wait loop + if waitall: dirname = pathname else: @@ -358,6 +380,7 @@ if sys.platform.startswith("win"): # pragma: no cover # Testing on an i7@4.3GHz shows that usually only 1 iteration is # required when contention occurs. timeout = 0.001 + while timeout < 1.0: # Note we are only testing for the existence of the file(s) in # the contents of the directory regardless of any security or @@ -367,6 +390,7 @@ if sys.platform.startswith("win"): # pragma: no cover # Other Windows APIs can fail or give incorrect results when # dealing with files that are pending deletion. L = os.listdir(dirname) + if not (L if waitall else name in L): return # Increase the timeout and try again @@ -395,17 +419,20 @@ def unlink(filename): def _is_ipv6_enabled(): # pragma: no cover """Check whether IPv6 is enabled on this host.""" + if compat.HAS_IPV6: sock = None try: sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) sock.bind(("::1", 0)) + return True - except socket.error: + except OSError: pass finally: if sock: sock.close() + return False @@ -487,6 +514,7 @@ class HelperFunctionTests(unittest.TestCase): # Only the attribute modified by the routine we expect to be # called should be True. + for attr in attributes: self.assertEqual(getattr(tobj, attr), attr == expectedattr) @@ -513,6 +541,7 @@ class HelperFunctionTests(unittest.TestCase): l = [] testmap = {} + for i in range(10): c = dummychannel() l.append(c) @@ -606,6 +635,7 @@ class DispatcherTests(unittest.TestCase): def test_strerror(self): # refers to bug #8573 err = asyncore._strerror(errno.EPERM) + if hasattr(os, "strerror"): self.assertEqual(err, os.strerror(errno.EPERM)) err = asyncore._strerror(-1) @@ -656,6 +686,7 @@ class DispatcherWithSendTests(unittest.TestCase): d.send(b"\n") n = 1000 + while d.out_buffer and n > 0: # pragma: no cover asyncore.poll() n -= 1 @@ -723,6 +754,7 @@ class FileWrapperTest(unittest.TestCase): def test_resource_warning(self): # Issue #11453 got_warning = False + while got_warning is False: # we try until we get the outcome we want because this # test is not deterministic (gc_collect() may not @@ -732,7 +764,7 @@ class FileWrapperTest(unittest.TestCase): os.close(fd) try: - with check_warnings(("", compat.ResourceWarning)): + with check_warnings(("", ResourceWarning)): f = None gc_collect() except AssertionError: # pragma: no cover @@ -819,8 +851,10 @@ class BaseTestAPI: def loop_waiting_for_flag(self, instance, timeout=5): # pragma: no cover timeout = float(timeout) / 100 count = 100 + while asyncore.socket_map and count > 0: asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: return count -= 1 @@ -966,6 +1000,7 @@ class BaseTestAPI: # Make sure handle_expt is called on OOB data received. # Note: this might fail on some platforms as OOB data is # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: self.skipTest("Not applicable to AF_UNIX sockets.") @@ -980,7 +1015,7 @@ class BaseTestAPI: class TestHandler(BaseTestHandler): def __init__(self, conn): BaseTestHandler.__init__(self, conn) - self.socket.send(compat.tobytes(chr(244)), socket.MSG_OOB) + self.socket.send(chr(244).encode("latin-1"), socket.MSG_OOB) server = BaseServer(self.family, self.addr, TestHandler) client = TestClient(self.family, server.address) @@ -1082,6 +1117,7 @@ class BaseTestAPI: @reap_threads def test_quick_connect(self): # pragma: no cover # see: http://bugs.python.org/issue10340 + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): self.skipTest("test specific to AF_INET and AF_INET6") @@ -1420,7 +1456,7 @@ class Test_dispatcher(unittest.TestCase): sock = dummysocket() def getpeername(): - raise socket.error(errno.EBADF) + raise OSError(errno.EBADF) map = {} sock.getpeername = getpeername @@ -1454,7 +1490,7 @@ class Test_dispatcher(unittest.TestCase): def setsockopt(*arg, **kw): sock.errored = True - raise socket.error + raise OSError sock.setsockopt = setsockopt sock.getsockopt = lambda *arg: 0 @@ -1486,7 +1522,7 @@ class Test_dispatcher(unittest.TestCase): map = {} def accept(*arg, **kw): - raise socket.error(122) + raise OSError(122) sock.accept = accept inst = self._makeOne(sock=sock, map=map) @@ -1497,7 +1533,7 @@ class Test_dispatcher(unittest.TestCase): map = {} def send(*arg, **kw): - raise socket.error(errno.EWOULDBLOCK) + raise OSError(errno.EWOULDBLOCK) sock.send = send inst = self._makeOne(sock=sock, map=map) @@ -1509,7 +1545,7 @@ class Test_dispatcher(unittest.TestCase): map = {} def send(*arg, **kw): - raise socket.error(122) + raise OSError(122) sock.send = send inst = self._makeOne(sock=sock, map=map) @@ -1520,7 +1556,7 @@ class Test_dispatcher(unittest.TestCase): map = {} def recv(*arg, **kw): - raise socket.error(errno.ECONNRESET) + raise OSError(errno.ECONNRESET) def handle_close(): inst.close_handled = True @@ -1537,7 +1573,7 @@ class Test_dispatcher(unittest.TestCase): map = {} def close(): - raise socket.error(122) + raise OSError(122) sock.close = close inst = self._makeOne(sock=sock, map=map) @@ -1680,7 +1716,7 @@ class Test_close_all(unittest.TestCase): self.assertRaises(RuntimeError, self._callFUT, map) -class DummyDispatcher(object): +class DummyDispatcher: read_event_handled = False write_event_handled = False expt_event_handled = False @@ -1693,16 +1729,19 @@ class DummyDispatcher(object): def handle_read_event(self): self.read_event_handled = True + if self.exc is not None: raise self.exc def handle_write_event(self): self.write_event_handled = True + if self.exc is not None: raise self.exc def handle_expt_event(self): self.expt_event_handled = True + if self.exc is not None: raise self.exc @@ -1723,7 +1762,7 @@ class DummyDispatcher(object): raise self.exc -class DummyTime(object): +class DummyTime: def __init__(self): self.sleepvals = [] @@ -1731,7 +1770,7 @@ class DummyTime(object): self.sleepvals.append(val) -class DummySelect(object): +class DummySelect: error = select.error def __init__(self, exc=None, pollster=None): @@ -1741,6 +1780,7 @@ class DummySelect(object): def select(self, *arg): self.selected.append(arg) + if self.exc is not None: raise self.exc @@ -1748,13 +1788,14 @@ class DummySelect(object): return self.pollster -class DummyPollster(object): +class DummyPollster: def __init__(self, exc=None): self.polled = [] self.exc = exc def poll(self, timeout): self.polled.append(timeout) + if self.exc is not None: raise self.exc else: # pragma: no cover @@ -1,8 +1,8 @@ [tox] envlist = lint, - py27,pypy, py35,py36,py37,py38,pypy3, + py39, docs, coverage isolated_build = True @@ -26,12 +26,13 @@ deps = coverage setenv = COVERAGE_FILE=.coverage -depends = py27, py38 +depends = py38 [testenv:lint] skip_install = True commands = black --check --diff . + isort --check-only --df src/waitress tests check-manifest # flake8 src/waitress/ tests # build sdist/wheel @@ -39,12 +40,13 @@ commands = twine check dist/* deps = black - readme_renderer check-manifest - pep517 - twine flake8 flake8-bugbear + isort + pep517 + readme_renderer + twine [testenv:docs] whitelist_externals = @@ -62,12 +64,14 @@ deps = flake8 flake8-bugbear -[testenv:run-black] +[testenv:run-format] skip_install = True commands = + isort src/waitress tests black . deps = black + isort [testenv:build] skip_install = true |