diff options
| author | Bert JW Regeer <bertjw@regeer.org> | 2019-07-24 21:06:34 -0600 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-07-24 21:06:34 -0600 |
| commit | 94e23114bf4e8db9507f3550294037a4804eb053 (patch) | |
| tree | 9048993d211447e6df25d4cfaacd48a370cec47e | |
| parent | e2210c9258702b7a46fa23f3f5e8389d56999748 (diff) | |
| parent | 0667b8eee7ce33af659f5e6e09da3b31a7f78cfd (diff) | |
| download | waitress-94e23114bf4e8db9507f3550294037a4804eb053.tar.gz | |
Merge pull request #259 from Pylons/proxy-header-middleware
move proxy headers to a middleware and return 400 for malformed values
| -rw-r--r-- | .coveragerc | 14 | ||||
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | setup.cfg | 1 | ||||
| -rw-r--r-- | tox.ini | 9 | ||||
| -rw-r--r-- | waitress/adjustments.py | 7 | ||||
| -rw-r--r-- | waitress/proxy_headers.py | 351 | ||||
| -rw-r--r-- | waitress/server.py | 16 | ||||
| -rw-r--r-- | waitress/task.py | 294 | ||||
| -rw-r--r-- | waitress/tests/fixtureapps/echo.py | 54 | ||||
| -rw-r--r-- | waitress/tests/test_functional.py | 97 | ||||
| -rw-r--r-- | waitress/tests/test_proxy_headers.py | 695 | ||||
| -rw-r--r-- | waitress/tests/test_task.py | 617 | ||||
| -rw-r--r-- | waitress/utilities.py | 139 |
13 files changed, 1295 insertions, 1000 deletions
diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..cbacd63 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,14 @@ +[run] +parallel = true +concurrency = + thread + multiprocessing +source = + waitress + +omit = + waitress/tests/fixtureapps/getline.py + +[report] +show_missing = true +precision = 2 @@ -2,6 +2,7 @@ *.pyc env*/ .coverage +.coverage.* .idea/ .tox/ nosetests.xml @@ -3,7 +3,6 @@ zip_ok = false [nosetests] match=^test -where=waitress nocapture=1 cover-package=waitress cover-erase=1 @@ -6,7 +6,7 @@ envlist = [testenv] commands = - nosetests --with-xunit --xunit-file=nosetests-{envname}.xml {posargs:} + nosetests --with-xunit --xunit-file=nosetests-{envname}.xml {posargs:waitress} extras = testing @@ -21,7 +21,7 @@ extras = [py-cover] commands = - coverage run --source=waitress --parallel-mode {envbindir}/nosetests + coverage run {envbindir}/nosetests waitress coverage combine coverage xml -o {envname}.xml @@ -31,22 +31,23 @@ extras = [testenv:py27-cover] commands = {[py-cover]commands} + setenv = COVERAGE_FILE=.coverage.py2 [testenv:py35-cover] commands = {[py-cover]commands} + setenv = COVERAGE_FILE=.coverage.py3 [testenv:coverage] basepython = python3.5 commands = - coverage erase coverage combine coverage xml - coverage report --show-missing --fail-under=100 --omit=waitress/tests/fixtureapps/getline.py + coverage report --show-missing --fail-under=100 deps = coverage setenv = diff --git a/waitress/adjustments.py b/waitress/adjustments.py index 5c1879b..f33c570 100644 --- a/waitress/adjustments.py +++ b/waitress/adjustments.py @@ -17,7 +17,7 @@ import getopt import socket import warnings -from .utilities import PROXY_HEADERS +from .proxy_headers import PROXY_HEADERS from .compat import ( PY2, WIN, @@ -27,7 +27,10 @@ from .compat import ( truthy = frozenset(('t', 'true', 'y', 'yes', 'on', '1')) -KNOWN_PROXY_HEADERS = {header.lower().replace('_', '-') for header in PROXY_HEADERS} +KNOWN_PROXY_HEADERS = frozenset( + header.lower().replace('_', '-') + for header in PROXY_HEADERS +) def asbool(s): """ Return the boolean value ``True`` if the case-lowered value of string diff --git a/waitress/proxy_headers.py b/waitress/proxy_headers.py new file mode 100644 index 0000000..132fea8 --- /dev/null +++ b/waitress/proxy_headers.py @@ -0,0 +1,351 @@ +from collections import namedtuple + +from .utilities import logger, undquote, BadRequest + + +PROXY_HEADERS = frozenset({ + 'X_FORWARDED_FOR', + 'X_FORWARDED_HOST', + 'X_FORWARDED_PROTO', + 'X_FORWARDED_PORT', + 'X_FORWARDED_BY', + 'FORWARDED', +}) + +Forwarded = namedtuple('Forwarded', ['by', 'for_', 'host', 'proto']) + + +class MalformedProxyHeader(Exception): + def __init__(self, header, reason, value): + self.header = header + self.reason = reason + self.value = value + super(MalformedProxyHeader, self).__init__(header, reason, value) + + +def proxy_headers_middleware( + app, + trusted_proxy=None, + trusted_proxy_count=1, + trusted_proxy_headers=None, + clear_untrusted=True, + log_untrusted=False, + logger=logger, +): + def translate_proxy_headers(environ, start_response): + untrusted_headers = PROXY_HEADERS + remote_peer = environ['REMOTE_ADDR'] + if trusted_proxy == '*' or remote_peer == trusted_proxy: + try: + untrusted_headers = parse_proxy_headers( + environ, + trusted_proxy_count=trusted_proxy_count, + trusted_proxy_headers=trusted_proxy_headers, + logger=logger, + ) + except MalformedProxyHeader as ex: + logger.warning( + 'Malformed proxy header "%s" from "%s": %s value: %s', + ex.header, remote_peer, ex.reason, ex.value) + error = BadRequest('Header "{0}" malformed.'.format(ex.header)) + return error.wsgi_response(environ, start_response) + + # Clear out the untrusted proxy headers + if clear_untrusted: + clear_untrusted_headers( + environ, + untrusted_headers, + log_warning=log_untrusted, + logger=logger, + ) + + return app(environ, start_response) + return translate_proxy_headers + + +def parse_proxy_headers( + environ, + trusted_proxy_count, + trusted_proxy_headers, + logger=logger, +): + if trusted_proxy_headers is None: + trusted_proxy_headers = set() + + forwarded_for = [] + forwarded_host = forwarded_proto = forwarded_port = forwarded = "" + client_addr = None + untrusted_headers = set(PROXY_HEADERS) + + def raise_for_multiple_values(): + raise ValueError( + 'Unspecified behavior for multiple values found in header', + ) + + if ( + "x-forwarded-for" in trusted_proxy_headers + and "HTTP_X_FORWARDED_FOR" in environ + ): + try: + forwarded_for = [] + + for forward_hop in environ["HTTP_X_FORWARDED_FOR"].split(","): + forward_hop = forward_hop.strip() + forward_hop = undquote(forward_hop) + + # Make sure that all IPv6 addresses are surrounded by brackets, + # this is assuming that the IPv6 representation here does not + # include a port number. + + if "." not in forward_hop and ( + ":" in forward_hop and forward_hop[-1] != "]" + ): + forwarded_for.append("[{}]".format(forward_hop)) + else: + forwarded_for.append(forward_hop) + + forwarded_for = forwarded_for[-trusted_proxy_count:] + client_addr = forwarded_for[0] + + untrusted_headers.remove("X_FORWARDED_FOR") + except Exception as ex: + raise MalformedProxyHeader( + "X-Forwarded-For", + str(ex), + environ['HTTP_X_FORWARDED_FOR'], + ) + + if ( + "x-forwarded-host" in trusted_proxy_headers + and "HTTP_X_FORWARDED_HOST" in environ + ): + try: + forwarded_host_multiple = [] + + for forward_host in environ["HTTP_X_FORWARDED_HOST"].split(","): + forward_host = forward_host.strip() + forward_host = undquote(forward_host) + forwarded_host_multiple.append(forward_host) + + forwarded_host_multiple = forwarded_host_multiple[-trusted_proxy_count:] + forwarded_host = forwarded_host_multiple[0] + + untrusted_headers.remove("X_FORWARDED_HOST") + except Exception as ex: + raise MalformedProxyHeader( + "X-Forwarded-Host", + str(ex), + environ['HTTP_X_FORWARDED_HOST'], + ) + + if "x-forwarded-proto" in trusted_proxy_headers: + try: + forwarded_proto = undquote(environ.get("HTTP_X_FORWARDED_PROTO", "")) + if ',' in forwarded_proto: + raise_for_multiple_values() + untrusted_headers.remove("X_FORWARDED_PROTO") + except Exception as ex: + raise MalformedProxyHeader( + "X-Forwarded-Proto", + str(ex), + environ['HTTP_X_FORWARDED_PROTO'], + ) + + if "x-forwarded-port" in trusted_proxy_headers: + try: + forwarded_port = undquote(environ.get("HTTP_X_FORWARDED_PORT", "")) + if ',' in forwarded_port: + raise_for_multiple_values() + untrusted_headers.remove("X_FORWARDED_PORT") + except Exception as ex: + raise MalformedProxyHeader( + "X-Forwarded-Port", + str(ex), + environ['HTTP_X_FORWARDED_PORT'], + ) + + if "x-forwarded-by" in trusted_proxy_headers: + # Waitress itself does not use X-Forwarded-By, but we can not + # remove it so it can get set in the environ + untrusted_headers.remove("X_FORWARDED_BY") + + if "forwarded" in trusted_proxy_headers: + forwarded = environ.get("HTTP_FORWARDED", None) + untrusted_headers = PROXY_HEADERS - {"FORWARDED"} + + # If the Forwarded header exists, it gets priority + if forwarded: + proxies = [] + try: + for forwarded_element in forwarded.split(","): + # Remove whitespace that may have been introduced when + # appending a new entry + forwarded_element = forwarded_element.strip() + + forwarded_for = forwarded_host = forwarded_proto = "" + forwarded_port = forwarded_by = "" + + for pair in forwarded_element.split(";"): + pair = pair.lower() + + if not pair: + continue + + token, equals, value = pair.partition("=") + + if equals != "=": + raise ValueError('Invalid forwarded-pair missing "="') + + if token.strip() != token: + raise ValueError('Token may not be surrounded by whitespace') + + if value.strip() != value: + raise ValueError('Value may not be surrounded by whitespace') + + if token == "by": + forwarded_by = undquote(value) + + elif token == "for": + forwarded_for = undquote(value) + + elif token == "host": + forwarded_host = undquote(value) + + elif token == "proto": + forwarded_proto = undquote(value) + + else: + logger.warning("Unknown Forwarded token: %s" % token) + + proxies.append( + Forwarded( + forwarded_by, forwarded_for, forwarded_host, forwarded_proto + ) + ) + except Exception as ex: + raise MalformedProxyHeader( + "Forwarded", str(ex), environ['HTTP_FORWARDED'], + ) + + proxies = proxies[-trusted_proxy_count:] + + # Iterate backwards and fill in some values, the oldest entry that + # contains the information we expect is the one we use. We expect + # that intermediate proxies may re-write the host header or proto, + # but the oldest entry is the one that contains the information the + # client expects when generating URL's + # + # Forwarded: for="[2001:db8::1]";host="example.com:8443";proto="https" + # Forwarded: for=192.0.2.1;host="example.internal:8080" + # + # (After HTTPS header folding) should mean that we use as values: + # + # Host: example.com + # Protocol: https + # Port: 8443 + + for proxy in proxies[::-1]: + client_addr = proxy.for_ or client_addr + forwarded_host = proxy.host or forwarded_host + forwarded_proto = proxy.proto or forwarded_proto + + if forwarded_proto: + forwarded_proto = forwarded_proto.lower() + + if forwarded_proto not in {"http", "https"}: + raise MalformedProxyHeader( + "Forwarded Proto=" if forwarded else "X-Forwarded-Proto", + "unsupported proto value", + forwarded_proto, + ) + + # Set the URL scheme to the proxy provided proto + environ["wsgi.url_scheme"] = forwarded_proto + + if not forwarded_port: + if forwarded_proto == "http": + forwarded_port = "80" + + if forwarded_proto == "https": + forwarded_port = "443" + + if forwarded_host: + if ":" in forwarded_host and forwarded_host[-1] != "]": + host, port = forwarded_host.rsplit(":", 1) + host, port = host.strip(), str(port) + + # We trust the port in the Forwarded Host/X-Forwarded-Host over + # X-Forwarded-Port, or whatever we got from Forwarded + # Proto/X-Forwarded-Proto. + + if forwarded_port != port: + forwarded_port = port + + # We trust the proxy server's forwarded Host + environ["SERVER_NAME"] = host + environ["HTTP_HOST"] = forwarded_host + else: + # We trust the proxy server's forwarded Host + environ["SERVER_NAME"] = forwarded_host + environ["HTTP_HOST"] = forwarded_host + + if forwarded_port: + if forwarded_port not in {"443", "80"}: + environ["HTTP_HOST"] = "{}:{}".format( + forwarded_host, forwarded_port + ) + elif ( + forwarded_port == "80" + and environ["wsgi.url_scheme"] != "http" + ): + environ["HTTP_HOST"] = "{}:{}".format( + forwarded_host, forwarded_port + ) + elif ( + forwarded_port == "443" + and environ["wsgi.url_scheme"] != "https" + ): + environ["HTTP_HOST"] = "{}:{}".format( + forwarded_host, forwarded_port + ) + + if forwarded_port: + environ["SERVER_PORT"] = str(forwarded_port) + + if client_addr: + if ":" in client_addr and client_addr[-1] != "]": + addr, port = client_addr.rsplit(":", 1) + environ["REMOTE_ADDR"] = strip_brackets(addr.strip()) + environ["REMOTE_PORT"] = port.strip() + else: + environ["REMOTE_ADDR"] = strip_brackets(client_addr.strip()) + environ["REMOTE_HOST"] = environ["REMOTE_ADDR"] + + return untrusted_headers + + +def strip_brackets(addr): + if addr[0] == "[" and addr[-1] == "]": + return addr[1:-1] + return addr + + +def clear_untrusted_headers( + environ, untrusted_headers, log_warning=False, logger=logger +): + untrusted_headers_removed = [ + header + for header in untrusted_headers + if environ.pop('HTTP_' + header, False) is not False + ] + + if log_warning and untrusted_headers_removed: + untrusted_headers_removed = [ + "-".join(x.capitalize() for x in header.split("_")) + for header in untrusted_headers_removed + ] + logger.warning( + "Removed untrusted headers (%s). Waitress recommends these be " + "removed upstream.", + ", ".join(untrusted_headers_removed), + ) diff --git a/waitress/server.py b/waitress/server.py index 7ef930e..307c377 100644 --- a/waitress/server.py +++ b/waitress/server.py @@ -28,6 +28,7 @@ from waitress.compat import ( IPV6_V6ONLY, ) from . import wasyncore +from .proxy_headers import proxy_headers_middleware def create_server(application, map=None, @@ -182,6 +183,21 @@ class BaseWSGIServer(wasyncore.dispatcher, object): ): if adj is None: adj = Adjustments(**kw) + + if adj.trusted_proxy or adj.clear_untrusted_proxy_headers: + # wrap the application to deal with proxy headers + # we wrap it here because webtest subclasses the TcpWSGIServer + # directly and thus doesn't run any code that's in create_server + application = proxy_headers_middleware( + application, + trusted_proxy=adj.trusted_proxy, + trusted_proxy_count=adj.trusted_proxy_count, + trusted_proxy_headers=adj.trusted_proxy_headers, + clear_untrusted=adj.clear_untrusted_proxy_headers, + log_untrusted=adj.log_untrusted_proxy_headers, + logger=self.logger, + ) + if map is None: # use a nonglobal socket map by default to hopefully prevent # conflicts with apps and libs that use the wasyncore global socket diff --git a/waitress/task.py b/waitress/task.py index 81e512f..78d8b83 100644 --- a/waitress/task.py +++ b/waitress/task.py @@ -21,13 +21,9 @@ import time from .buffers import ReadOnlyFileBasedBuffer from .compat import reraise, tobytes from .utilities import ( - Forwarded, - PROXY_HEADERS, build_http_date, - clear_untrusted_headers, logger, queue_logger, - undquote, ) rename_headers = { # or keep them without the HTTP_ prefix added @@ -357,14 +353,9 @@ class ErrorTask(Task): def execute(self): e = self.request.error - body = '%s\r\n\r\n%s' % (e.reason, e.body) - tag = '\r\n\r\n(generated by waitress)' - body = body + tag - self.status = '%s %s' % (e.code, e.reason) - cl = len(body) - self.content_length = cl - self.response_headers.append(('Content-Length', str(cl))) - self.response_headers.append(('Content-Type', 'text/plain')) + status, headers, body = e.to_response() + self.status = status + self.response_headers.extend(headers) if self.version == '1.1': connection = self.request.headers.get('CONNECTION', '').lower() if connection == 'close': @@ -374,15 +365,17 @@ class ErrorTask(Task): # HTTP 1.0 self.response_headers.append(('Connection', 'close')) self.close_on_finish = True + self.content_length = len(body) self.write(tobytes(body)) + class WSGITask(Task): """A WSGI task produces a response from a WSGI application. """ environ = None def execute(self): - env = self.get_environment() + environ = self.get_environment() def start_response(status, headers, exc_info=None): if self.complete and not exc_info: @@ -444,7 +437,7 @@ class WSGITask(Task): return self.write # Call the application to handle the request and write a response - app_iter = self.channel.server.application(env, start_response) + app_iter = self.channel.server.application(environ, start_response) can_close_app_iter = True try: @@ -501,236 +494,6 @@ class WSGITask(Task): if can_close_app_iter and hasattr(app_iter, 'close'): app_iter.close() - def parse_proxy_headers( - self, - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers=None, - ): - if trusted_proxy_headers is None: - trusted_proxy_headers = set() - - forwarded_for = [] - forwarded_host = forwarded_proto = forwarded_port = forwarded = "" - client_addr = None - untrusted_headers = set(PROXY_HEADERS) - - def warn_unspecified_behavior(header): - self.logger.warning( - "Found multiple values in %s, this has unspecified behaviour. " - "Ignoring header value.", - header, - ) - - if "x-forwarded-for" in trusted_proxy_headers and "X_FORWARDED_FOR" in headers: - forwarded_for = [] - - for forward_hop in headers["X_FORWARDED_FOR"].split(","): - forward_hop = forward_hop.strip() - forward_hop = undquote(forward_hop) - - # Make sure that all IPv6 addresses are surrounded by brackets, - # this is assuming that the IPv6 representation here does not - # include a port number. - - if "." not in forward_hop and ( - ":" in forward_hop and forward_hop[-1] != "]" - ): - forwarded_for.append("[{}]".format(forward_hop)) - else: - forwarded_for.append(forward_hop) - - forwarded_for = forwarded_for[-trusted_proxy_count:] - client_addr = forwarded_for[0] - - untrusted_headers.remove("X_FORWARDED_FOR") - - if "x-forwarded-host" in trusted_proxy_headers and "X_FORWARDED_HOST" in headers: - forwarded_host_multiple = [] - - for forward_host in headers["X_FORWARDED_HOST"].split(","): - forward_host = forward_host.strip() - forward_host = undquote(forward_host) - forwarded_host_multiple.append(forward_host) - - forwarded_host_multiple = forwarded_host_multiple[-trusted_proxy_count:] - forwarded_host = forwarded_host_multiple[0] - - untrusted_headers.remove("X_FORWARDED_HOST") - - if "x-forwarded-proto" in trusted_proxy_headers: - forwarded_proto = undquote(headers.get("X_FORWARDED_PROTO", "")) - untrusted_headers.remove("X_FORWARDED_PROTO") - - if "," in forwarded_proto: - forwarded_proto = "" - warn_unspecified_behavior("X-Forwarded-Proto") - - if "x-forwarded-port" in trusted_proxy_headers: - forwarded_port = undquote(headers.get("X_FORWARDED_PORT", "")) - untrusted_headers.remove("X_FORWARDED_PORT") - - if "," in forwarded_port: - forwarded_port = "" - warn_unspecified_behavior("X-Forwarded-Port") - - if "x-forwarded-by" in trusted_proxy_headers: - # Waitress itself does not use X-Forwarded-By, but we can not - # remove it so it can get set in the environ - untrusted_headers.remove("X_FORWARDED_BY") - - if "forwarded" in trusted_proxy_headers: - forwarded = headers.get("FORWARDED", None) - untrusted_headers = PROXY_HEADERS - {"FORWARDED"} - - # If the Forwarded header exists, it gets priority - if forwarded: - proxies = [] - - for forwarded_element in forwarded.split(","): - # Remove whitespace that may have been introduced when - # appending a new entry - forwarded_element = forwarded_element.strip() - - forwarded_for = forwarded_host = forwarded_proto = "" - forwarded_port = forwarded_by = "" - - for pair in forwarded_element.split(";"): - pair = pair.lower() - - if not pair: - continue - - token, equals, value = pair.partition("=") - - if equals != "=": - raise ValueError("Invalid forwarded-pair in Forwarded element") - - if token.strip() != token: - raise ValueError("token may not be surrounded by whitespace") - - if value.strip() != value: - raise ValueError("value may not be surrounded by whitespace") - - if token == "by": - forwarded_by = undquote(value) - - elif token == "for": - forwarded_for = undquote(value) - - elif token == "host": - forwarded_host = undquote(value) - - elif token == "proto": - forwarded_proto = undquote(value) - - else: - self.logger.warning("Unknown Forwarded token: %s" % token) - - proxies.append( - Forwarded( - forwarded_by, forwarded_for, forwarded_host, forwarded_proto - ) - ) - - proxies = proxies[-trusted_proxy_count:] - - # Iterate backwards and fill in some values, the oldest entry that - # contains the information we expect is the one we use. We expect - # that intermediate proxies may re-write the host header or proto, - # but the oldest entry is the one that contains the information the - # client expects when generating URL's - # - # Forwarded: for="[2001:db8::1]";host="example.com:8443";proto="https" - # Forwarded: for=192.0.2.1;host="example.internal:8080" - # - # (After HTTPS header folding) should mean that we use as values: - # - # Host: example.com - # Protocol: https - # Port: 8443 - - for proxy in proxies[::-1]: - client_addr = proxy.for_ or client_addr - forwarded_host = proxy.host or forwarded_host - forwarded_proto = proxy.proto or forwarded_proto - - if forwarded_proto: - forwarded_proto = forwarded_proto.lower() - - if forwarded_proto not in {"http", "https"}: - raise ValueError( - 'Invalid "Forwarded Proto=" or "X-Forwarded-Proto" value.' - ) - - # Set the URL scheme to the proxy provided proto - environ["wsgi.url_scheme"] = forwarded_proto - - if not forwarded_port: - if forwarded_proto == "http": - forwarded_port = "80" - - if forwarded_proto == "https": - forwarded_port = "443" - - if forwarded_host: - if ":" in forwarded_host and forwarded_host[-1] != "]": - host, port = forwarded_host.rsplit(":", 1) - host, port = host.strip(), str(port) - - # We trust the port in the Forwarded Host/X-Forwarded-Host over - # X-Forwarded-Port, or whatever we got from Forwarded - # Proto/X-Forwarded-Proto. - - if forwarded_port != port: - forwarded_port = port - - # We trust the proxy server's forwarded Host - environ["SERVER_NAME"] = host - environ["HTTP_HOST"] = forwarded_host - else: - # We trust the proxy server's forwarded Host - environ["SERVER_NAME"] = forwarded_host - environ["HTTP_HOST"] = forwarded_host - - if forwarded_port: - if forwarded_port not in {"443", "80"}: - environ["HTTP_HOST"] = "{}:{}".format( - forwarded_host, forwarded_port - ) - elif ( - forwarded_port == "80" and environ["wsgi.url_scheme"] != "http" - ): - environ["HTTP_HOST"] = "{}:{}".format( - forwarded_host, forwarded_port - ) - elif ( - forwarded_port == "443" - and environ["wsgi.url_scheme"] != "https" - ): - environ["HTTP_HOST"] = "{}:{}".format( - forwarded_host, forwarded_port - ) - - if forwarded_port: - environ["SERVER_PORT"] = str(forwarded_port) - - if client_addr: - def strip_brackets(addr): - if addr[0] == "[" and addr[-1] == "]": - return addr[1:-1] - return addr - - if ":" in client_addr and client_addr[-1] != "]": - addr, port = client_addr.rsplit(":", 1) - environ["REMOTE_ADDR"] = strip_brackets(addr.strip()) - environ["REMOTE_PORT"] = port.strip() - else: - environ["REMOTE_ADDR"] = strip_brackets(client_addr.strip()) - - return untrusted_headers - def get_environment(self): """Returns a WSGI environment.""" environ = self.environ @@ -766,6 +529,14 @@ class WSGITask(Task): path = path[len(url_prefix):] environ = { + 'REMOTE_ADDR': channel.addr[0], + # Nah, we aren't actually going to look up the reverse DNS for + # REMOTE_ADDR, but we will happily set this environment variable + # for the WSGI application. Spec says we can just set this to + # REMOTE_ADDR, so we do. + 'REMOTE_HOST': channel.addr[0], + # try and set the REMOTE_PORT to something useful, but maybe None + 'REMOTE_PORT': str(channel.addr[1]), 'REQUEST_METHOD': request.command.upper(), 'SERVER_PORT': str(server.effective_port), 'SERVER_NAME': server.server_name, @@ -788,43 +559,12 @@ class WSGITask(Task): 'wsgi.file_wrapper': ReadOnlyFileBasedBuffer, 'wsgi.input_terminated': True, # wsgi.input is EOF terminated } - remote_peer = environ['REMOTE_ADDR'] = channel.addr[0] - - headers = dict(request.headers) - - untrusted_headers = PROXY_HEADERS - if server.adj.trusted_proxy == '*' or remote_peer == server.adj.trusted_proxy: - untrusted_headers = self.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=server.adj.trusted_proxy_count, - trusted_proxy_headers=server.adj.trusted_proxy_headers, - ) - else: - # If we are not relying on a proxy, we still want to try and set - # the REMOTE_PORT to something useful, maybe None though. - environ["REMOTE_PORT"] = str(channel.addr[1]) - - # Nah, we aren't actually going to look up the reverse DNS for - # REMOTE_ADDR, but we will happily set this environment variable for - # the WSGI application. Spec says we can just set this to REMOTE_ADDR, - # so we do. - environ["REMOTE_HOST"] = environ["REMOTE_ADDR"] - - # Clear out the untrusted proxy headers - if server.adj.clear_untrusted_proxy_headers: - clear_untrusted_headers( - headers, - untrusted_headers, - log_warning=server.adj.log_untrusted_proxy_headers, - logger=self.logger, - ) - for key, value in headers.items(): + for key, value in dict(request.headers).items(): value = value.strip() mykey = rename_headers.get(key, None) if mykey is None: - mykey = 'HTTP_%s' % key + mykey = 'HTTP_' + key if mykey not in environ: environ[mykey] = value diff --git a/waitress/tests/fixtureapps/echo.py b/waitress/tests/fixtureapps/echo.py index f5fd5d1..5509703 100644 --- a/waitress/tests/fixtureapps/echo.py +++ b/waitress/tests/fixtureapps/echo.py @@ -1,11 +1,55 @@ -def app(environ, start_response): # pragma: no cover +from collections import namedtuple +import json + +def app_body_only(environ, start_response): # pragma: no cover cl = environ.get('CONTENT_LENGTH', None) if cl is not None: cl = int(cl) body = environ['wsgi.input'].read(cl) cl = str(len(body)) - start_response( - '200 OK', - [('Content-Length', cl), ('Content-Type', 'text/plain')] - ) + start_response('200 OK', [ + ('Content-Length', cl), + ('Content-Type', 'text/plain'), + ]) return [body] + +def app(environ, start_response): # pragma: no cover + cl = environ.get('CONTENT_LENGTH', None) + if cl is not None: + cl = int(cl) + request_body = environ['wsgi.input'].read(cl) + cl = str(len(request_body)) + meta = { + 'method': environ['REQUEST_METHOD'], + 'path_info': environ['PATH_INFO'], + 'script_name': environ['SCRIPT_NAME'], + 'query_string': environ['QUERY_STRING'], + 'content_length': cl, + 'scheme': environ['wsgi.url_scheme'], + 'remote_addr': environ['REMOTE_ADDR'], + 'remote_host': environ['REMOTE_HOST'], + 'server_port': environ['SERVER_PORT'], + 'server_name': environ['SERVER_NAME'], + 'headers': { + k[len('HTTP_'):]: v + for k, v in environ.items() + if k.startswith('HTTP_') + }, + } + response = json.dumps(meta).encode('utf8') + b'\r\n\r\n' + request_body + start_response('200 OK', [ + ('Content-Length', str(len(response))), + ('Content-Type', 'text/plain'), + ]) + return [response] + + +Echo = namedtuple('Echo', ( + 'method path_info script_name query_string content_length scheme ' + 'remote_addr remote_host server_port server_name headers body' +)) + +def parse_response(response): + meta, body = response.split(b'\r\n\r\n', 1) + meta = json.loads(meta.decode('utf8')) + return Echo(body=body, **meta) diff --git a/waitress/tests/test_functional.py b/waitress/tests/test_functional.py index f3170b9..0a39771 100644 --- a/waitress/tests/test_functional.py +++ b/waitress/tests/test_functional.py @@ -2,6 +2,7 @@ import errno import logging import multiprocessing import os +import signal import socket import string import subprocess @@ -28,8 +29,23 @@ def start_server(app, svr, queue, **kwargs): # pragma: no cover """Run a fixture application. """ logging.getLogger('waitress').addHandler(NullHandler()) + try_register_coverage() svr(app, queue, **kwargs).run() +def try_register_coverage(): # pragma: no cover + # Hack around multiprocessing exiting early and not triggering coverage's + # atexit handler by trapping the SIGTERM and saving coverage explicitly. + if '_COVERAGE_RCFILE' in os.environ: + import coverage + cov = coverage.Coverage(config_file=os.getenv('_COVERAGE_RCFILE')) + cov.start() + + def sigterm(*args): + cov.stop() + cov.save() + sys.exit(0) + signal.signal(signal.SIGTERM, sigterm) + class FixtureTcpWSGIServer(server.TcpWSGIServer): """A version of TcpWSGIServer that relays back what it's bound to. """ @@ -139,11 +155,22 @@ class EchoTests(object): def setUp(self): from waitress.tests.fixtureapps import echo - self.start_subprocess(echo.app) + self.start_subprocess( + echo.app, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto'}, + clear_untrusted_proxy_headers=True, + ) def tearDown(self): self.stop_subprocess() + def _read_echo(self, fp): + from waitress.tests.fixtureapps import echo + line, headers, body = read_http(fp) + return line, headers, echo.parse_response(body) + def test_date_and_server(self): to_send = ("GET / HTTP/1.0\n" "Content-Length: 0\n\n") @@ -151,7 +178,7 @@ class EchoTests(object): self.connect() self.sock.send(to_send) fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.0') self.assertEqual(headers.get('server'), 'waitress') self.assertTrue(headers.get('date')) @@ -177,10 +204,10 @@ class EchoTests(object): self.connect() self.sock.send(to_send) fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.0') - self.assertEqual(headers.get('content-length'), '5') - self.assertEqual(response_body, b'hello') + self.assertEqual(echo.content_length, '5') + self.assertEqual(echo.body, b'hello') def test_send_empty_body(self): to_send = ("GET / HTTP/1.0\n" @@ -189,10 +216,10 @@ class EchoTests(object): self.connect() self.sock.send(to_send) fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.0') - self.assertEqual(headers.get('content-length'), '0') - self.assertEqual(response_body, b'') + self.assertEqual(echo.content_length, '0') + self.assertEqual(echo.body, b'') def test_multiple_requests_with_body(self): orig_sock = self.sock @@ -222,11 +249,11 @@ class EchoTests(object): self.connect() self.sock.send(s) fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.0') - self.assertEqual(int(headers['content-length']), len(data)) - self.assertEqual(len(response_body), len(data)) - self.assertEqual(response_body, tobytes(data)) + self.assertEqual(int(echo.content_length), len(data)) + self.assertEqual(len(echo.body), len(data)) + self.assertEqual(echo.body, tobytes(data)) def test_large_body(self): # 1024 characters. @@ -240,10 +267,10 @@ class EchoTests(object): self.connect() self.sock.send(s) fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.0') - self.assertEqual(headers.get('content-length'), '1024') - self.assertEqual(response_body, tobytes(body)) + self.assertEqual(echo.content_length, '1024') + self.assertEqual(echo.body, tobytes(body)) def test_many_clients(self): conns = [] @@ -270,10 +297,10 @@ class EchoTests(object): self.sock.send(header) self.sock.send(b"0\r\n\r\n") fp = self.sock.makefile('rb', 0) - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.1') - self.assertEqual(response_body, b'') - self.assertEqual(headers['content-length'], '0') + self.assertEqual(echo.body, b'') + self.assertEqual(echo.content_length, '0') self.assertFalse('transfer-encoding' in headers) def test_chunking_request_with_content(self): @@ -291,10 +318,10 @@ class EchoTests(object): self.sock.send(control_line) self.sock.send(s) self.sock.send(b"0\r\n\r\n") - line, headers, response_body = read_http(fp) + line, headers, echo = self._read_echo(fp) self.assertline(line, '200', 'OK', 'HTTP/1.1') - self.assertEqual(response_body, expected) - self.assertEqual(headers['content-length'], str(len(expected))) + self.assertEqual(echo.body, expected) + self.assertEqual(echo.content_length, str(len(expected))) self.assertFalse('transfer-encoding' in headers) def test_broken_chunked_encoding(self): @@ -411,11 +438,35 @@ class EchoTests(object): self.assertEqual(int(response.status), 200) self.assertEqual(response.getheader('connection'), 'close') + def test_proxy_headers(self): + to_send = ( + "GET / HTTP/1.0\n" + "Content-Length: 0\n" + "Host: www.google.com:8080\n" + "X-Forwarded-For: 192.168.1.1\n" + "X-Forwarded-Proto: https\n" + "X-Forwarded-Port: 5000\n\n" + ) + to_send = tobytes(to_send) + self.connect() + self.sock.send(to_send) + fp = self.sock.makefile('rb', 0) + line, headers, echo = self._read_echo(fp) + self.assertline(line, '200', 'OK', 'HTTP/1.0') + self.assertEqual(headers.get('server'), 'waitress') + self.assertTrue(headers.get('date')) + self.assertIsNone(echo.headers.get('X_FORWARDED_PORT')) + self.assertEqual(echo.headers['HOST'], 'www.google.com:8080') + self.assertEqual(echo.scheme, 'https') + self.assertEqual(echo.remote_addr, '192.168.1.1') + self.assertEqual(echo.remote_host, '192.168.1.1') + + class PipeliningTests(object): def setUp(self): from waitress.tests.fixtureapps import echo - self.start_subprocess(echo.app) + self.start_subprocess(echo.app_body_only) def tearDown(self): self.stop_subprocess() @@ -454,7 +505,7 @@ class ExpectContinueTests(object): def setUp(self): from waitress.tests.fixtureapps import echo - self.start_subprocess(echo.app) + self.start_subprocess(echo.app_body_only) def tearDown(self): self.stop_subprocess() diff --git a/waitress/tests/test_proxy_headers.py b/waitress/tests/test_proxy_headers.py new file mode 100644 index 0000000..f3af58a --- /dev/null +++ b/waitress/tests/test_proxy_headers.py @@ -0,0 +1,695 @@ +import unittest + +from waitress.compat import tobytes + +class TestProxyHeadersMiddleware(unittest.TestCase): + + def _makeOne(self, app, **kw): + from waitress.proxy_headers import proxy_headers_middleware + return proxy_headers_middleware(app, **kw) + + def _callFUT(self, app, **kw): + response = DummyResponse() + environ = DummyEnviron(**kw) + def start_response(status, response_headers): + response.status = status + response.headers = response_headers + response.steps = list(app(environ, start_response)) + response.body = b''.join(tobytes(s) for s in response.steps) + return response + + def test_get_environment_values_w_scheme_override_untrusted(self): + inner = DummyApp() + app = self._makeOne(inner) + response = self._callFUT(app, headers={ + 'X_FOO': 'BAR', + 'X_FORWARDED_PROTO': 'https', + }) + self.assertEqual(response.status, '200 OK') + self.assertEqual(inner.environ['wsgi.url_scheme'], 'http') + + def test_get_environment_values_w_scheme_override_trusted(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_headers={'x-forwarded-proto'}, + ) + response = self._callFUT( + app, + addr=['192.168.1.1', 8080], + headers={ + 'X_FOO': 'BAR', + 'X_FORWARDED_PROTO': 'https', + }, + ) + + environ = inner.environ + self.assertEqual(response.status, '200 OK') + self.assertEqual(environ['SERVER_PORT'], '443') + self.assertEqual(environ['SERVER_NAME'], 'localhost') + self.assertEqual(environ['REMOTE_ADDR'], '192.168.1.1') + self.assertEqual(environ['HTTP_X_FOO'], 'BAR') + + def test_get_environment_values_w_bogus_scheme_override(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_headers={'x-forwarded-proto'}, + ) + response = self._callFUT( + app, + addr=['192.168.1.1', 80], + headers={ + 'X_FOO': 'BAR', + 'X_FORWARDED_PROTO': 'http://p02n3e.com?url=http', + }, + ) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "X-Forwarded-Proto" malformed', response.body) + + def test_get_environment_warning_other_proxy_headers(self): + inner = DummyApp() + logger = DummyLogger() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + log_untrusted=True, + logger=logger, + ) + response = self._callFUT( + app, + addr=['192.168.1.1', 80], + headers={ + 'X_FORWARDED_FOR': '[2001:db8::1]', + 'FORWARDED': 'For=198.51.100.2;host=example.com:8080;proto=https' + }, + ) + self.assertEqual(response.status, '200 OK') + + self.assertEqual(len(logger.logged), 1) + + environ = inner.environ + self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_get_environment_contains_all_headers_including_untrusted(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-by'}, + clear_untrusted=False, + ) + headers_orig = { + 'X_FORWARDED_FOR': '198.51.100.2', + 'X_FORWARDED_BY': 'Waitress', + 'X_FORWARDED_PROTO': 'https', + 'X_FORWARDED_HOST': 'example.org', + } + response = self._callFUT( + app, + addr=['192.168.1.1', 80], + headers=headers_orig.copy(), + ) + self.assertEqual(response.status, '200 OK') + environ = inner.environ + for k, expected in headers_orig.items(): + result = environ['HTTP_%s' % k] + self.assertEqual(result, expected) + + def test_get_environment_contains_only_trusted_headers(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-by'}, + clear_untrusted=True, + ) + response = self._callFUT( + app, + addr=['192.168.1.1', 80], + headers={ + 'X_FORWARDED_FOR': '198.51.100.2', + 'X_FORWARDED_BY': 'Waitress', + 'X_FORWARDED_PROTO': 'https', + 'X_FORWARDED_HOST': 'example.org', + }, + ) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['HTTP_X_FORWARDED_BY'], 'Waitress') + self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) + self.assertNotIn('HTTP_X_FORWARDED_PROTO', environ) + self.assertNotIn('HTTP_X_FORWARDED_HOST', environ) + + def test_get_environment_clears_headers_if_untrusted_proxy(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='192.168.1.1', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-by'}, + clear_untrusted=True, + ) + response = self._callFUT( + app, + addr=['192.168.1.255', 80], + headers={ + 'X_FORWARDED_FOR': '198.51.100.2', + 'X_FORWARDED_BY': 'Waitress', + 'X_FORWARDED_PROTO': 'https', + 'X_FORWARDED_HOST': 'example.org', + }, + ) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertNotIn('HTTP_X_FORWARDED_BY', environ) + self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) + self.assertNotIn('HTTP_X_FORWARDED_PROTO', environ) + self.assertNotIn('HTTP_X_FORWARDED_HOST', environ) + + def test_parse_proxy_headers_forwarded_for(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_headers={'x-forwarded-for'}, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '192.0.2.1') + + def test_parse_proxy_headers_forwarded_for_v6_missing_brackets(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_headers={'x-forwarded-for'}, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '2001:db8::0' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::0') + + def test_parse_proxy_headers_forwared_for_multiple(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={'x-forwarded-for'}, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + + def test_parse_forwarded_multiple_proxies_trust_only_two(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': ( + 'For=192.0.2.1;host=fake.com, ' + 'For=198.51.100.2;host=example.com:8080, ' + 'For=203.0.113.1' + ), + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + + def test_parse_forwarded_multiple_proxies(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': ( + 'for="[2001:db8::1]:3821";host="example.com:8443";proto="https", ' + 'for=192.0.2.1;host="example.internal:8080"' + ), + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::1') + self.assertEqual(environ['REMOTE_PORT'], '3821') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8443') + self.assertEqual(environ['SERVER_PORT'], '8443') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_parse_forwarded_multiple_proxies_minimal(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': ( + 'for="[2001:db8::1]";proto="https", ' + 'for=192.0.2.1;host="example.org"' + ), + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::1') + self.assertEqual(environ['SERVER_NAME'], 'example.org') + self.assertEqual(environ['HTTP_HOST'], 'example.org') + self.assertEqual(environ['SERVER_PORT'], '443') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_parse_proxy_headers_forwarded_host_with_port(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={ + 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com:8080', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + + def test_parse_proxy_headers_forwarded_host_without_port(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={ + 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com') + self.assertEqual(environ['SERVER_PORT'], '80') + + def test_parse_proxy_headers_forwarded_host_with_forwarded_port(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={ + 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', + 'x-forwarded-port', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com', + 'X_FORWARDED_PORT': '8080' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + + def test_parse_proxy_headers_forwarded_host_multiple_with_forwarded_port(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=2, + trusted_proxy_headers={ + 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', + 'x-forwarded-port', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com, example.org', + 'X_FORWARDED_PORT': '8080' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + + def test_parse_proxy_headers_forwarded_host_multiple_with_forwarded_port_limit_one_trusted(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={ + 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', + 'x-forwarded-port', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com, example.org', + 'X_FORWARDED_PORT': '8080' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '203.0.113.1') + self.assertEqual(environ['SERVER_NAME'], 'example.org') + self.assertEqual(environ['HTTP_HOST'], 'example.org:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + + def test_parse_forwarded(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': 'For=198.51.100.2:5858;host=example.com:8080;proto=https', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['REMOTE_PORT'], '5858') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_parse_forwarded_empty_pair(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': 'For=198.51.100.2;;proto=https;by=_unused', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + + def test_parse_forwarded_pair_token_whitespace(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': 'For=198.51.100.2; proto =https', + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "Forwarded" malformed', response.body) + + def test_parse_forwarded_pair_value_whitespace(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': 'For= "198.51.100.2"; proto =https', + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "Forwarded" malformed', response.body) + + def test_parse_forwarded_pair_no_equals(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': 'For' + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "Forwarded" malformed', response.body) + + def test_parse_forwarded_warning_unknown_token(self): + inner = DummyApp() + logger = DummyLogger() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'forwarded'}, + logger=logger, + ) + response = self._callFUT(app, headers={ + 'FORWARDED': ( + 'For=198.51.100.2;host=example.com:8080;proto=https;' + 'unknown="yolo"' + ), + }) + self.assertEqual(response.status, '200 OK') + + self.assertEqual(len(logger.logged), 1) + self.assertIn('Unknown Forwarded token', logger.logged[0]) + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') + self.assertEqual(environ['SERVER_PORT'], '8080') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_parse_no_valid_proxy_headers(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '198.51.100.2', + 'FORWARDED': 'For=198.51.100.2;host=example.com:8080;proto=https' + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['REMOTE_ADDR'], '127.0.0.1') + self.assertEqual(environ['SERVER_NAME'], 'localhost') + self.assertEqual(environ['HTTP_HOST'], '192.168.1.1:80') + self.assertEqual(environ['SERVER_PORT'], '8080') + self.assertEqual(environ['wsgi.url_scheme'], 'http') + + def test_parse_multiple_x_forwarded_proto(self): + inner = DummyApp() + logger = DummyLogger() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-proto'}, + logger=logger, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_PROTO': 'http, https', + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "X-Forwarded-Proto" malformed', response.body) + + def test_parse_multiple_x_forwarded_port(self): + inner = DummyApp() + logger = DummyLogger() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-port'}, + logger=logger, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_PORT': '443, 80', + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "X-Forwarded-Port" malformed', response.body) + + def test_parse_forwarded_port_wrong_proto_port_80(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={ + 'x-forwarded-port', 'x-forwarded-host', 'x-forwarded-proto', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_PORT': '80', + 'X_FORWARDED_PROTO': 'https', + 'X_FORWARDED_HOST': 'example.com', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:80') + self.assertEqual(environ['SERVER_PORT'], '80') + self.assertEqual(environ['wsgi.url_scheme'], 'https') + + def test_parse_forwarded_port_wrong_proto_port_443(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={ + 'x-forwarded-port', 'x-forwarded-host', 'x-forwarded-proto', + }, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_PORT': '443', + 'X_FORWARDED_PROTO': 'http', + 'X_FORWARDED_HOST': 'example.com', + }) + self.assertEqual(response.status, '200 OK') + + environ = inner.environ + self.assertEqual(environ['SERVER_NAME'], 'example.com') + self.assertEqual(environ['HTTP_HOST'], 'example.com:443') + self.assertEqual(environ['SERVER_PORT'], '443') + self.assertEqual(environ['wsgi.url_scheme'], 'http') + + def test_parse_forwarded_for_bad_quote(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-for'}, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_FOR': '"foo' + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "X-Forwarded-For" malformed', response.body) + + def test_parse_forwarded_host_bad_quote(self): + inner = DummyApp() + app = self._makeOne( + inner, + trusted_proxy='*', + trusted_proxy_count=1, + trusted_proxy_headers={'x-forwarded-host'}, + ) + response = self._callFUT(app, headers={ + 'X_FORWARDED_HOST': '"foo' + }) + self.assertEqual(response.status, '400 Bad Request') + self.assertIn(b'Header "X-Forwarded-Host" malformed', response.body) + + +class DummyLogger(object): + def __init__(self): + self.logged = [] + + def warning(self, msg, *args): + self.logged.append(msg % args) + + +class DummyApp(object): + def __call__(self, environ, start_response): + self.environ = environ + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield 'hello' + + +class DummyResponse(object): + status = None + headers = None + body = None + + +def DummyEnviron( + addr=('127.0.0.1', 8080), + scheme='http', + server='localhost', + headers=None, +): + environ = { + 'REMOTE_ADDR': addr[0], + 'REMOTE_HOST': addr[0], + 'REMOTE_PORT': addr[1], + 'SERVER_PORT': str(addr[1]), + 'SERVER_NAME': server, + 'wsgi.url_scheme': scheme, + 'HTTP_HOST': '192.168.1.1:80', + } + if headers: + environ.update({ + 'HTTP_' + key.upper().replace('-', '_'): value + for key, value in headers.items() + }) + return environ diff --git a/waitress/tests/test_task.py b/waitress/tests/test_task.py index ffc34b7..a75b87f 100644 --- a/waitress/tests/test_task.py +++ b/waitress/tests/test_task.py @@ -770,603 +770,6 @@ class TestWSGITask(unittest.TestCase): self.assertEqual(environ['wsgi.input_terminated'], True) self.assertEqual(inst.environ, environ) - def test_get_environment_values_w_scheme_override_untrusted(self): - inst = self._makeOne() - request = DummyParser() - request.headers = { - 'CONTENT_TYPE': 'abc', - 'CONTENT_LENGTH': '10', - 'X_FOO': 'BAR', - 'X_FORWARDED_PROTO': 'https', - 'CONNECTION': 'close', - } - request.query = 'abc' - inst.request = request - environ = inst.get_environment() - self.assertEqual(environ['wsgi.url_scheme'], 'http') - - def test_get_environment_values_w_scheme_override_trusted(self): - import sys - inst = self._makeOne() - inst.channel.addr = ['192.168.1.1', 8080] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_headers = {'x-forwarded-proto'} - request = DummyParser() - request.headers = { - 'CONTENT_TYPE': 'abc', - 'CONTENT_LENGTH': '10', - 'X_FOO': 'BAR', - 'X_FORWARDED_PROTO': 'https', - 'CONNECTION': 'close', - } - request.query = 'abc' - inst.request = request - environ = inst.get_environment() - - # nail the keys of environ - self.assertEqual(sorted(environ.keys()), [ - 'CONTENT_LENGTH', 'CONTENT_TYPE', 'HTTP_CONNECTION', 'HTTP_X_FOO', - 'HTTP_X_FORWARDED_PROTO', 'PATH_INFO', 'QUERY_STRING', - 'REMOTE_ADDR', 'REMOTE_HOST', 'REQUEST_METHOD', 'SCRIPT_NAME', - 'SERVER_NAME', 'SERVER_PORT', 'SERVER_PROTOCOL', 'SERVER_SOFTWARE', - 'wsgi.errors', 'wsgi.file_wrapper', 'wsgi.input', - 'wsgi.input_terminated', 'wsgi.multiprocess', 'wsgi.multithread', - 'wsgi.run_once', 'wsgi.url_scheme', 'wsgi.version' - ]) - - self.assertEqual(environ['REQUEST_METHOD'], 'GET') - self.assertEqual(environ['SERVER_PORT'], '443') - self.assertEqual(environ['SERVER_NAME'], 'localhost') - self.assertEqual(environ['SERVER_SOFTWARE'], 'waitress') - self.assertEqual(environ['SERVER_PROTOCOL'], 'HTTP/1.0') - self.assertEqual(environ['SCRIPT_NAME'], '') - self.assertEqual(environ['HTTP_CONNECTION'], 'close') - self.assertEqual(environ['PATH_INFO'], '/') - self.assertEqual(environ['QUERY_STRING'], 'abc') - self.assertEqual(environ['REMOTE_ADDR'], '192.168.1.1') - self.assertEqual(environ['CONTENT_TYPE'], 'abc') - self.assertEqual(environ['CONTENT_LENGTH'], '10') - self.assertEqual(environ['HTTP_X_FOO'], 'BAR') - self.assertEqual(environ['wsgi.version'], (1, 0)) - self.assertEqual(environ['wsgi.url_scheme'], 'https') - self.assertEqual(environ['wsgi.errors'], sys.stderr) - self.assertEqual(environ['wsgi.multithread'], True) - self.assertEqual(environ['wsgi.multiprocess'], False) - self.assertEqual(environ['wsgi.run_once'], False) - self.assertEqual(environ['wsgi.input'], 'stream') - self.assertEqual(environ['wsgi.input_terminated'], True) - self.assertEqual(inst.environ, environ) - - def test_get_environment_values_w_bogus_scheme_override(self): - inst = self._makeOne() - inst.channel.addr = ['192.168.1.1', 80] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_headers = {'x-forwarded-proto'} - request = DummyParser() - request.headers = { - 'CONTENT_TYPE': 'abc', - 'CONTENT_LENGTH': '10', - 'X_FOO': 'BAR', - 'X_FORWARDED_PROTO': 'http://p02n3e.com?url=http', - 'CONNECTION': 'close', - } - request.query = 'abc' - inst.request = request - self.assertRaises(ValueError, inst.get_environment) - - def test_get_environment_warning_other_proxy_headers(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - inst.request.headers = { - 'X_FORWARDED_FOR': '[2001:db8::1]', - 'FORWARDED': 'For=198.51.100.2;host=example.com:8080;proto=https' - } - inst.channel.addr = ['192.168.1.1', 80] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_count = 1 - inst.channel.server.adj.trusted_proxy_headers = {'forwarded'} - inst.channel.server.adj.log_untrusted_proxy_headers = True - environ = inst.get_environment() - - self.assertEqual(len(inst.logger.logged), 1) - self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_get_environment_contains_all_headers_including_untrusted(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - inst.request.headers = { - 'X_FORWARDED_FOR': '198.51.100.2', - 'X_FORWARDED_BY': 'Waitress', - 'X_FORWARDED_PROTO': 'https', - 'X_FORWARDED_HOST': 'example.org', - } - headers_orig = inst.request.headers.copy() - inst.channel.addr = ['192.168.1.1', 80] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_count = 1 - inst.channel.server.adj.trusted_proxy_headers = {'x-forwarded-by'} - inst.channel.server.adj.clear_untrusted_proxy_headers = False - environ = inst.get_environment() - - for k, expected in headers_orig.items(): - result = environ['HTTP_%s' % k] - self.assertEqual(result, expected) - - def test_get_environment_contains_only_trusted_headers(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - inst.request.headers = { - 'X_FORWARDED_FOR': '198.51.100.2', - 'X_FORWARDED_BY': 'Waitress', - 'X_FORWARDED_PROTO': 'https', - 'X_FORWARDED_HOST': 'example.org', - } - inst.channel.addr = ['192.168.1.1', 80] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_count = 1 - inst.channel.server.adj.trusted_proxy_headers = {'x-forwarded-by'} - inst.channel.server.adj.clear_untrusted_proxy_headers = True - environ = inst.get_environment() - - self.assertEqual(environ['HTTP_X_FORWARDED_BY'], 'Waitress') - self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) - self.assertNotIn('HTTP_X_FORWARDED_PROTO', environ) - self.assertNotIn('HTTP_X_FORWARDED_HOST', environ) - - def test_get_environment_clears_headers_if_untrusted_proxy(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - inst.request.headers = { - 'X_FORWARDED_FOR': '198.51.100.2', - 'X_FORWARDED_BY': 'Waitress', - 'X_FORWARDED_PROTO': 'https', - 'X_FORWARDED_HOST': 'example.org', - } - inst.channel.addr = ['192.168.1.255', 80] - inst.channel.server.adj.trusted_proxy = '192.168.1.1' - inst.channel.server.adj.trusted_proxy_count = 1 - inst.channel.server.adj.trusted_proxy_headers = {'x-forwarded-by'} - inst.channel.server.adj.clear_untrusted_proxy_headers = True - environ = inst.get_environment() - - self.assertNotIn('HTTP_X_FORWARDED_BY', environ) - self.assertNotIn('HTTP_X_FORWARDED_FOR', environ) - self.assertNotIn('HTTP_X_FORWARDED_PROTO', environ) - self.assertNotIn('HTTP_X_FORWARDED_HOST', environ) - - def test_parse_proxy_headers_forwarded_for(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-for'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '192.0.2.1') - - def test_parse_proxy_headers_forwarded_for_v6_missing_brackets(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '2001:db8::0' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-for'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::0') - - def test_parse_proxy_headers_forwared_for_multiple(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'x-forwarded-for'} - ) - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - - def test_parse_forwarded_multiple_proxies_trust_only_two(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For=192.0.2.1;host=fake.com, For=198.51.100.2;host=example.com:8080, For=203.0.113.1' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - - def test_parse_forwarded_multiple_proxies(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'for="[2001:db8::1]:3821";host="example.com:8443";proto="https", for=192.0.2.1;host="example.internal:8080"' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::1') - self.assertEqual(environ['REMOTE_PORT'], '3821') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8443') - self.assertEqual(environ['SERVER_PORT'], '8443') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_parse_forwarded_multiple_proxies_minimal(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'for="[2001:db8::1]";proto="https", for=192.0.2.1;host="example.org"' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '2001:db8::1') - self.assertEqual(environ['SERVER_NAME'], 'example.org') - self.assertEqual(environ['HTTP_HOST'], 'example.org') - self.assertEqual(environ['SERVER_PORT'], '443') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_parse_proxy_headers_forwarded_host_with_port(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com:8080', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - - def test_parse_proxy_headers_forwarded_host_without_port(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com') - self.assertEqual(environ['SERVER_PORT'], '80') - - def test_parse_proxy_headers_forwarded_host_with_forwarded_port(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com', - 'X_FORWARDED_PORT': '8080' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', 'x-forwarded-port'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - - def test_parse_proxy_headers_forwarded_host_multiple_with_forwarded_port(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com, example.org', - 'X_FORWARDED_PORT': '8080' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=2, - trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', 'x-forwarded-port'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - - def test_parse_proxy_headers_forwarded_host_multiple_with_forwarded_port_limit_one_trusted(self): - inst = self._makeOne() - - headers = { - 'X_FORWARDED_FOR': '192.0.2.1, 198.51.100.2, 203.0.113.1', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com, example.org', - 'X_FORWARDED_PORT': '8080' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', 'x-forwarded-port'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '203.0.113.1') - self.assertEqual(environ['SERVER_NAME'], 'example.org') - self.assertEqual(environ['HTTP_HOST'], 'example.org:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - - def test_parse_forwarded(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For=198.51.100.2:5858;host=example.com:8080;proto=https' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['REMOTE_PORT'], '5858') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_parse_forwarded_empty_pair(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For=198.51.100.2;;proto=https;by=_unused' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - - def test_parse_forwarded_pair_token_whitespace(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For=198.51.100.2; proto =https' - } - environ = {} - - with self.assertRaises(ValueError): - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - def test_parse_forwarded_pair_value_whitespace(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For= "198.51.100.2"; proto =https' - } - environ = {} - - with self.assertRaises(ValueError): - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - def test_parse_forwarded_pair_no_equals(self): - inst = self._makeOne() - - headers = { - 'FORWARDED': 'For' - } - environ = {} - - with self.assertRaises(ValueError): - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - def test_parse_forwarded_warning_unknown_token(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'FORWARDED': 'For=198.51.100.2;host=example.com:8080;proto=https;unknown="yolo"' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'forwarded'} - ) - - self.assertEqual(len(inst.logger.logged), 1) - self.assertIn('Unknown Forwarded token', inst.logger.logged[0]) - - self.assertEqual(environ['REMOTE_ADDR'], '198.51.100.2') - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:8080') - self.assertEqual(environ['SERVER_PORT'], '8080') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_parse_no_valid_proxy_headers(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'X_FORWARDED_FOR': '198.51.100.2', - 'FORWARDED': 'For=198.51.100.2;host=example.com:8080;proto=https' - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - ) - - self.assertEqual(environ, {}) - - def test_parse_multiple_x_forwarded_proto(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'X_FORWARDED_PROTO': 'http, https', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-proto'} - ) - - self.assertEqual(environ, {}) - self.assertEqual(len(inst.logger.logged), 1) - self.assertIn("Found multiple values in X-Forwarded-Proto", inst.logger.logged[0]) - - def test_parse_multiple_x_forwarded_port(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'X_FORWARDED_PORT': '443, 80', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-port'} - ) - - self.assertEqual(environ, {}) - self.assertEqual(len(inst.logger.logged), 1) - self.assertIn("Found multiple values in X-Forwarded-Port", inst.logger.logged[0]) - - def test_parse_forwarded_port_wrong_proto_port_80(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'X_FORWARDED_PORT': '80', - 'X_FORWARDED_PROTO': 'https', - 'X_FORWARDED_HOST': 'example.com', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-port', 'x-forwarded-host', 'x-forwarded-proto'} - ) - - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:80') - self.assertEqual(environ['SERVER_PORT'], '80') - self.assertEqual(environ['wsgi.url_scheme'], 'https') - - def test_parse_forwarded_port_wrong_proto_port_443(self): - inst = self._makeOne() - inst.logger = DummyLogger() - - headers = { - 'X_FORWARDED_PORT': '443', - 'X_FORWARDED_PROTO': 'http', - 'X_FORWARDED_HOST': 'example.com', - } - environ = {} - inst.parse_proxy_headers( - environ, - headers, - trusted_proxy_count=1, - trusted_proxy_headers={'x-forwarded-port', 'x-forwarded-host', 'x-forwarded-proto'} - ) - - self.assertEqual(environ['SERVER_NAME'], 'example.com') - self.assertEqual(environ['HTTP_HOST'], 'example.com:443') - self.assertEqual(environ['SERVER_PORT'], '443') - self.assertEqual(environ['wsgi.url_scheme'], 'http') - class TestErrorTask(unittest.TestCase): @@ -1375,10 +778,17 @@ class TestErrorTask(unittest.TestCase): channel = DummyChannel() if request is None: request = DummyParser() - request.error = DummyError() + request.error = self._makeDummyError() from waitress.task import ErrorTask return ErrorTask(channel, request) + def _makeDummyError(self): + from waitress.utilities import Error + e = Error('body') + e.code = 432 + e.reason = 'Too Ugly' + return e + def test_execute_http_10(self): inst = self._makeOne() inst.execute() @@ -1442,12 +852,6 @@ class TestErrorTask(unittest.TestCase): self.assertEqual(lines[6], b'body') self.assertEqual(lines[7], b'(generated by waitress)') - -class DummyError(object): - code = '432' - reason = 'Too Ugly' - body = 'body' - class DummyTask(object): serviced = False cancelled = False @@ -1464,11 +868,6 @@ class DummyAdj(object): host = '127.0.0.1' port = 80 url_prefix = '' - trusted_proxy = None - trusted_proxy_count = 1 - trusted_proxy_headers = set() - log_untrusted_proxy_headers = True - clear_untrusted_proxy_headers = True class DummyServer(object): server_name = 'localhost' diff --git a/waitress/utilities.py b/waitress/utilities.py index a21ca4e..f06bdc9 100644 --- a/waitress/utilities.py +++ b/waitress/utilities.py @@ -21,20 +21,10 @@ import os import re import stat import time -from collections import namedtuple logger = logging.getLogger('waitress') queue_logger = logging.getLogger('waitress.queue') -PROXY_HEADERS = frozenset({ - 'X_FORWARDED_FOR', - 'X_FORWARDED_HOST', - 'X_FORWARDED_PROTO', - 'X_FORWARDED_PORT', - 'X_FORWARDED_BY', - 'FORWARDED', -}) - def find_double_newline(s): """Returns the position just after a double newline in the given string.""" pos1 = s.find(b'\n\r\n') # One kind of double newline @@ -119,17 +109,19 @@ def unpack_rfc822(m): # rfc850 format rfc850_date = join( - [concat(long_day_reg, ','), - join( - [group('[0-9][0-9]?'), - months_reg, - group('[0-9]+') - ], - '-' - ), - hms_reg, - 'gmt' - ], + [ + concat(long_day_reg, ','), + join( + [ + group('[0-9][0-9]?'), + months_reg, + group('[0-9]+') + ], + '-' + ), + hms_reg, + 'gmt' + ], ' ' ) @@ -180,42 +172,6 @@ def parse_http_date(d): return retval -def cleanup_unix_socket(path): - try: - st = os.stat(path) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise # pragma: no cover - else: - if stat.S_ISSOCK(st.st_mode): - try: - os.remove(path) - except OSError: # pragma: no cover - # avoid race condition error during tests - pass - -class Error(object): - - def __init__(self, body): - self.body = body - -class BadRequest(Error): - code = 400 - reason = 'Bad Request' - -class RequestHeaderFieldsTooLarge(BadRequest): - code = 431 - reason = 'Request Header Fields Too Large' - -class RequestEntityTooLarge(BadRequest): - code = 413 - reason = 'Request Entity Too Large' - -class InternalServerError(Error): - code = 500 - reason = 'Internal Server Error' - - # RFC 5234 Appendix B.1 "Core Rules": # VCHAR = %x21-7E # ; visible (printing) characters @@ -259,25 +215,50 @@ def undquote(value): raise ValueError('Invalid quoting in value') -Forwarded = namedtuple('Forwarded', ['by', 'for_', 'host', 'proto']) - - -def clear_untrusted_headers( - headers, untrusted_headers, log_warning=False, logger=logger -): - untrusted_headers_removed = [ - header - for header in untrusted_headers - if headers.pop(header, False) is not False - ] - - if log_warning and untrusted_headers_removed: - untrusted_headers_removed = [ - "-".join([x.capitalize() for x in header.split("_")]) - for header in untrusted_headers_removed - ] - logger.warning( - "Removed untrusted headers (%s). Waitress recommends these be " - "removed upstream.", - ", ".join(untrusted_headers_removed), - ) +def cleanup_unix_socket(path): + try: + st = os.stat(path) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise # pragma: no cover + else: + if stat.S_ISSOCK(st.st_mode): + try: + os.remove(path) + except OSError: # pragma: no cover + # avoid race condition error during tests + pass + +class Error(object): + + def __init__(self, body): + self.body = body + + def to_response(self): + status = '%s %s' % (self.code, self.reason) + body = '%s\r\n\r\n%s' % (self.reason, self.body) + tag = '\r\n\r\n(generated by waitress)' + body = body + tag + headers = [('Content-Type', 'text/plain')] + return status, headers, body + + def wsgi_response(self, environ, start_response): + status, headers, body = self.to_response() + start_response(status, headers) + yield body + +class BadRequest(Error): + code = 400 + reason = 'Bad Request' + +class RequestHeaderFieldsTooLarge(BadRequest): + code = 431 + reason = 'Request Header Fields Too Large' + +class RequestEntityTooLarge(BadRequest): + code = 413 + reason = 'Request Entity Too Large' + +class InternalServerError(Error): + code = 500 + reason = 'Internal Server Error' |