diff options
Diffstat (limited to 'Lib/http')
| -rw-r--r-- | Lib/http/client.py | 161 | ||||
| -rw-r--r-- | Lib/http/cookiejar.py | 37 | ||||
| -rw-r--r-- | Lib/http/cookies.py | 12 | ||||
| -rw-r--r-- | Lib/http/server.py | 71 |
4 files changed, 162 insertions, 119 deletions
diff --git a/Lib/http/client.py b/Lib/http/client.py index e05c84dab7..6de4b0e73c 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -73,7 +73,6 @@ import os import socket import collections from urllib.parse import urlsplit -import warnings __all__ = ["HTTPResponse", "HTTPConnection", "HTTPException", "NotConnected", "UnknownProtocol", @@ -271,8 +270,6 @@ def parse_headers(fp, _class=HTTPMessage): return email.parser.Parser(_class=_class).parsestr(hstring) -_strict_sentinel = object() - class HTTPResponse(io.RawIOBase): # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details. @@ -282,7 +279,7 @@ class HTTPResponse(io.RawIOBase): # text following RFC 2047. The basic status line parsing only # accepts iso-8859-1. - def __init__(self, sock, debuglevel=0, strict=_strict_sentinel, method=None, url=None): + def __init__(self, sock, debuglevel=0, method=None, url=None): # If the response includes a content-length header, we need to # make sure that the client doesn't read more than the # specified number of bytes. If it does, it will block until @@ -292,10 +289,6 @@ class HTTPResponse(io.RawIOBase): # clients unless they know what they are doing. self.fp = sock.makefile("rb") self.debuglevel = debuglevel - if strict is not _strict_sentinel: - warnings.warn("the 'strict' argument isn't supported anymore; " - "http.client now always assumes HTTP/1.x compliant servers.", - DeprecationWarning, 2) self._method = method # The HTTPResponse object is returned via urllib. The clients @@ -732,13 +725,17 @@ class HTTPConnection: default_port = HTTP_PORT auto_open = 1 debuglevel = 0 - - def __init__(self, host, port=None, strict=_strict_sentinel, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): - if strict is not _strict_sentinel: - warnings.warn("the 'strict' argument isn't supported anymore; " - "http.client now always assumes HTTP/1.x compliant servers.", - DeprecationWarning, 2) + # TCP Maximum Segment Size (MSS) is determined by the TCP stack on + # a per-connection basis. There is no simple and efficient + # platform independent mechanism for determining the MSS, so + # instead a reasonable estimate is chosen. The getsockopt() + # interface using the TCP_MAXSEG parameter may be a suitable + # approach on some operating systems. A value of 16KiB is chosen + # as a reasonable estimate of the maximum MSS. + mss = 16384 + + def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None): self.timeout = timeout self.source_address = source_address self.sock = None @@ -750,22 +747,37 @@ class HTTPConnection: self._tunnel_port = None self._tunnel_headers = {} - self._set_hostport(host, port) + (self.host, self.port) = self._get_hostport(host, port) + + # This is stored as an instance variable to allow unit + # tests to replace it with a suitable mockup + self._create_connection = socket.create_connection def set_tunnel(self, host, port=None, headers=None): - """ Sets up the host and the port for the HTTP CONNECT Tunnelling. + """Set up host and port for HTTP CONNECT tunnelling. - The headers argument should be a mapping of extra HTTP headers - to send with the CONNECT request. + In a connection that uses HTTP CONNECT tunneling, the host passed to the + constructor is used as a proxy server that relays all communication to + the endpoint passed to `set_tunnel`. This done by sending an HTTP + CONNECT request to the proxy server when the connection is established. + + This method must be called before the HTML connection has been + established. + + The headers argument should be a mapping of extra HTTP headers to send + with the CONNECT request. """ - self._tunnel_host = host - self._tunnel_port = port + + if self.sock: + raise RuntimeError("Can't set up tunnel for established connection") + + self._tunnel_host, self._tunnel_port = self._get_hostport(host, port) if headers: self._tunnel_headers = headers else: self._tunnel_headers.clear() - def _set_hostport(self, host, port): + def _get_hostport(self, host, port): if port is None: i = host.rfind(':') j = host.rfind(']') # ipv6 addresses have [...] @@ -782,15 +794,15 @@ class HTTPConnection: port = self.default_port if host and host[0] == '[' and host[-1] == ']': host = host[1:-1] - self.host = host - self.port = port + + return (host, port) def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - self._set_hostport(self._tunnel_host, self._tunnel_port) - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port) + connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, + self._tunnel_port) connect_bytes = connect_str.encode("ascii") self.send(connect_bytes) for header, value in self._tunnel_headers.items(): @@ -804,8 +816,8 @@ class HTTPConnection: if code != 200: self.close() - raise socket.error("Tunnel connection failed: %d %s" % (code, - message.strip())) + raise OSError("Tunnel connection failed: %d %s" % (code, + message.strip())) while True: line = response.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -818,8 +830,9 @@ class HTTPConnection: def connect(self): """Connect to the host and port specified in __init__.""" - self.sock = socket.create_connection((self.host,self.port), - self.timeout, self.source_address) + self.sock = self._create_connection((self.host,self.port), + self.timeout, self.source_address) + if self._tunnel_host: self._tunnel() @@ -899,8 +912,11 @@ class HTTPConnection: del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction - # between delayed ack and the Nagle algorithm. - if isinstance(message_body, bytes): + # between delayed ack and the Nagle algorithm. However, + # there is no performance gain if the message is larger + # than MSS (and there is a memory penalty for the message + # copy). + if isinstance(message_body, bytes) and len(message_body) < self.mss: msg += message_body message_body = None self.send(msg) @@ -985,22 +1001,29 @@ class HTTPConnection: netloc_enc = netloc.encode("idna") self.putheader('Host', netloc_enc) else: + if self._tunnel_host: + host = self._tunnel_host + port = self._tunnel_port + else: + host = self.host + port = self.port + try: - host_enc = self.host.encode("ascii") + host_enc = host.encode("ascii") except UnicodeEncodeError: - host_enc = self.host.encode("idna") + host_enc = host.encode("idna") # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - if self.host.find(':') >= 0: + if host.find(':') >= 0: host_enc = b'[' + host_enc + b']' - if self.port == self.default_port: + if port == self.default_port: self.putheader('Host', host_enc) else: host_enc = host_enc.decode("ascii") - self.putheader('Host', "%s:%s" % (host_enc, self.port)) + self.putheader('Host', "%s:%s" % (host_enc, port)) # note: we are assuming that clients will not attempt to set these # headers since *this* library must deal with the @@ -1144,18 +1167,22 @@ class HTTPConnection: else: response = self.response_class(self.sock, method=self._method) - response.begin() - assert response.will_close != _UNKNOWN - self.__state = _CS_IDLE + try: + response.begin() + assert response.will_close != _UNKNOWN + self.__state = _CS_IDLE - if response.will_close: - # this effectively passes the connection to the response - self.close() - else: - # remember this, so we can tell when it is complete - self.__response = response + if response.will_close: + # this effectively passes the connection to the response + self.close() + else: + # remember this, so we can tell when it is complete + self.__response = response - return response + return response + except: + response.close() + raise try: import ssl @@ -1170,20 +1197,19 @@ else: # XXX Should key_file and cert_file be deprecated in favour of context? def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=_strict_sentinel, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, *, context=None, check_hostname=None): - super(HTTPSConnection, self).__init__(host, port, strict, timeout, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, *, context=None, + check_hostname=None): + super(HTTPSConnection, self).__init__(host, port, timeout, source_address) self.key_file = key_file self.cert_file = cert_file if context is None: - # Some reasonable defaults - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.options |= ssl.OP_NO_SSLv2 + context = ssl._create_default_https_context() will_verify = context.verify_mode != ssl.CERT_NONE if check_hostname is None: - check_hostname = will_verify - elif check_hostname and not will_verify: + check_hostname = context.check_hostname + if check_hostname and not will_verify: raise ValueError("check_hostname needs a SSL context with " "either CERT_OPTIONAL or CERT_REQUIRED") if key_file or cert_file: @@ -1194,23 +1220,22 @@ else: def connect(self): "Connect to a host on a given (SSL) port." - sock = socket.create_connection((self.host, self.port), - self.timeout, self.source_address) + super().connect() if self._tunnel_host: - self.sock = sock - self._tunnel() + server_hostname = self._tunnel_host + else: + server_hostname = self.host - server_hostname = self.host if ssl.HAS_SNI else None - self.sock = self._context.wrap_socket(sock, + self.sock = self._context.wrap_socket(self.sock, server_hostname=server_hostname) - try: - if self._check_hostname: - ssl.match_hostname(self.sock.getpeercert(), self.host) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise + if not self._context.check_hostname and self._check_hostname: + try: + ssl.match_hostname(self.sock.getpeercert(), server_hostname) + except Exception: + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + raise __all__.append("HTTPSConnection") diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 9fcd4c6f46..7e4982f3e4 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1193,8 +1193,7 @@ def deepvalues(mapping): pass else: mapping = True - for subobj in deepvalues(obj): - yield subobj + yield from deepvalues(obj) if not mapping: yield obj @@ -1723,16 +1722,16 @@ class CookieJar: def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) -# derives from IOError for backwards-compatibility with Python 2.4.0 -class LoadError(IOError): pass +# derives from OSError for backwards-compatibility with Python 2.4.0 +class LoadError(OSError): pass class FileCookieJar(CookieJar): """CookieJar that can be loaded from and saved to a file.""" @@ -1762,17 +1761,14 @@ class FileCookieJar(CookieJar): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - f = open(filename) - try: + with open(filename) as f: self._really_load(f, filename, ignore_discard, ignore_expires) - finally: - f.close() def revert(self, filename=None, ignore_discard=False, ignore_expires=False): """Clear all cookies and reload cookies from a saved file. - Raises LoadError (or IOError) if reversion is not successful; the + Raises LoadError (or OSError) if reversion is not successful; the object's state will not be altered if this happens. """ @@ -1787,7 +1783,7 @@ class FileCookieJar(CookieJar): self._cookies = {} try: self.load(filename, ignore_discard, ignore_expires) - except (LoadError, IOError): + except OSError: self._cookies = old_state raise @@ -1796,7 +1792,7 @@ class FileCookieJar(CookieJar): def lwp_cookie_str(cookie): - """Return string representation of Cookie in an the LWP cookie file format. + """Return string representation of Cookie in the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. @@ -1857,15 +1853,12 @@ class LWPCookieJar(FileCookieJar): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - f = open(filename, "w") - try: + with open(filename, "w") as f: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. f.write("#LWP-Cookies-2.0\n") f.write(self.as_lwp_str(ignore_discard, ignore_expires)) - finally: - f.close() def _really_load(self, f, filename, ignore_discard, ignore_expires): magic = f.readline() @@ -1938,8 +1931,7 @@ class LWPCookieJar(FileCookieJar): if not ignore_expires and c.is_expired(now): continue self.set_cookie(c) - - except IOError: + except OSError: raise except Exception: _warn_unhandled_exception() @@ -2045,7 +2037,7 @@ class MozillaCookieJar(FileCookieJar): continue self.set_cookie(c) - except IOError: + except OSError: raise except Exception: _warn_unhandled_exception() @@ -2057,8 +2049,7 @@ class MozillaCookieJar(FileCookieJar): if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - f = open(filename, "w") - try: + with open(filename, "w") as f: f.write(self.header) now = time.time() for cookie in self: @@ -2087,5 +2078,3 @@ class MozillaCookieJar(FileCookieJar): "\t".join([cookie.domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") - finally: - f.close() diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 556d101fb0..3e1abd7ecf 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -330,8 +330,8 @@ class Morsel(dict): "comment" : "Comment", "domain" : "Domain", "max-age" : "Max-Age", - "secure" : "secure", - "httponly" : "httponly", + "secure" : "Secure", + "httponly" : "HttpOnly", "version" : "Version", } @@ -486,8 +486,12 @@ class BaseCookie(dict): def __setitem__(self, key, value): """Dictionary style assignment.""" - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + if isinstance(value, Morsel): + # allow assignment of constructed Morsels (e.g. for pickling) + dict.__setitem__(self, key, value) + else: + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): """Return a string suitable for HTTP.""" diff --git a/Lib/http/server.py b/Lib/http/server.py index 7d3b506075..cfa29f44d3 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -85,8 +85,6 @@ __version__ = "0.6" __all__ = ["HTTPServer", "BaseHTTPRequestHandler"] import html -import email.message -import email.parser import http.client import io import mimetypes @@ -401,12 +399,17 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): while not self.close_connection: self.handle_one_request() - def send_error(self, code, message=None): + def send_error(self, code, message=None, explain=None): """Send and log an error reply. - Arguments are the error code, and a detailed message. - The detailed message defaults to the short entry matching the - response code. + Arguments are + * code: an HTTP error code + 3 digits + * message: a simple optional 1 line reason phrase. + *( HTAB / SP / VCHAR / %x80-FF ) + defaults to short entry matching the response code + * explain: a detailed message defaults to the long entry + matching the response code. This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends @@ -420,17 +423,20 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): shortmsg, longmsg = '???', '???' if message is None: message = shortmsg - explain = longmsg + if explain is None: + explain = longmsg self.log_error("code %d, message %s", code, message) # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': explain}) + {'code': code, 'message': _quote_html(message), 'explain': _quote_html(explain)}) + body = content.encode('UTF-8', 'replace') self.send_response(code, message) self.send_header("Content-Type", self.error_content_type) self.send_header('Connection', 'close') + self.send_header('Content-Length', int(len(body))) self.end_headers() if self.command != 'HEAD' and code >= 200 and code not in (204, 304): - self.wfile.write(content.encode('UTF-8', 'replace')) + self.wfile.write(body) def send_response(self, code, message=None): """Add the response header to the headers buffer and log the @@ -695,10 +701,14 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): path = self.translate_path(self.path) f = None if os.path.isdir(path): - if not self.path.endswith('/'): + parts = urllib.parse.urlsplit(self.path) + if not parts.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) - self.send_header("Location", self.path + "/") + new_parts = (parts[0], parts[1], parts[2] + '/', + parts[3], parts[4]) + new_url = urllib.parse.urlunsplit(new_parts) + self.send_header("Location", new_url) self.end_headers() return None for index in "index.html", "index.htm": @@ -711,7 +721,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): ctype = self.guess_type(path) try: f = open(path, 'rb') - except IOError: + except OSError: self.send_error(404, "File not found") return None try: @@ -736,12 +746,17 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """ try: list = os.listdir(path) - except os.error: + except OSError: self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) r = [] - displaypath = html.escape(urllib.parse.unquote(self.path)) + try: + displaypath = urllib.parse.unquote(self.path, + errors='surrogatepass') + except UnicodeDecodeError: + displaypath = urllib.parse.unquote(path) + displaypath = html.escape(displaypath) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" ' @@ -763,9 +778,11 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): displayname = name + "@" # Note: a link to a directory displays with @ and links with / r.append('<li><a href="%s">%s</a></li>' - % (urllib.parse.quote(linkname), html.escape(displayname))) + % (urllib.parse.quote(linkname, + errors='surrogatepass'), + html.escape(displayname))) r.append('</ul>\n<hr>\n</body>\n</html>\n') - encoded = '\n'.join(r).encode(enc) + encoded = '\n'.join(r).encode(enc, 'surrogateescape') f = io.BytesIO() f.write(encoded) f.seek(0) @@ -788,7 +805,11 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): path = path.split('#',1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 trailing_slash = path.rstrip().endswith('/') - path = posixpath.normpath(urllib.parse.unquote(path)) + try: + path = urllib.parse.unquote(path, errors='surrogatepass') + except UnicodeDecodeError: + path = urllib.parse.unquote(path) + path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) path = os.getcwd() @@ -1130,7 +1151,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): try: try: os.setuid(nobody) - except os.error: + except OSError: pass os.dup2(self.rfile.fileno(), 0) os.dup2(self.wfile.fileno(), 1) @@ -1183,15 +1204,15 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): self.log_message("CGI script exited OK") -def test(HandlerClass = BaseHTTPRequestHandler, - ServerClass = HTTPServer, protocol="HTTP/1.0", port=8000): +def test(HandlerClass=BaseHTTPRequestHandler, + ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the first command line argument). """ - server_address = ('', port) + server_address = (bind, port) HandlerClass.protocol_version = protocol httpd = ServerClass(server_address, HandlerClass) @@ -1209,12 +1230,16 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--cgi', action='store_true', help='Run as CGI Server') + parser.add_argument('--bind', '-b', default='', metavar='ADDRESS', + help='Specify alternate bind address ' + '[default: all interfaces]') parser.add_argument('port', action='store', default=8000, type=int, nargs='?', help='Specify alternate port [default: 8000]') args = parser.parse_args() if args.cgi: - test(HandlerClass=CGIHTTPRequestHandler, port=args.port) + handler_class = CGIHTTPRequestHandler else: - test(HandlerClass=SimpleHTTPRequestHandler, port=args.port) + handler_class = SimpleHTTPRequestHandler + test(HandlerClass=handler_class, port=args.port, bind=args.bind) |