diff options
| -rw-r--r-- | swift/common/bufferedhttp.py | 11 | ||||
| -rw-r--r-- | test/functional/swift_test_client.py | 88 | ||||
| -rw-r--r-- | test/unit/common/test_bufferedhttp.py | 5 |
3 files changed, 101 insertions, 3 deletions
diff --git a/swift/common/bufferedhttp.py b/swift/common/bufferedhttp.py index 129460913..ecf9b155e 100644 --- a/swift/common/bufferedhttp.py +++ b/swift/common/bufferedhttp.py @@ -35,7 +35,7 @@ import socket import eventlet from eventlet.green.httplib import CONTINUE, HTTPConnection, HTTPMessage, \ HTTPResponse, HTTPSConnection, _UNKNOWN -from six.moves.urllib.parse import quote +from six.moves.urllib.parse import quote, parse_qsl, urlencode import six if six.PY2: @@ -240,6 +240,15 @@ def http_connect_raw(ipaddr, port, method, path, headers=None, else: conn = BufferedHTTPConnection('%s:%s' % (ipaddr, port)) if query_string: + # Round trip to ensure proper quoting + if six.PY2: + query_string = urlencode(parse_qsl( + query_string, keep_blank_values=True)) + else: + query_string = urlencode( + parse_qsl(query_string, keep_blank_values=True, + encoding='latin1'), + encoding='latin1') path += '?' + query_string conn.path = path conn.putrequest(method, path, skip_host=(headers and 'Host' in headers)) diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index 3b68d3a44..9ff8a22ba 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -104,6 +104,91 @@ def listing_items(method): items = [] +def putrequest(self, method, url, skip_host=False, skip_accept_encoding=False): + '''Send a request to the server. + + This is mostly a regurgitation of CPython's HTTPConnection.putrequest, + but fixed up so we can still send arbitrary bytes in the request line + on py3. See also: https://bugs.python.org/issue36274 + + To use, swap out a HTTP(S)Connection's putrequest with something like:: + + conn.putrequest = putrequest.__get__(conn) + + :param method: specifies an HTTP request method, e.g. 'GET'. + :param url: specifies the object being requested, e.g. '/index.html'. + :param skip_host: if True does not add automatically a 'Host:' header + :param skip_accept_encoding: if True does not add automatically an + 'Accept-Encoding:' header + ''' + # (Mostly) inline the HTTPConnection implementation; just fix it + # so we can send non-ascii request lines. For comparison, see + # https://github.com/python/cpython/blob/v2.7.16/Lib/httplib.py#L888-L1003 + # and https://github.com/python/cpython/blob/v3.7.2/ + # Lib/http/client.py#L1061-L1183 + if self._HTTPConnection__response \ + and self._HTTPConnection__response.isclosed(): + self._HTTPConnection__response = None + + if self._HTTPConnection__state == http_client._CS_IDLE: + self._HTTPConnection__state = http_client._CS_REQ_STARTED + else: + raise http_client.CannotSendRequest(self._HTTPConnection__state) + + self._method = method + if not url: + url = '/' + self._path = url + request = '%s %s %s' % (method, url, self._http_vsn_str) + if not isinstance(request, bytes): + # This choice of encoding is the whole reason we copy/paste from + # cpython. When making backend requests, it should never be + # necessary; however, we have some functional tests that want + # to send non-ascii bytes. + # TODO: when https://bugs.python.org/issue36274 is resolved, make + # sure we fix up our API to match whatever upstream chooses to do + self._output(request.encode('latin1')) + else: + self._output(request) + + if self._http_vsn == 11: + if not skip_host: + netloc = '' + if url.startswith('http'): + nil, netloc, nil, nil, nil = urllib.parse.urlsplit(url) + + if netloc: + try: + netloc_enc = netloc.encode("ascii") + except UnicodeEncodeError: + netloc_enc = netloc.encode("idna") + self.putheader('Host', netloc_enc) + else: + if self._tunnel_host: + host = self._tunnel_host + port = self._tunnel_port + else: + host = self.host + port = self.port + + try: + host_enc = host.encode("ascii") + except UnicodeEncodeError: + host_enc = host.encode("idna") + + if host.find(':') >= 0: + host_enc = b'[' + host_enc + b']' + + if port == self.default_port: + self.putheader('Host', host_enc) + else: + host_enc = host_enc.decode("ascii") + self.putheader('Host', "%s:%s" % (host_enc, port)) + + if not skip_accept_encoding: + self.putheader('Accept-Encoding', 'identity') + + class Connection(object): def __init__(self, config): for key in 'auth_host auth_port auth_ssl username password'.split(): @@ -125,6 +210,7 @@ class Connection(object): self.storage_host = None self.storage_port = None self.storage_url = None + self.connection = None # until you call .http_connect()/.put_start() self.conn_class = None @@ -209,6 +295,7 @@ class Connection(object): self.connection = self.conn_class(self.storage_host, port=self.storage_port) # self.connection.set_debuglevel(3) + self.connection.putrequest = putrequest.__get__(self.connection) def make_path(self, path=None, cfg=None): if path is None: @@ -338,6 +425,7 @@ class Connection(object): self.connection = self.conn_class(self.storage_host, port=self.storage_port) # self.connection.set_debuglevel(3) + self.connection.putrequest = putrequest.__get__(self.connection) self.connection.putrequest('PUT', path) for key, value in headers.items(): self.connection.putheader(key, value) diff --git a/test/unit/common/test_bufferedhttp.py b/test/unit/common/test_bufferedhttp.py index 9ea07ec3b..319986dd1 100644 --- a/test/unit/common/test_bufferedhttp.py +++ b/test/unit/common/test_bufferedhttp.py @@ -59,7 +59,8 @@ class TestBufferedHTTP(unittest.TestCase): fp.flush() self.assertEqual( fp.readline(), - 'PUT /dev/%s/path/..%%25/?omg&no=%%7f HTTP/1.1\r\n' % + 'PUT /dev/%s/path/..%%25/?omg=&no=%%7F&%%FF=%%FF' + '&no=%%25ff HTTP/1.1\r\n' % expected_par) headers = {} line = fp.readline() @@ -82,7 +83,7 @@ class TestBufferedHTTP(unittest.TestCase): 'PUT', '/path/..%/', { 'content-length': 7, 'x-header': 'value'}, - query_string='omg&no=%7f') + query_string='omg&no=%7f&\xff=%ff&no=%25ff') conn.send('REQUEST\r\n') self.assertTrue(conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)) |