Properly handle unicode headers.

Fix unicode handling in Python 3 and Python 2. There are currently two
failure modes. In python 2, swiftclient fails to log in debug mode if
the account name has a non-ASCII character. This is because the account
name will appear in the storage URL, which we attempt to pass to the
logger as a byte string (whereas it should be a unicode string). This
patch changes the behavior to convert the path strings into unicode by
calling the parse_header_string() function.

The second failure mode is with Python 3, where http_lib returns headers
that are latin-1 encoded, but swiftclient expects UTF-8. The patch
automatically converts headers from latin-1 (iso-8859-1) to UTF-8, so
that we can properly handle non-ASCII headers in responses.

Change-Id: Ifa7f3d5af71bde8127129f1f8603772d80d063c1

2 files changed, 85 insertions, 5 deletions

diff --git a/swiftclient/client.py b/swiftclient/client.py
index 8cbdf45..a518d32 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -151,7 +151,7 @@ def http_log(args, kwargs, resp, body):
         elif element in ('GET', 'POST', 'PUT'):
             string_parts.append(' -X %s' % element)
         else:
-            string_parts.append(' %s' % element)
+            string_parts.append(' %s' % parse_header_string(element))
     if 'headers' in kwargs:
         headers = scrub_headers(kwargs['headers'])
         for element in headers:
@@ -455,11 +455,23 @@ class HTTPConnection(object):
         self.resp.status = self.resp.status_code
         old_getheader = self.resp.raw.getheader
 
+        def _decode_header(string):
+            if string is None or six.PY2:
+                return string
+            return string.encode('iso-8859-1').decode('utf-8')
+
+        def _encode_header(string):
+            if string is None or six.PY2:
+                return string
+            return string.encode('utf-8').decode('iso-8859-1')
+
         def getheaders():
-            return self.resp.headers.items()
+            return [(_decode_header(k), _decode_header(v))
+                    for k, v in self.resp.headers.items()]
 
         def getheader(k, v=None):
-            return old_getheader(k.lower(), v)
+            return _decode_header(old_getheader(
+                _encode_header(k.lower()), _encode_header(v)))
 
         def releasing_read(*args, **kwargs):
             chunk = self.resp.raw.read(*args, **kwargs)
@@ -513,8 +525,11 @@ def get_auth_1_0(url, user, key, snet, **kwargs):
         netloc = parsed[1]
         parsed[1] = 'snet-' + netloc
         url = urlunparse(parsed)
-    return url, resp.getheader('x-storage-token',
-                               resp.getheader('x-auth-token'))
+
+    auth_token = resp.getheader('x-auth-token')
+    if auth_token is not None:
+        auth_token = parse_header_string(auth_token)
+    return url, resp.getheader('x-storage-token', auth_token)
 
 
 def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs):
@@ -694,10 +709,14 @@ def get_auth(auth_url, user, key, **kwargs):
         raise ClientException('Unknown auth_version %s specified and no '
                               'session found.' % auth_version)
 
+    if token is not None:
+        token = parse_header_string(token)
     # Override storage url, if necessary
     if os_options.get('object_storage_url'):
         return os_options['object_storage_url'], token
     else:
+        if storage_url is not None:
+            return parse_header_string(storage_url), token
         return storage_url, token
 
 
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index b6d6856..3303372 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -1896,6 +1896,57 @@ class TestHTTPConnection(MockHttpTest):
         self.assertFalse(resp.read())
         self.assertTrue(resp.closed)
 
+    @unittest.skipIf(six.PY3, 'python2 specific test')
+    def test_response_python2_headers(self):
+        '''Test utf-8 headers in Python 2.
+        '''
+        _, conn = c.http_connection(u'http://www.test.com/')
+        conn.resp = MockHttpResponse(
+            status=200,
+            headers={
+                '\xd8\xaa-unicode': '\xd8\xaa-value',
+                'empty-header': ''
+            }
+        )
+
+        resp = conn.getresponse()
+        self.assertEqual(
+            '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode'))
+        self.assertEqual(
+            '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE'))
+        self.assertEqual('', resp.getheader('empty-header'))
+        self.assertEqual(
+            dict([('\xd8\xaa-unicode', '\xd8\xaa-value'),
+                  ('empty-header', ''),
+                  ('etag', '"%s"' % EMPTY_ETAG)]),
+            dict(resp.getheaders()))
+
+    @unittest.skipIf(six.PY2, 'python3 specific test')
+    def test_response_python3_headers(self):
+        '''Test latin1-encoded headers in Python 3.
+        '''
+        _, conn = c.http_connection(u'http://www.test.com/')
+        conn.resp = MockHttpResponse(
+            status=200,
+            headers={
+                b'\xd8\xaa-unicode'.decode('iso-8859-1'):
+                b'\xd8\xaa-value'.decode('iso-8859-1'),
+                'empty-header': ''
+            }
+        )
+
+        resp = conn.getresponse()
+        self.assertEqual(
+            '\u062a-value', resp.getheader('\u062a-unicode'))
+        self.assertEqual(
+            '\u062a-value', resp.getheader('\u062a-UNICODE'))
+        self.assertEqual('', resp.getheader('empty-header'))
+        self.assertEqual(
+            dict([('\u062a-unicode', '\u062a-value'),
+                  ('empty-header', ''),
+                  ('etag', ('"%s"' % EMPTY_ETAG))]),
+            dict(resp.getheaders()))
+
 
 class TestConnection(MockHttpTest):
 
@@ -2839,6 +2890,16 @@ class TestLogging(MockHttpTest):
             self.assertIn('X-Storage-Token', output)
             self.assertIn(unicode_token_value, output)
 
+    @mock.patch('swiftclient.client.logger.debug')
+    def test_unicode_path(self, mock_log):
+        path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8')
+        c.http_log(['GET', path], {},
+                   MockHttpResponse(status=200, headers=[]), '')
+        request_log_line = mock_log.mock_calls[0]
+        self.assertEqual('REQ: %s', request_log_line[1][0])
+        self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'),
+                         request_log_line[1][1])
+
 
 class TestCloseConnection(MockHttpTest):