From f728027bed59d08a6491ae9c14d2f5968f8d6fa3 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 21 May 2015 22:44:36 -0700 Subject: Accept gzip-encoded API responses Previously, we would accept gzip-encoded responses, but only because we were letting requests decode *all* responses (even object data). This restores the previous capability, but with tighter controls about which requests will accept gzipped responses and where the decoding happens. Change-Id: I4fd8b97207b9ab01b1bcf825cc16efd8ad46344a Related-Bug: 1282861 Related-Bug: 1338464 --- tests/unit/test_swiftclient.py | 21 ++++++++++++++++++--- tests/unit/test_utils.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 0095447..da43510 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -581,6 +581,7 @@ class TestGetAccount(MockHttpTest): self.assertEqual(value, []) self.assertRequests([ ('GET', '/v1/acct?format=json', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -591,6 +592,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') self.assertRequests([ ('GET', '/v1/acct?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -601,6 +603,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) self.assertRequests([ ('GET', '/v1/acct?format=json&limit=10', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -611,6 +614,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') self.assertRequests([ ('GET', '/v1/acct?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -622,6 +626,7 @@ class TestGetAccount(MockHttpTest): end_marker='end_marker') self.assertRequests([ ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -700,6 +705,7 @@ class TestGetContainer(MockHttpTest): self.assertEqual(value, []) self.assertRequests([ ('GET', '/v1/acct/container?format=json', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -711,6 +717,7 @@ class TestGetContainer(MockHttpTest): marker='marker') self.assertRequests([ ('GET', '/v1/acct/container?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -722,6 +729,7 @@ class TestGetContainer(MockHttpTest): limit=10) self.assertRequests([ ('GET', '/v1/acct/container?format=json&limit=10', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -733,6 +741,7 @@ class TestGetContainer(MockHttpTest): prefix='asdf/') self.assertRequests([ ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -744,6 +753,7 @@ class TestGetContainer(MockHttpTest): delimiter='/') self.assertRequests([ ('GET', '/v1/acct/container?format=json&delimiter=/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -755,7 +765,7 @@ class TestGetContainer(MockHttpTest): end_marker='end_marker') self.assertRequests([ ('GET', '/v1/acct/container?format=json&end_marker=end_marker', - '', {'x-auth-token': 'token'}), + '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}), ]) def test_param_path(self): @@ -766,6 +776,7 @@ class TestGetContainer(MockHttpTest): path='asdf') self.assertRequests([ ('GET', '/v1/acct/container?format=json&path=asdf', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -780,6 +791,7 @@ class TestGetContainer(MockHttpTest): ('GET', '/container?format=json', '', { 'x-auth-token': 'TOKEN', 'x-client-key': 'client key', + 'accept-encoding': 'gzip', }), ]) @@ -790,6 +802,7 @@ class TestGetContainer(MockHttpTest): query_string="hello=20") self.assertRequests([ ('GET', '/asdf?format=json&hello=20', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -1583,7 +1596,7 @@ class TestGetCapabilities(MockHttpTest): http_conn = conn('http://www.test.com/info') info = c.get_capabilities(http_conn) self.assertRequests([ - ('GET', '/info', '', {}), + ('GET', '/info', '', {'Accept-Encoding': 'gzip'}), ]) self.assertEqual(info, {}) self.assertTrue(http_conn[1].resp.has_been_read) @@ -1619,7 +1632,8 @@ class TestGetCapabilities(MockHttpTest): ('GET', '/auth/v1.0', '', { 'x-auth-user': 'user', 'x-auth-key': 'key'}), - ('GET', 'http://storage.example.com/info', '', {}), + ('GET', 'http://storage.example.com/info', '', { + 'accept-encoding': 'gzip'}), ]) def test_conn_get_capabilities_with_os_auth(self): @@ -2341,6 +2355,7 @@ class TestConnection(MockHttpTest): ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', + 'accept-encoding': 'gzip', }), ]) self.assertEqual(conn.attempts, 1) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0f210a3..f0de79c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import gzip import unittest import mock import six @@ -394,3 +395,39 @@ class TestGroupers(unittest.TestCase): result = list(u.n_groups(range(100), 12)) self.assertEqual([9] * 11 + [1], list(map(len, result))) + + +class TestApiResponeParser(unittest.TestCase): + + def test_utf8_default(self): + result = u.parse_api_response( + {}, u'{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_bad_json(self): + self.assertRaises(ValueError, u.parse_api_response, + {}, b'{"foo": "bar}') + + def test_bad_utf8(self): + self.assertRaises(UnicodeDecodeError, u.parse_api_response, + {}, b'{"foo": "b\xffr"}') + + def test_latin_1(self): + result = u.parse_api_response( + {'content-type': 'application/json; charset=iso8859-1'}, + b'{"t\xe9st": "\xff"}') + self.assertEqual({u't\xe9st': u'\xff'}, result) + + def test_gzipped_utf8(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) -- cgit v1.2.1