diff options
Diffstat (limited to 'tests/unit/test_swiftclient.py')
| -rw-r--r-- | tests/unit/test_swiftclient.py | 369 |
1 files changed, 307 insertions, 62 deletions
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 0969f41..a51fe1a 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -17,7 +17,8 @@ import logging import mock import six import socket -import testtools +import string +import unittest import warnings import tempfile from hashlib import md5 @@ -33,7 +34,7 @@ import swiftclient.utils import swiftclient -class TestClientException(testtools.TestCase): +class TestClientException(unittest.TestCase): def test_is_exception(self): self.assertTrue(issubclass(c.ClientException, Exception)) @@ -50,6 +51,7 @@ class TestClientException(testtools.TestCase): 'status', 'reason', 'device', + 'response_content', ) for value in test_kwargs: kwargs = { @@ -58,6 +60,26 @@ class TestClientException(testtools.TestCase): exc = c.ClientException('test', **kwargs) self.assertIn(value, str(exc)) + def test_attrs(self): + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + 'response_headers', + ) + for value in test_kwargs: + key = 'http_%s' % value + kwargs = {key: value} + exc = c.ClientException('test', **kwargs) + self.assertIs(True, hasattr(exc, key)) + self.assertEqual(getattr(exc, key), value) + class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): @@ -250,12 +272,12 @@ class TestGetAuth(MockHttpTest): self.assertEqual(url, 'storageURL') self.assertEqual(token, 'someauthtoken') - e = self.assertRaises(c.ClientException, c.get_auth, - 'http://www.test.com/invalid_cert', - 'asdf', 'asdf', auth_version='1.0') + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') # TODO: this test is really on validating the mock and not the # the full plumbing into the requests's 'verify' option - self.assertIn('invalid_certificate', str(e)) + self.assertIn('invalid_certificate', str(exc_context.exception)) def test_auth_v1_timeout(self): # this test has some overlap with @@ -581,9 +603,12 @@ class TestHeadAccount(MockHttpTest): def test_server_error(self): body = 'c' * 65 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_account, - 'http://www.tests.com', 'asdf') + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -595,6 +620,40 @@ class TestHeadAccount(MockHttpTest): self.assertEqual(e.__str__()[-89:], new_body) +class TestPostAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'X-Account-Meta-Color': 'blue', + }, body='foo') + resp_headers, body = c.post_account( + 'http://www.tests.com/path/to/account', 'asdf', + {'x-account-meta-shape': 'square'}, query_string='bar=baz', + data='some data') + self.assertEqual('blue', resp_headers.get('x-account-meta-color')) + self.assertEqual('foo', body) + self.assertRequests([ + ('POST', 'http://www.tests.com/path/to/account?bar=baz', + 'some data', {'x-auth-token': 'asdf', + 'x-account-meta-shape': 'square'}) + ]) + + def test_server_error(self): + body = 'c' * 65 + c.http_connection = self.fake_http_connection(500, body=body) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) + self.assertRequests([ + ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) + + class TestGetContainer(MockHttpTest): def test_no_content(self): @@ -705,14 +764,18 @@ class TestHeadContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_container, - 'http://www.test.com', 'asdf', 'container') + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception self.assertRequests([ ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), ]) self.assertEqual(e.http_status, 500) self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) class TestPutContainer(MockHttpTest): @@ -729,10 +792,13 @@ class TestPutContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.put_container, - 'http://www.test.com', 'token', 'container') - self.assertEqual(e.http_response_content, body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -755,9 +821,14 @@ class TestDeleteContainer(MockHttpTest): class TestGetObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -974,9 +1045,14 @@ class TestGetObject(MockHttpTest): class TestHeadObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.head_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_request_headers(self): c.http_connection = self.fake_http_connection(204) @@ -1018,7 +1094,7 @@ class TestPutObject(MockHttpTest): mock_file) text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, - 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'} + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -1053,10 +1129,15 @@ class TestPutObject(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, c.put_object, *args) + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { @@ -1234,13 +1315,16 @@ class TestPostObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! args = ('http://www.test.com', 'token', 'container', 'obj', - {'X-Object-Meta-Test': 'mymeta'}) + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) c.post_object(*args) self.assertRequests([ ('POST', '/container/obj', '', { 'x-auth-token': 'token', - 'X-Object-Meta-Test': 'mymeta'}), + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), ]) def test_unicode_ok(self): @@ -1252,7 +1336,7 @@ class TestPostObject(MockHttpTest): text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', - 'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', 'X-Object-Meta-Header-not-encoded': text, b'X-Object-Meta-Header-encoded': 'value'} @@ -1273,10 +1357,14 @@ class TestPostObject(MockHttpTest): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'token', 'container', 'obj', {}) - e = self.assertRaises(c.ClientException, c.post_object, *args) - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('POST', 'http://www.test.com/container/obj', '', { 'x-auth-token': 'token', @@ -1296,9 +1384,14 @@ class TestDeleteObject(MockHttpTest): ]) def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.delete_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -1325,9 +1418,15 @@ class TestGetCapabilities(MockHttpTest): self.assertTrue(http_conn[1].resp.has_been_read) def test_server_error(self): - conn = self.fake_http_connection(500) + body = 'c' * 60 + headers = {'foo': 'bar'} + conn = self.fake_http_connection( + StubResponse(500, body, headers)) http_conn = conn('http://www.test.com/info') - self.assertRaises(c.ClientException, c.get_capabilities, http_conn) + with self.assertRaises(c.ClientException) as exc_context: + c.get_capabilities(http_conn) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_conn_get_capabilities_with_auth(self): auth_headers = { @@ -1430,17 +1529,23 @@ class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): url = u'www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) url = u'://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) url = u'blah://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) @@ -1607,8 +1712,9 @@ class TestConnection(MockHttpTest): } c.http_connection = self.fake_http_connection( *code_iter, headers=auth_resp_headers) - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, conn.retries + 1) # test default no-retry @@ -1616,8 +1722,9 @@ class TestConnection(MockHttpTest): 200, 498, headers=auth_resp_headers) conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) def test_resp_read_on_server_error(self): @@ -1887,9 +1994,10 @@ class TestConnection(MockHttpTest): # v2 auth timeouts = [] + os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} conn = c.Connection( 'http://auth.example.com', 'user', 'password', timeout=33.0, - os_options=dict(tenant_name='tenant'), auth_version=2.0) + os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') with mock.patch('swiftclient.client._import_keystone_client', _make_fake_import_keystone_client(fake_ks)): @@ -1904,28 +2012,33 @@ class TestConnection(MockHttpTest): # check timeout passed to HEAD for account self.assertEqual(timeouts, [33.0]) + # check token passed to keystone client + self.assertIn('token', fake_ks.calls[0]) + self.assertEqual('meta-token', fake_ks.calls[0].get('token')) + def test_reset_stream(self): class LocalContents(object): def __init__(self, tell_value=0): - self.already_read = False + self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data.seek(tell_value) + self.reads = [] self.seeks = [] - self.tell_value = tell_value + self.tells = [] def tell(self): - return self.tell_value + self.tells.append(self.data.tell()) + return self.tells[-1] - def seek(self, position): - self.seeks.append(position) - self.already_read = False + def seek(self, position, mode=0): + self.seeks.append((position, mode)) + self.data.seek(position, mode) def read(self, size=-1): - if self.already_read: - return '' - else: - self.already_read = True - return 'abcdef' + read_data = self.data.read(size) + self.reads.append((size, read_data)) + return read_data class LocalConnection(object): @@ -1936,7 +2049,7 @@ class TestConnection(MockHttpTest): self.port = parsed_url.netloc def putrequest(self, *args, **kwargs): - self.send() + self.send('PUT', *args, **kwargs) def putheader(self, *args, **kwargs): return @@ -1945,6 +2058,13 @@ class TestConnection(MockHttpTest): return def send(self, *args, **kwargs): + data = kwargs.get('data') + if data is not None: + if hasattr(data, 'read'): + data.read() + else: + for datum in data: + pass raise socket.error('oops') def request(self, *args, **kwargs): @@ -1979,7 +2099,12 @@ class TestConnection(MockHttpTest): conn.put_object('c', 'o', contents) except socket.error as err: exc = err - self.assertEqual(contents.seeks, [0]) + self.assertEqual(contents.tells, [0]) + self.assertEqual(contents.seeks, [(0, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) self.assertEqual(str(exc), 'oops') contents = LocalContents(tell_value=123) @@ -1988,9 +2113,29 @@ class TestConnection(MockHttpTest): conn.put_object('c', 'o', contents) except socket.error as err: exc = err - self.assertEqual(contents.seeks, [123]) + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) self.assertEqual(str(exc), 'oops') + contents = LocalContents(tell_value=123) + wrapped_contents = swiftclient.utils.LengthWrapper( + contents, 6, md5=True) + exc = None + try: + conn.put_object('c', 'o', wrapped_contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + self.assertEqual(contents.reads, [(6, b'tuvwxy')] * 2) + self.assertEqual(str(exc), 'oops') + self.assertEqual(md5(b'tuvwxy').hexdigest(), + wrapped_contents.get_md5sum()) + contents = LocalContents() contents.tell = None exc = None @@ -2060,7 +2205,8 @@ class TestResponseDict(MockHttpTest): """ Verify handling of optional response_dict argument. """ - calls = [('post_container', 'c', {}), + calls = [('post_account', {}), + ('post_container', 'c', {}), ('put_container', 'c'), ('delete_container', 'c'), ('post_object', 'c', 'o', {}), @@ -2176,9 +2322,86 @@ class TestLogging(MockHttpTest): def test_get_error(self): c.http_connection = self.fake_http_connection(404) - e = self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(e.http_status, 404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) + + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) class TestCloseConnection(MockHttpTest): @@ -2374,6 +2597,28 @@ class TestServiceToken(MockHttpTest): actual['full_path']) self.assertEqual(conn.attempts, 1) + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + def test_service_token_head_container(self): with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): |