diff options
| -rw-r--r-- | doc/source/service-api.rst | 2 | ||||
| -rw-r--r-- | examples/upload.py | 3 | ||||
| -rw-r--r-- | swiftclient/client.py | 8 | ||||
| -rw-r--r-- | swiftclient/service.py | 11 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 2 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 355 |
6 files changed, 236 insertions, 145 deletions
diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index ea530b0..02b12d0 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -833,6 +833,8 @@ the method docstring. For each successful call to list capabilities, a result dictionary will be returned with the contents described below: +.. code-block:: python + { 'action': 'capabilities', 'timestamp': <time of the call>, diff --git a/examples/upload.py b/examples/upload.py index 1b1e349..0715b19 100644 --- a/examples/upload.py +++ b/examples/upload.py @@ -1,6 +1,7 @@ import logging -from os.path import join, walk +from os import walk +from os.path import join from swiftclient.multithreading import OutputManager from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject from sys import argv diff --git a/swiftclient/client.py b/swiftclient/client.py index 602489d..988c7d9 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1619,7 +1619,7 @@ class Connection(object): backoff = self.starting_backoff caller_response_dict = kwargs.pop('response_dict', None) self.attempts = kwargs.pop('attempts', 0) - while self.attempts <= self.retries: + while self.attempts <= self.retries or retried_auth: self.attempts += 1 try: if not self.url or not self.token: @@ -1648,9 +1648,6 @@ class Connection(object): self.http_conn = None except ClientException as err: self._add_response_dict(caller_response_dict, kwargs) - if self.attempts > self.retries or err.http_status is None: - logger.exception(err) - raise if err.http_status == 401: self.url = self.token = self.service_token = None if retried_auth or not all((self.authurl, @@ -1659,6 +1656,9 @@ class Connection(object): logger.exception(err) raise retried_auth = True + elif self.attempts > self.retries or err.http_status is None: + logger.exception(err) + raise elif err.http_status == 408: self.http_conn = None elif 500 <= err.http_status <= 599: diff --git a/swiftclient/service.py b/swiftclient/service.py index af412d1..f204895 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -282,7 +282,7 @@ def split_headers(options, prefix=''): for item in options: split_item = item.split(':', 1) if len(split_item) == 2: - headers[(prefix + split_item[0]).title()] = split_item[1] + headers[(prefix + split_item[0]).title()] = split_item[1].strip() else: raise SwiftError( "Metadata parameter %s must contain a ':'.\n%s" @@ -323,13 +323,12 @@ class SwiftPostObject(object): specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not isinstance(object_name, string_types) or not object_name: + if not (isinstance(object_name, string_types) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) - else: - self.object_name = object_name - self.options = options + self.object_name = object_name + self.options = options class SwiftCopyObject(object): @@ -340,7 +339,7 @@ class SwiftCopyObject(object): destination and fresh_metadata should be set in options """ def __init__(self, object_name, options=None): - if not isinstance(object_name, string_types) or not object_name: + if not (isinstance(object_name, string_types) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index d306fdb..546e495 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -532,7 +532,7 @@ class TestServiceUtils(unittest.TestCase): self.assertEqual(opt_c['key'], 'key') def test_split_headers(self): - mock_headers = ['color:blue', 'size:large'] + mock_headers = ['color:blue', 'SIZE: large'] expected = {'Color': 'blue', 'Size': 'large'} actual = swiftclient.service.split_headers(mock_headers) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 4e4c9f4..0095447 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -335,79 +335,84 @@ class TestGetAuth(MockHttpTest): def test_auth_v2_with_tenant_name(self): os_options = {'tenant_name': 'asdf'} req_args = {'auth_version': '2.0'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_with_tenant_id(self): os_options = {'tenant_id': 'asdf'} req_args = {'auth_version': '2.0'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_with_project_name(self): os_options = {'project_name': 'asdf'} req_args = {'auth_version': '2.0'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_with_project_id(self): os_options = {'project_id': 'asdf'} req_args = {'auth_version': '2.0'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_no_tenant_name_or_tenant_id(self): - c.get_auth_keystone = fake_get_auth_keystone({}) - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options={}, - auth_version='2.0') + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({})): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') def test_auth_v2_with_tenant_name_none_and_tenant_id_none(self): os_options = {'tenant_name': None, 'tenant_id': None} - c.get_auth_keystone = fake_get_auth_keystone(os_options) - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options=os_options, - auth_version='2.0') + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, + auth_version='2.0') def test_auth_v2_with_tenant_user_in_user(self): tenant_option = {'tenant_name': 'foo'} - c.get_auth_keystone = fake_get_auth_keystone(tenant_option) - url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', - os_options={}, - auth_version="2.0") + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + os_options={}, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_tenant_name_no_os_options(self): tenant_option = {'tenant_name': 'asdf'} - c.get_auth_keystone = fake_get_auth_keystone(tenant_option) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - tenant_name='asdf', - os_options={}, - auth_version="2.0") + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + tenant_name='asdf', + os_options={}, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) @@ -415,138 +420,141 @@ class TestGetAuth(MockHttpTest): os_options = {'service_type': 'object-store', 'endpoint_type': 'internalURL', 'tenant_name': 'asdf'} - c.get_auth_keystone = fake_get_auth_keystone(os_options) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_with_tenant_user_in_user_no_os_options(self): tenant_option = {'tenant_name': 'foo'} - c.get_auth_keystone = fake_get_auth_keystone(tenant_option) - url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', - auth_version="2.0") + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(tenant_option)): + url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf', + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_with_os_region_name(self): os_options = {'region_name': 'good-region', 'tenant_name': 'asdf'} - c.get_auth_keystone = fake_get_auth_keystone(os_options) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="2.0") + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options)): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="2.0") self.assertTrue(url.startswith("http")) self.assertTrue(token) def test_auth_v2_no_endpoint(self): os_options = {'region_name': 'unknown_region', 'tenant_name': 'asdf'} - c.get_auth_keystone = fake_get_auth_keystone( - os_options, c.ClientException) - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') def test_auth_v2_ks_exception(self): - c.get_auth_keystone = fake_get_auth_keystone( - {}, c.ClientException) - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.tests.com', 'asdf', 'asdf', - os_options={}, - auth_version='2.0') + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone({}, c.ClientException)): + self.assertRaises(c.ClientException, c.get_auth, + 'http://www.tests.com', 'asdf', 'asdf', + os_options={}, + auth_version='2.0') def test_auth_v2_cacert(self): os_options = {'tenant_name': 'foo'} - c.get_auth_keystone = fake_get_auth_keystone( - os_options, None) - auth_url_secure = 'https://www.tests.com' auth_url_insecure = 'https://www.tests.com/self-signed-certificate' - url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cacert='ca.pem', insecure=False) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cacert='ca.pem', insecure=False) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) def test_auth_v2_insecure(self): os_options = {'tenant_name': 'foo'} - c.get_auth_keystone = fake_get_auth_keystone( - os_options, None) - auth_url_secure = 'https://www.tests.com' auth_url_insecure = 'https://www.tests.com/invalid-certificate' - url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=True) - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_insecure, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - insecure=False) + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_secure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=True) + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_insecure, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + insecure=False) def test_auth_v2_cert(self): os_options = {'tenant_name': 'foo'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, None) - auth_url_no_sslauth = 'https://www.tests.com' auth_url_sslauth = 'https://www.tests.com/client-certificate' - url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cert='minnie', cert_key='mickey') - self.assertTrue(url.startswith("http")) - self.assertTrue(token) - - self.assertRaises(c.ClientException, c.get_auth, - auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0') - self.assertRaises(c.ClientException, c.get_auth, - auth_url_sslauth, 'asdf', 'asdf', - os_options=os_options, auth_version='2.0', - cert='minnie') + with mock.patch('swiftclient.client.get_auth_keystone', + fake_get_auth_keystone(os_options, None)): + url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie', cert_key='mickey') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie') def test_auth_v3_with_tenant_name(self): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} req_args = {'auth_version': '3'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', - os_options=os_options, - auth_version="3") + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf', + os_options=os_options, + auth_version="3") + self.assertTrue(url.startswith("http")) self.assertTrue(token) @@ -554,10 +562,13 @@ class TestGetAuth(MockHttpTest): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} req_args = {'auth_version': '2.0'} - c.get_auth_keystone = fake_get_auth_keystone(os_options, - required_kwargs=req_args) - url, token = c.get_keystoneclient_2_0('http://www.test.com', 'asdf', - 'asdf', os_options=os_options) + + ks = fake_get_auth_keystone(os_options, required_kwargs=req_args) + with mock.patch('swiftclient.client.get_auth_keystone', ks): + url, token = c.get_keystoneclient_2_0('http://www.test.com', + 'asdf', 'asdf', + os_options=os_options) + self.assertTrue(url.startswith("http")) self.assertTrue(token) @@ -2691,6 +2702,84 @@ class TestServiceToken(MockHttpTest): self.assertEqual('service_project_name', auth_kwargs['os_options']['tenant_name']) + def test_service_token_reauth_retries_0(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + # Ensure this is not an endless loop - it fails after the second 401 + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 401, 401, 401)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options, + retries=0) + + self.assertEqual(conn.attempts, 0) + self.assertRaises(c.ClientException, conn.head_account) + self.assertEqual(conn.attempts, 2) + unused_responses = list(self.fake_connect.code_iter) + self.assertEqual(unused_responses, [401, 401]) + def test_service_token_get_account(self): with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): |