diff options
| -rw-r--r-- | CONTRIBUTING.md | 5 | ||||
| -rw-r--r-- | README.rst | 2 | ||||
| -rw-r--r-- | swiftclient/client.py | 84 | ||||
| -rw-r--r-- | swiftclient/multithreading.py | 3 | ||||
| -rw-r--r-- | swiftclient/service.py | 115 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 39 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 335 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 489 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 533 | ||||
| -rw-r--r-- | tests/unit/utils.py | 256 |
10 files changed, 1410 insertions, 451 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 866da8a..88c1bb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,9 @@ If you would like to contribute to the development of OpenStack, -you must follow the steps in the "If you're a developer" -section of this page: [http://wiki.openstack.org/HowToContribute](http://wiki.openstack.org/HowToContribute#If_you.27re_a_developer) +you must follow the steps in this page: [http://docs.openstack.org/infra/manual/developers.html](http://docs.openstack.org/infra/manual/developers.html) Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following -the workflow documented at [http://wiki.openstack.org/GerritWorkflow](http://wiki.openstack.org/GerritWorkflow). +the workflow documented at [http://docs.openstack.org/infra/manual/developers.html#development-workflow](http://docs.openstack.org/infra/manual/developers.html#development-workflow). Gerrit is the review system used in the OpenStack projects. We're sorry, but we won't be able to respond to pull requests submitted through GitHub. @@ -11,7 +11,7 @@ __ http://docs.openstack.org/developer/python-swiftclient/ Development takes place via the usual OpenStack processes as outlined in the `OpenStack wiki`__. The master repository is on GitHub__. -__ http://wiki.openstack.org/HowToContribute +__ http://docs.openstack.org/infra/manual/developers.html __ http://github.com/openstack/python-swiftclient This code is based on original the client previously included with diff --git a/swiftclient/client.py b/swiftclient/client.py index 84b50d2..40cb9fe 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -19,10 +19,13 @@ OpenStack Swift client library used internally import socket import requests -import sys import logging import warnings import functools +try: + from simplejson import loads as json_loads +except ImportError: + from json import loads as json_loads from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError @@ -38,6 +41,8 @@ from swiftclient.utils import LengthWrapper AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) +USER_METADATA_TYPE = tuple('x-%s-meta-' % type_ for type_ in + ('container', 'account', 'object')) try: from logging import NullHandler @@ -119,16 +124,22 @@ def encode_utf8(value): return value -# look for a real json parser first -try: - # simplejson is popular and pretty good - from simplejson import loads as json_loads -except ImportError: - # 2.6 will have a json module in the stdlib - from json import loads as json_loads +def encode_meta_headers(headers): + """Only encode metadata headers keys""" + ret = {} + for header, value in headers.items(): + value = encode_utf8(value) + header = header.lower() + if (isinstance(header, six.string_types) + and header.startswith(USER_METADATA_TYPE)): + header = encode_utf8(header) -class HTTPConnection: + ret[header] = value + return ret + + +class HTTPConnection(object): def __init__(self, url, proxy=None, cacert=None, insecure=False, ssl_compression=False, default_user_agent=None): """ @@ -184,27 +195,12 @@ class HTTPConnection: """ Final wrapper before requests call, to be patched in tests """ return self.request_session.request(*arg, **kwarg) - def _encode_meta_headers(self, items): - """Only encode metadata headers keys""" - ret = {} - for header, value in items: - value = encode_utf8(value) - header = header.lower() - if isinstance(header, six.string_types): - for target_type in 'container', 'account', 'object': - prefix = 'x-%s-meta-' % target_type - if header.startswith(prefix): - header = encode_utf8(header) - break - ret[header] = value - return ret - def request(self, method, full_path, data=None, headers=None, files=None): """ Encode url and header, then call requests.request """ if headers is None: headers = {} else: - headers = self._encode_meta_headers(headers.items()) + headers = encode_meta_headers(headers) # set a default User-Agent header if it wasn't passed in if 'user-agent' not in headers: @@ -298,7 +294,7 @@ def _import_keystone_client(auth_version): logging.getLogger('keystoneclient').addHandler(NullHandler()) return ksclient, exceptions except ImportError: - sys.exit(''' + raise ClientException(''' Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment variables to be set or overridden with -A, -U, or -K.''') @@ -352,13 +348,22 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): except exceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) - return (endpoint, _ksclient.auth_token) + return endpoint, _ksclient.auth_token def get_auth(auth_url, user, key, **kwargs): """ Get authentication/authorization credentials. + :kwarg auth_version: the api version of the supplied auth params + :kwarg os_options: a dict, the openstack idenity service options + + :returns: a tuple, (storage_url, token) + + N.B. if the optional os_options paramater includes an non-empty + 'object_storage_url' key it will override the the default storage url + returned by the auth service. + The snet parameter is used for Rackspace's ServiceNet internal network implementation. In this function, it simply adds *snet-* to the beginning of the host name for the returned storage URL. With Rackspace Cloud Files, @@ -377,13 +382,6 @@ def get_auth(auth_url, user, key, **kwargs): kwargs.get('snet'), insecure=insecure) elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3: - # We are allowing to specify a token/storage-url to re-use - # without having to re-authenticate. - if (os_options.get('object_storage_url') and - os_options.get('auth_token')): - return (os_options.get('object_storage_url'), - os_options.get('auth_token')) - # We are handling a special use case here where the user argument # specifies both the user name and tenant name in the form tenant:user if user and not kwargs.get('tenant_name') and ':' in user: @@ -465,9 +463,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, listing = rv[1] while listing: marker = listing[-1]['name'] - listing = \ - get_account(url, token, marker, limit, prefix, - end_marker, http_conn)[1] + listing = get_account(url, token, marker, limit, prefix, + end_marker, http_conn)[1] if listing: rv[1].extend(listing) return rv @@ -1180,8 +1177,6 @@ class Connection(object): self.key = key self.retries = retries self.http_conn = None - self.url = preauthurl - self.token = preauthtoken self.attempts = 0 self.snet = snet self.starting_backoff = starting_backoff @@ -1190,6 +1185,10 @@ class Connection(object): self.os_options = os_options or {} if tenant_name: self.os_options['tenant_name'] = tenant_name + if preauthurl: + self.os_options['object_storage_url'] = preauthurl + self.url = preauthurl or self.os_options.get('object_storage_url') + self.token = preauthtoken or self.os_options.get('auth_token') self.cacert = cacert self.insecure = insecure self.ssl_compression = ssl_compression @@ -1197,10 +1196,12 @@ class Connection(object): self.retry_on_ratelimit = retry_on_ratelimit def close(self): - if self.http_conn and type(self.http_conn) is tuple\ - and len(self.http_conn) > 1: + if (self.http_conn and isinstance(self.http_conn, tuple) + and len(self.http_conn) > 1): conn = self.http_conn[1] if hasattr(conn, 'close') and callable(conn.close): + # XXX: Our HTTPConnection object has no close, should be + # trying to close the requests.Session here? conn.close() self.http_conn = None @@ -1385,6 +1386,7 @@ class Connection(object): response_dict=response_dict) def get_capabilities(self, url=None): + url = url or self.url if not url: url, _ = self.get_auth() scheme = urlparse(url).scheme diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index a2dcd71..ade0f7b 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -86,6 +86,9 @@ class OutputManager(object): msg = msg % fmt_args self.error_print_pool.submit(self._print_error, msg) + def get_error_count(self): + return self.error_count + def _print(self, item, stream=None): if stream is None: stream = self.print_stream diff --git a/swiftclient/service.py b/swiftclient/service.py index 1b591d3..55e6daa 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -313,36 +313,37 @@ class _SwiftReader(object): self._actual_md5 = None self._expected_etag = headers.get('etag') - if 'x-object-manifest' not in headers and \ - 'x-static-large-object' not in headers: - self.actual_md5 = md5() + if ('x-object-manifest' not in headers + and 'x-static-large-object' not in headers): + self._actual_md5 = md5() if 'content-length' in headers: - self._content_length = int(headers.get('content-length')) + try: + self._content_length = int(headers.get('content-length')) + except ValueError: + raise SwiftError('content-length header must be an integer') def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - if self._actual_md5 is not None: + if self._actual_md5 and self._expected_etag: etag = self._actual_md5.hexdigest() if etag != self._expected_etag: - raise SwiftError( - 'Error downloading %s: md5sum != etag, %s != %s' % - (self._path, etag, self._expected_etag) - ) + raise SwiftError('Error downloading {0}: md5sum != etag, ' + '{1} != {2}'.format( + self._path, etag, self._expected_etag)) - if self._content_length is not None and \ - self._actual_read != self._content_length: - raise SwiftError( - 'Error downloading %s: read_length != content_length, ' - '%d != %d' % (self._path, self._actual_read, - self._content_length) - ) + if (self._content_length is not None + and self._actual_read != self._content_length): + raise SwiftError('Error downloading {0}: read_length != ' + 'content_length, {1:d} != {2:d}'.format( + self._path, self._actual_read, + self._content_length)) def buffer(self): for chunk in self._body: - if self._actual_md5 is not None: + if self._actual_md5: self._actual_md5.update(chunk) self._actual_read += len(chunk) yield chunk @@ -1529,8 +1530,8 @@ class SwiftService(object): old_manifest = None old_slo_manifest_paths = [] new_slo_manifest_paths = set() - if options['changed'] or options['skip_identical'] \ - or not options['leave_segments']: + if (options['changed'] or options['skip_identical'] + or not options['leave_segments']): checksum = None if options['skip_identical']: try: @@ -1555,11 +1556,12 @@ class SwiftService(object): 'status': 'skipped-identical' }) return res + cl = int(headers.get('content-length')) mt = headers.get('x-object-meta-mtime') - if path is not None and options['changed']\ - and cl == getsize(path) and \ - mt == put_headers['x-object-meta-mtime']: + if (path is not None and options['changed'] + and cl == getsize(path) + and mt == put_headers['x-object-meta-mtime']): res.update({ 'success': True, 'status': 'skipped-changed' @@ -1593,8 +1595,8 @@ class SwiftService(object): # a segment job if we're reading from a stream - we may fail if we # go over the single object limit, but this gives us a nice way # to create objects from memory - if path is not None and options['segment_size'] and \ - getsize(path) > int(options['segment_size']): + if (path is not None and options['segment_size'] + and getsize(path) > int(options['segment_size'])): res['large_object'] = True seg_container = container + '_segments' if options['segment_container']: @@ -1850,9 +1852,8 @@ class SwiftService(object): # Cancel the remaining container deletes, but yield # any pending results - if not cancelled and \ - options['fail_fast'] and \ - not res['success']: + if (not cancelled and options['fail_fast'] + and not res['success']): cancelled = True @staticmethod @@ -1860,24 +1861,17 @@ class SwiftService(object): results_dict = {} try: conn.delete_object(container, obj, response_dict=results_dict) - res = { - 'action': 'delete_segment', - 'container': container, - 'object': obj, - 'success': True, - 'attempts': conn.attempts, - 'response_dict': results_dict - } + res = {'success': True} except Exception as e: - res = { - 'action': 'delete_segment', - 'container': container, - 'object': obj, - 'success': False, - 'attempts': conn.attempts, - 'response_dict': results_dict, - 'exception': e - } + res = {'success': False, 'error': e} + + res.update({ + 'action': 'delete_segment', + 'container': container, + 'object': obj, + 'attempts': conn.attempts, + 'response_dict': results_dict + }) if results_queue is not None: results_queue.put(res) @@ -1898,8 +1892,7 @@ class SwiftService(object): try: headers = conn.head_object(container, obj) old_manifest = headers.get('x-object-manifest') - if config_true_value( - headers.get('x-static-large-object')): + if config_true_value(headers.get('x-static-large-object')): query_string = 'multipart-manifest=delete' except ClientException as err: if err.http_status != 404: @@ -1957,23 +1950,17 @@ class SwiftService(object): results_dict = {} try: conn.delete_container(container, response_dict=results_dict) - res = { - 'action': 'delete_container', - 'container': container, - 'object': None, - 'success': True, - 'attempts': conn.attempts, - 'response_dict': results_dict - } + res = {'success': True} except Exception as e: - res = { - 'action': 'delete_container', - 'container': container, - 'object': None, - 'success': False, - 'response_dict': results_dict, - 'error': e - } + res = {'success': False, 'error': e} + + res.update({ + 'action': 'delete_container', + 'container': container, + 'object': None, + 'attempts': conn.attempts, + 'response_dict': results_dict + }) return res def _delete_container(self, container, options): @@ -1981,9 +1968,7 @@ class SwiftService(object): objs = [] for part in self.list(container=container): if part["success"]: - objs.extend([ - o['name'] for o in part['listing'] - ]) + objs.extend([o['name'] for o in part['listing']]) else: raise part["error"] diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 2c627ad..d58de60 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -118,34 +118,29 @@ def st_delete(parser, args, output_manager): del_iter = swift.delete(container=container) for r in del_iter: + c = r.get('container', '') + o = r.get('object', '') + a = r.get('attempts') + if r['success']: if options.verbose: + a = ' [after {0} attempts]'.format(a) if a > 1 else '' + if r['action'] == 'delete_object': - c = r['container'] - o = r['object'] - p = '%s/%s' % (c, o) if options.yes_all else o - a = r['attempts'] - if a > 1: - output_manager.print_msg( - '%s [after %d attempts]', p, a) + if options.yes_all: + p = '{0}/{1}'.format(c, o) else: - output_manager.print_msg(p) - + p = o elif r['action'] == 'delete_segment': - c = r['container'] - o = r['object'] - p = '%s/%s' % (c, o) - a = r['attempts'] - if a > 1: - output_manager.print_msg( - '%s [after %d attempts]', p, a) - else: - output_manager.print_msg(p) + p = '{0}/{1}'.format(c, o) + elif r['action'] == 'delete_container': + p = c + output_manager.print_msg('{0}{1}'.format(p, a)) else: - # Special case error prints - output_manager.error("An unexpected error occurred whilst " - "deleting: %s" % r['error']) + p = '{0}/{1}'.format(c, o) if o else c + output_manager.error('Error Deleting: {0}: {1}' + .format(p, r['error'])) except SwiftError as err: output_manager.error(err.value) @@ -1308,7 +1303,7 @@ Examples: except (ClientException, RequestException, socket.error) as err: output.error(str(err)) - if output.error_count > 0: + if output.get_error_count() > 0: exit(1) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 53867ae..e10c065 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -12,10 +12,343 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - +import mock import testtools +from mock import Mock, PropertyMock +from six.moves.queue import Queue, Empty as QueueEmptyError +from hashlib import md5 +import swiftclient from swiftclient.service import SwiftService, SwiftError +from swiftclient.client import Connection + + +class TestSwiftPostObject(testtools.TestCase): + + def setUp(self): + self.spo = swiftclient.service.SwiftPostObject + super(TestSwiftPostObject, self).setUp() + + def test_create(self): + spo = self.spo('obj_name') + + self.assertEqual(spo.object_name, 'obj_name') + self.assertEqual(spo.options, None) + + def test_create_with_invalid_name(self): + # empty strings are not allowed as names + self.assertRaises(SwiftError, self.spo, '') + + # names cannot be anything but strings + self.assertRaises(SwiftError, self.spo, 1) + + +class TestSwiftReader(testtools.TestCase): + + def setUp(self): + self.sr = swiftclient.service._SwiftReader + super(TestSwiftReader, self).setUp() + self.md5_type = type(md5()) + + def test_create(self): + sr = self.sr('path', 'body', {}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, None) + self.assertEqual(sr._expected_etag, None) + + self.assertNotEqual(sr._actual_md5, None) + self.assertTrue(isinstance(sr._actual_md5, self.md5_type)) + + def test_create_with_large_object_headers(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'x-object-manifest': 'test'}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, None) + self.assertEqual(sr._expected_etag, None) + self.assertEqual(sr._actual_md5, None) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test'}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, None) + self.assertEqual(sr._expected_etag, None) + self.assertEqual(sr._actual_md5, None) + + def test_create_with_content_length(self): + sr = self.sr('path', 'body', {'content-length': 5}) + + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertEqual(sr._content_length, 5) + self.assertEqual(sr._expected_etag, None) + + self.assertNotEqual(sr._actual_md5, None) + self.assertTrue(isinstance(sr._actual_md5, self.md5_type)) + + # Check Contentlength raises error if it isnt an integer + self.assertRaises(SwiftError, self.sr, 'path', 'body', + {'content-length': 'notanint'}) + + def test_context_usage(self): + def _context(sr): + with sr: + pass + + sr = self.sr('path', 'body', {}) + _context(sr) + + # Check error is raised if expected etag doesnt match calculated md5. + # md5 for a SwiftReader that has done nothing is + # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing + sr = self.sr('path', 'body', {'etag': 'doesntmatch'}) + self.assertRaises(SwiftError, _context, sr) + + sr = self.sr('path', 'body', + {'etag': 'd41d8cd98f00b204e9800998ecf8427e'}) + _context(sr) + + # Check error is raised if SwiftReader doesnt read the same length + # as the content length it is created with + sr = self.sr('path', 'body', {'content-length': 5}) + self.assertRaises(SwiftError, _context, sr) + + sr = self.sr('path', 'body', {'content-length': 5}) + sr._actual_read = 5 + _context(sr) + + def test_buffer(self): + # md5 = 97ac82a5b825239e782d0339e2d7b910 + mock_buffer_content = ['abc'.encode()] * 3 + + sr = self.sr('path', mock_buffer_content, {}) + for x in sr.buffer(): + pass + + self.assertEqual(sr._actual_read, 9) + self.assertEqual(sr._actual_md5.hexdigest(), + '97ac82a5b825239e782d0339e2d7b910') + + +class TestServiceDelete(testtools.TestCase): + def setUp(self): + super(TestServiceDelete, self).setUp() + self.opts = {'leave_segments': False, 'yes_all': False} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be string in the form delete_XX + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'success': None # Should be a bool + } + + def _get_mock_connection(self, attempts=2): + m = Mock(spec=Connection) + type(m).attempts = PropertyMock(return_value=attempts) + return m + + def _get_queue(self, q): + # Instead of blocking pull items straight from the queue. + # expects at least one item otherwise the test will fail. + try: + return q.get_nowait() + except QueueEmptyError: + self.fail('Expected item in queue but found none') + + def _get_expected(self, update=None): + expected = self.expected.copy() + if update: + expected.update(update) + + return expected + + def _assertDictEqual(self, a, b, m=None): + # assertDictEqual is not available in py2.6 so use a shallow check + # instead + if hasattr(self, 'assertDictEqual'): + self.assertDictEqual(a, b, m) + else: + self.assertTrue(isinstance(a, dict)) + self.assertTrue(isinstance(b, dict)) + self.assertEqual(len(a), len(b), m) + for k, v in a.items(): + self.assertTrue(k in b, m) + self.assertEqual(b[k], v, m) + + def test_delete_segment(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': True, + }) + + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self._assertDictEqual(expected_r, r) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + + def test_delete_segment_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_segment', + 'object': 'test_s', + 'success': False, + 'error': self.exc + }) + + r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_s', response_dict={} + ) + self._assertDictEqual(expected_r, r) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + + def test_delete_object(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock(return_value={}) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with('test_c', 'test_o') + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={} + ) + self._assertDictEqual(expected_r, r) + + def test_delete_object_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.delete_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': False, + 'error': self.exc + }) + # _delete_object doesnt populate attempts or response dict if it hits + # an error. This may not be the correct behaviour. + del expected_r['response_dict'], expected_r['attempts'] + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with('test_c', 'test_o') + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', query_string=None, response_dict={} + ) + self._assertDictEqual(expected_r, r) + + def test_delete_object_slo_support(self): + # If SLO headers are present the delete call should include an + # additional query string to cause the right delete server side + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.head_object = Mock( + return_value={'x-static-large-object': True} + ) + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True + }) + + s = SwiftService() + r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + + mock_conn.head_object.assert_called_once_with('test_c', 'test_o') + mock_conn.delete_object.assert_called_once_with( + 'test_c', 'test_o', + query_string='multipart-manifest=delete', + response_dict={} + ) + self._assertDictEqual(expected_r, r) + + def test_delete_object_dlo_support(self): + mock_q = Queue() + s = SwiftService() + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_object', + 'success': True, + 'dlo_segments_deleted': True + }) + # A DLO object is determined in _delete_object by heading the object + # and checking for the existence of a x-object-manifest header. + # Mock that here. + mock_conn.head_object = Mock( + return_value={'x-object-manifest': 'manifest_c/manifest_p'} + ) + mock_conn.get_container = Mock( + side_effect=[(None, [{'name': 'test_seg_1'}, + {'name': 'test_seg_2'}]), + (None, {})] + ) + + def get_mock_list_conn(options): + return mock_conn + + with mock.patch('swiftclient.service.get_conn', get_mock_list_conn): + r = s._delete_object( + mock_conn, 'test_c', 'test_o', self.opts, mock_q + ) + + self._assertDictEqual(expected_r, r) + expected = [ + mock.call('test_c', 'test_o', query_string=None, response_dict={}), + mock.call('manifest_c', 'test_seg_1', response_dict={}), + mock.call('manifest_c', 'test_seg_2', response_dict={})] + mock_conn.delete_object.assert_has_calls(expected, any_order=True) + + def test_delete_empty_container(self): + mock_conn = self._get_mock_connection() + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': True, + 'object': None + }) + + r = SwiftService._delete_empty_container(mock_conn, 'test_c') + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={} + ) + self._assertDictEqual(expected_r, r) + + def test_delete_empty_container_excpetion(self): + mock_conn = self._get_mock_connection() + mock_conn.delete_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'delete_container', + 'success': False, + 'object': None, + 'error': self.exc + }) + + s = SwiftService() + r = s._delete_empty_container(mock_conn, 'test_c') + + mock_conn.delete_container.assert_called_once_with( + 'test_c', response_dict={} + ) + self._assertDictEqual(expected_r, r) class TestService(testtools.TestCase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index bdba193..e6434dc 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -17,6 +17,7 @@ import mock import os import tempfile import unittest +from testtools import ExpectedException import six @@ -24,10 +25,11 @@ import swiftclient from swiftclient.service import SwiftError import swiftclient.shell import swiftclient.utils -from swiftclient.multithreading import OutputManager from os.path import basename, dirname from tests.unit.test_swiftclient import MockHttpTest +from tests.unit.utils import CaptureOutput, fake_get_auth_keystone + if six.PY2: BUILTIN_OPEN = '__builtin__.open' @@ -40,6 +42,12 @@ mocked_os_environ = { 'ST_KEY': 'testing' } +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' + def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): """ @@ -88,9 +96,8 @@ class TestShell(unittest.TestCase): except OSError: pass - @mock.patch('swiftclient.shell.OutputManager._print') @mock.patch('swiftclient.service.Connection') - def test_stat_account(self, connection, mock_print): + def test_stat_account(self, connection): argv = ["", "stat"] return_headers = { 'x-account-container-count': '1', @@ -100,16 +107,17 @@ class TestShell(unittest.TestCase): 'date': ''} connection.return_value.head_account.return_value = return_headers connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - swiftclient.shell.main(argv) - calls = [mock.call(' Account: AUTH_account\n' + - 'Containers: 1\n' + - ' Objects: 2\n' + - ' Bytes: 3')] - mock_print.assert_has_calls(calls) + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEquals(output.out, + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') - @mock.patch('swiftclient.shell.OutputManager._print') @mock.patch('swiftclient.service.Connection') - def test_stat_container(self, connection, mock_print): + def test_stat_container(self, connection): return_headers = { 'x-container-object-count': '1', 'x-container-bytes-used': '2', @@ -121,20 +129,21 @@ class TestShell(unittest.TestCase): argv = ["", "stat", "container"] connection.return_value.head_container.return_value = return_headers connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - swiftclient.shell.main(argv) - calls = [mock.call(' Account: AUTH_account\n' + - 'Container: container\n' + - ' Objects: 1\n' + - ' Bytes: 2\n' + - ' Read ACL: test2:tester2\n' + - 'Write ACL: test3:tester3\n' + - ' Sync To: other\n' + - ' Sync Key: secret')] - mock_print.assert_has_calls(calls) - - @mock.patch('swiftclient.shell.OutputManager._print') + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEquals(output.out, + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') + @mock.patch('swiftclient.service.Connection') - def test_stat_object(self, connection, mock_print): + def test_stat_object(self, connection): return_headers = { 'x-object-manifest': 'manifest', 'etag': 'md5', @@ -145,20 +154,22 @@ class TestShell(unittest.TestCase): argv = ["", "stat", "container", "object"] connection.return_value.head_object.return_value = return_headers connection.return_value.url = 'http://127.0.0.1/v1/AUTH_account' - swiftclient.shell.main(argv) - calls = [mock.call(' Account: AUTH_account\n' + - ' Container: container\n' + - ' Object: object\n' + - ' Content Type: text/plain\n' + - 'Content Length: 42\n' + - ' Last Modified: yesterday\n' + - ' ETag: md5\n' + - ' Manifest: manifest')] - mock_print.assert_has_calls(calls) - - @mock.patch('swiftclient.shell.OutputManager._print') + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + + self.assertEquals(output.out, + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') + @mock.patch('swiftclient.service.Connection') - def test_list_account(self, connection, mock_print): + def test_list_account(self, connection): # Test account listing connection.return_value.get_account.side_effect = [ [None, [{'name': 'container'}]], @@ -166,16 +177,17 @@ class TestShell(unittest.TestCase): ] argv = ["", "list"] - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None), - mock.call(marker='container', prefix=None)] - connection.return_value.get_account.assert_has_calls(calls) - calls = [mock.call('container')] - mock_print.assert_has_calls(calls) - @mock.patch('swiftclient.shell.OutputManager._print') + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None), + mock.call(marker='container', prefix=None)] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEquals(output.out, 'container\n') + @mock.patch('swiftclient.service.Connection') - def test_list_account_long(self, connection, mock_print): + def test_list_account_long(self, connection): # Test account listing connection.return_value.get_account.side_effect = [ [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], @@ -183,13 +195,15 @@ class TestShell(unittest.TestCase): ] argv = ["", "list", "--lh"] - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None), - mock.call(marker='container', prefix=None)] - connection.return_value.get_account.assert_has_calls(calls) - calls = [mock.call(' 0 0 1970-01-01 00:00:01 container'), - mock.call(' 0 0')] - mock_print.assert_has_calls(calls) + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None), + mock.call(marker='container', prefix=None)] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEquals(output.out, + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') # Now test again, this time without returning metadata connection.return_value.head_container.return_value = {} @@ -201,30 +215,32 @@ class TestShell(unittest.TestCase): ] argv = ["", "list", "--lh"] - swiftclient.shell.main(argv) - calls = [mock.call(marker='', prefix=None), - mock.call(marker='container', prefix=None)] - connection.return_value.get_account.assert_has_calls(calls) - calls = [mock.call(' 0 0 ????-??-?? ??:??:?? container'), - mock.call(' 0 0')] - mock_print.assert_has_calls(calls) - - @mock.patch('swiftclient.shell.OutputManager._print') + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None), + mock.call(marker='container', prefix=None)] + connection.return_value.get_account.assert_has_calls(calls) + + self.assertEquals(output.out, + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') + @mock.patch('swiftclient.service.Connection') - def test_list_container(self, connection, mock_print): + def test_list_container(self, connection): connection.return_value.get_container.side_effect = [ [None, [{'name': 'object_a'}]], [None, []], ] argv = ["", "list", "container"] - swiftclient.shell.main(argv) - calls = [ - mock.call('container', marker='', delimiter=None, prefix=None), - mock.call('container', marker='object_a', - delimiter=None, prefix=None)] - connection.return_value.get_container.assert_has_calls(calls) - calls = [mock.call('object_a')] - mock_print.assert_has_calls(calls) + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', delimiter=None, prefix=None), + mock.call('container', marker='object_a', + delimiter=None, prefix=None)] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEquals(output.out, 'object_a\n') # Test container listing with --long connection.return_value.get_container.side_effect = [ @@ -233,16 +249,17 @@ class TestShell(unittest.TestCase): [None, []], ] argv = ["", "list", "container", "--long"] - swiftclient.shell.main(argv) - calls = [ - mock.call('container', marker='', delimiter=None, prefix=None), - mock.call('container', marker='object_a', - delimiter=None, prefix=None)] - connection.return_value.get_container.assert_has_calls(calls) - calls = [mock.call('object_a'), - mock.call(' 0 123 456 object_a'), - mock.call(' 0')] - mock_print.assert_has_calls(calls) + with CaptureOutput() as output: + swiftclient.shell.main(argv) + calls = [ + mock.call('container', marker='', delimiter=None, prefix=None), + mock.call('container', marker='object_a', + delimiter=None, prefix=None)] + connection.return_value.get_container.assert_has_calls(calls) + + self.assertEquals(output.out, + ' 0 123 456 object_a\n' + ' 0\n') @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') @@ -387,6 +404,61 @@ class TestShell(unittest.TestCase): connection.return_value.delete_object.assert_called_with( 'container', 'object', query_string=None, response_dict={}) + def test_delete_verbose_output(self): + del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': 'test_c', 'action': 'delete_object', + 'object': 'test_o'} + + del_seg_res = del_obj_res.copy() + del_seg_res.update({'action': 'delete_segment'}) + + del_con_res = del_obj_res.copy() + del_con_res.update({'action': 'delete_container', 'object': None}) + + test_exc = Exception('test_exc') + error_res = del_obj_res.copy() + error_res.update({'success': False, 'error': test_exc, 'object': None}) + + mock_delete = mock.Mock() + base_argv = ['', '--verbose', 'delete'] + + with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): + with CaptureOutput() as out: + mock_delete.return_value = [del_obj_res] + swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + + mock_delete.assert_called_once_with(container='test_c', + objects=['test_o']) + self.assertTrue(out.out.find( + 'test_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_seg_res] + swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + + mock_delete.assert_called_with(container='test_c', + objects=['test_o']) + self.assertTrue(out.out.find( + 'test_c/test_o [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [del_con_res] + swiftclient.shell.main(base_argv + ['test_c']) + + mock_delete.assert_called_with(container='test_c') + self.assertTrue(out.out.find( + 'test_c [after 2 attempts]') >= 0) + + with CaptureOutput() as out: + mock_delete.return_value = [error_res] + self.assertRaises(SystemExit, + swiftclient.shell.main, + base_argv + ['test_c']) + + mock_delete.assert_called_with(container='test_c') + self.assertTrue(out.err.find( + 'Error Deleting: test_c: test_exc') >= 0) + @mock.patch('swiftclient.service.Connection') def test_post_account(self, connection): argv = ["", "post"] @@ -394,23 +466,29 @@ class TestShell(unittest.TestCase): connection.return_value.post_account.assert_called_with( headers={}, response_dict={}) - @mock.patch('swiftclient.shell.OutputManager.error') @mock.patch('swiftclient.service.Connection') - def test_post_account_bad_auth(self, connection, error): + def test_post_account_bad_auth(self, connection): argv = ["", "post"] connection.return_value.post_account.side_effect = \ swiftclient.ClientException('bad auth') - swiftclient.shell.main(argv) - error.assert_called_with('bad auth') - @mock.patch('swiftclient.shell.OutputManager.error') + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + + self.assertEquals(output.err, 'bad auth\n') + @mock.patch('swiftclient.service.Connection') - def test_post_account_not_found(self, connection, error): + def test_post_account_not_found(self, connection): argv = ["", "post"] connection.return_value.post_account.side_effect = \ swiftclient.ClientException('test', http_status=404) - swiftclient.shell.main(argv) - error.assert_called_with('Account not found') + + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + + self.assertEquals(output.err, 'Account not found\n') @mock.patch('swiftclient.service.Connection') def test_post_container(self, connection): @@ -419,14 +497,17 @@ class TestShell(unittest.TestCase): connection.return_value.post_container.assert_called_with( 'container', headers={}, response_dict={}) - @mock.patch('swiftclient.shell.OutputManager.error') @mock.patch('swiftclient.service.Connection') - def test_post_container_bad_auth(self, connection, error): + def test_post_container_bad_auth(self, connection): argv = ["", "post", "container"] connection.return_value.post_container.side_effect = \ swiftclient.ClientException('bad auth') - swiftclient.shell.main(argv) - error.assert_called_with('bad auth') + + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + + self.assertEquals(output.err, 'bad auth\n') @mock.patch('swiftclient.service.Connection') def test_post_container_not_found_causes_put(self, connection): @@ -437,12 +518,14 @@ class TestShell(unittest.TestCase): self.assertEqual('container', connection.return_value.put_container.call_args[0][0]) - @mock.patch('swiftclient.shell.OutputManager.error') - def test_post_container_with_bad_name(self, error): + def test_post_container_with_bad_name(self): argv = ["", "post", "conta/iner"] - swiftclient.shell.main(argv) - self.assertTrue(error.called) - self.assertTrue(error.call_args[0][0].startswith('WARNING: / in')) + + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('WARNING: / in')) @mock.patch('swiftclient.service.Connection') def test_post_container_with_options(self, connection): @@ -472,21 +555,27 @@ class TestShell(unittest.TestCase): 'Content-Type': 'text/plain', 'X-Object-Meta-Color': 'Blue'}, response_dict={}) - @mock.patch('swiftclient.shell.OutputManager.error') @mock.patch('swiftclient.service.Connection') - def test_post_object_bad_auth(self, connection, error): + def test_post_object_bad_auth(self, connection): argv = ["", "post", "container", "object"] connection.return_value.post_object.side_effect = \ swiftclient.ClientException("bad auth") - swiftclient.shell.main(argv) - error.assert_called_with('bad auth') - @mock.patch('swiftclient.shell.OutputManager.error') - def test_post_object_too_many_args(self, error): + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + + self.assertEquals(output.err, 'bad auth\n') + + def test_post_object_too_many_args(self): argv = ["", "post", "container", "object", "bad_arg"] - swiftclient.shell.main(argv) - self.assertTrue(error.called) - self.assertTrue(error.call_args[0][0].startswith('Usage')) + + with CaptureOutput() as output: + with ExpectedException(SystemExit): + swiftclient.shell.main(argv) + + self.assertTrue(output.err != '') + self.assertTrue(output.err.startswith('Usage')) @mock.patch('swiftclient.shell.generate_temp_url') def test_temp_url(self, temp_url): @@ -510,15 +599,9 @@ class TestShell(unittest.TestCase): actual = x.call_args_list[-1][1]["options"]["segment_size"] self.assertEqual(int(actual), expected) - mock_out = mock.MagicMock(spec=swiftclient.shell.OutputManager) - mock_out.__enter__.return_value = mock_out - mock_out.return_value = mock_out - type(mock_out).error_count = mock.PropertyMock(return_value=0) - mock_swift = mock.MagicMock(spec=swiftclient.shell.SwiftService) - with mock.patch("swiftclient.shell.SwiftService", mock_swift): - with mock.patch('swiftclient.shell.OutputManager', mock_out): + with CaptureOutput(suppress_systemexit=True) as output: # Test new behaviour with both upper and lower case # trailing characters argv = ["", "upload", "-S", "1B", "container", "object"] @@ -542,18 +625,24 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) _check_expected(mock_swift, 12345) - # Test invalid states - argv = ["", "upload", "-S", "1234X", "container", "object"] - swiftclient.shell.main(argv) - mock_out.error.assert_called_with("Invalid segment size") + with CaptureOutput() as output: + with ExpectedException(SystemExit): + # Test invalid states + argv = ["", "upload", "-S", "1234X", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "Invalid segment size\n") + output.clear() - argv = ["", "upload", "-S", "K1234", "container", "object"] - swiftclient.shell.main(argv) - mock_out.error.assert_called_with("Invalid segment size") + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "K1234", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "Invalid segment size\n") + output.clear() - argv = ["", "upload", "-S", "K", "container", "object"] - swiftclient.shell.main(argv) - mock_out.error.assert_called_with("Invalid segment size") + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "Invalid segment size\n") class TestSubcommandHelp(unittest.TestCase): @@ -562,20 +651,18 @@ class TestSubcommandHelp(unittest.TestCase): for command in swiftclient.shell.commands: help_var = 'st_%s_help' % command self.assertTrue(help_var in vars(swiftclient.shell)) - out = six.StringIO() - with mock.patch('sys.stdout', out): + with CaptureOutput() as out: argv = ['', command, '--help'] self.assertRaises(SystemExit, swiftclient.shell.main, argv) expected = vars(swiftclient.shell)[help_var] - self.assertEqual(out.getvalue().strip('\n'), expected) + self.assertEqual(out.strip('\n'), expected) def test_no_help(self): - out = six.StringIO() - with mock.patch('sys.stdout', out): + with CaptureOutput() as out: argv = ['', 'bad_command', '--help'] self.assertRaises(SystemExit, swiftclient.shell.main, argv) expected = 'no help for bad_command' - self.assertEqual(out.getvalue().strip('\n'), expected) + self.assertEqual(out.strip('\n'), expected) class TestParsing(unittest.TestCase): @@ -583,7 +670,7 @@ class TestParsing(unittest.TestCase): def setUp(self): super(TestParsing, self).setUp() self._environ_vars = {} - keys = os.environ.keys() + keys = list(os.environ.keys()) for k in keys: if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or k.startswith('OS_')): @@ -790,21 +877,15 @@ class TestParsing(unittest.TestCase): "tenant_name": "", "tenant_id": ""} - out = six.StringIO() - err = six.StringIO() - mock_output = _make_output_manager(out, err) - with mock.patch('swiftclient.shell.OutputManager', mock_output): + with CaptureOutput() as output: args = _make_args("stat", {}, os_opts) self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(err.getvalue().strip(), 'No tenant specified') + self.assertEqual(output.err.strip(), 'No tenant specified') - out = six.StringIO() - err = six.StringIO() - mock_output = _make_output_manager(out, err) - with mock.patch('swiftclient.shell.OutputManager', mock_output): + with CaptureOutput() as output: args = _make_args("stat", {}, os_opts, cmd_args=["testcontainer"]) self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(err.getvalue().strip(), 'No tenant specified') + self.assertEqual(output.err.strip(), 'No tenant specified') def test_no_tenant_name_or_id_v3(self): os_opts = {"password": "secret", @@ -813,23 +894,17 @@ class TestParsing(unittest.TestCase): "tenant_name": "", "tenant_id": ""} - out = six.StringIO() - err = six.StringIO() - mock_output = _make_output_manager(out, err) - with mock.patch('swiftclient.shell.OutputManager', mock_output): + with CaptureOutput() as output: args = _make_args("stat", {"auth_version": "3"}, os_opts) self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(err.getvalue().strip(), + self.assertEqual(output.err.strip(), 'No project name or project id specified.') - out = six.StringIO() - err = six.StringIO() - mock_output = _make_output_manager(out, err) - with mock.patch('swiftclient.shell.OutputManager', mock_output): + with CaptureOutput() as output: args = _make_args("stat", {"auth_version": "3"}, os_opts, cmd_args=["testcontainer"]) self.assertRaises(SystemExit, swiftclient.shell.main, args) - self.assertEqual(err.getvalue().strip(), + self.assertEqual(output.err.strip(), 'No project name or project id specified.') def test_insufficient_env_vars_v3(self): @@ -858,10 +933,8 @@ class TestParsing(unittest.TestCase): opts = {"help": ""} os_opts = {} args = _make_args("stat", opts, os_opts) - mock_stdout = six.StringIO() - with mock.patch('sys.stdout', mock_stdout): + with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) - out = mock_stdout.getvalue() self.assertTrue(out.find('[--key <api_key>]') > 0) self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) @@ -872,20 +945,16 @@ class TestParsing(unittest.TestCase): # "username": "user", # "auth_url": "http://example.com:5000/v3"} args = _make_args("", opts, os_opts) - mock_stdout = six.StringIO() - with mock.patch('sys.stdout', mock_stdout): + with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) - out = mock_stdout.getvalue() self.assertTrue(out.find('[--key <api_key>]') > 0) self.assertEqual(-1, out.find('--os-username=<auth-user-name>')) ## --os-help return os options help opts = {} args = _make_args("", opts, os_opts) - mock_stdout = six.StringIO() - with mock.patch('sys.stdout', mock_stdout): + with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) - out = mock_stdout.getvalue() self.assertTrue(out.find('[--key <api_key>]') > 0) self.assertTrue(out.find('--os-username=<auth-user-name>') > 0) @@ -1152,16 +1221,94 @@ class TestKeystoneOptions(MockHttpTest): self._test_options(opts, os_opts) -def _make_output_manager(stdout, stderr): - class MockOutputManager(OutputManager): - # This class is used to mock OutputManager so that we can - # override stdout and stderr. Mocking sys.stdout & sys.stdout - # doesn't work because they are argument defaults in the - # OutputManager constructor and those defaults are pinned to - # the value of sys.stdout/stderr before we get chance to mock them. - def __init__(self, print_stream=None, error_stream=None): - super(MockOutputManager, self).__init__() - self.print_stream = stdout - self.error_stream = stderr +@mock.patch.dict(os.environ, clean_os_environ) +class TestAuth(MockHttpTest): + + def test_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' - return MockOutputManager + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + pre_auth_env.update(mocked_os_environ) + pre_auth_env['OS_AUTH_TOKEN'] = 'expired' + fake_conn = self.fake_http_connection(401, 200, 200, headers={ + 'x-auth-token': token + '_new', + 'x-storage-url': url + '_not_used', + }) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('GET', mocked_os_environ['ST_AUTH'], '', { + 'x-auth-user': mocked_os_environ['ST_USER'], + 'x-auth-key': mocked_os_environ['ST_KEY'], + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) + + def test_os_pre_authed_request(self): + url = 'https://swift.storage.example.com/v1/AUTH_test' + token = 'AUTH_tk5b6b12' + + pre_auth_env = { + 'OS_STORAGE_URL': url, + 'OS_AUTH_TOKEN': token, + } + fake_conn = self.fake_http_connection(200) + with mock.patch('swiftclient.client.http_connection', new=fake_conn): + with mock.patch.dict(os.environ, pre_auth_env): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', {'x-auth-token': token}), + ]) + + # and again with re-auth + os_environ = { + 'OS_AUTH_URL': 'https://keystone.example.com/v2.0/', + 'OS_TENANT_NAME': 'demo', + 'OS_USERNAME': 'demo', + 'OS_PASSWORD': 'admin', + } + os_environ.update(pre_auth_env) + os_environ['OS_AUTH_TOKEN'] = 'expired' + + fake_conn = self.fake_http_connection(401, 200) + fake_keystone = fake_get_auth_keystone(storage_url=url + '_not_used', + token=token + '_new') + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + get_auth_keystone=fake_keystone, + sleep=mock.DEFAULT): + with mock.patch.dict(os.environ, os_environ): + argv = ['', 'stat'] + swiftclient.shell.main(argv) + self.assertRequests([ + ('HEAD', url, '', { + 'x-auth-token': 'expired', + }), + ('HEAD', url, '', { + 'x-auth-token': token + '_new', + }), + ]) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index e366f7d..0360016 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: More tests import logging +import json try: from unittest import mock @@ -29,11 +29,11 @@ import warnings from six.moves.urllib.parse import urlparse from six.moves import reload_module -# TODO: mock http connection class with more control over headers -from .utils import MockHttpTest, fake_get_auth_keystone +from .utils import MockHttpTest, fake_get_auth_keystone, StubResponse from swiftclient import client as c import swiftclient.utils +import swiftclient class TestClientException(testtools.TestCase): @@ -65,12 +65,7 @@ class TestClientException(testtools.TestCase): class TestJsonImport(testtools.TestCase): def tearDown(self): - try: - import json - except ImportError: - pass - else: - reload_module(json) + reload_module(json) try: import simplejson @@ -83,7 +78,7 @@ class TestJsonImport(testtools.TestCase): def test_any(self): self.assertTrue(hasattr(c, 'json_loads')) - def test_no_simplejson(self): + def test_no_simplejson_falls_back_to_stdlib_when_reloaded(self): # break simplejson try: import simplejson @@ -91,16 +86,10 @@ class TestJsonImport(testtools.TestCase): # not installed, so we don't have to break it for these tests pass else: - delattr(simplejson, 'loads') - reload_module(c) + delattr(simplejson, 'loads') # break simple json + reload_module(c) # reload to repopulate json_loads - try: - from json import loads - except ImportError: - # this case is stested in _no_json - pass - else: - self.assertEqual(loads, c.json_loads) + self.assertEqual(c.json_loads, json.loads) class MockHttpResponse(): @@ -160,6 +149,25 @@ class TestHttpHelpers(MockHttpTest): url = 'ftp://www.test.com' self.assertRaises(c.ClientException, c.http_connection, url) + def test_encode_meta_headers(self): + headers = {'abc': '123', + u'x-container-meta-\u0394': '123', + u'x-account-meta-\u0394': '123', + u'x-object-meta-\u0394': '123'} + + encoded_str_type = type(''.encode()) + r = swiftclient.encode_meta_headers(headers) + + self.assertEqual(len(headers), len(r)) + # ensure non meta headers are not encoded + self.assertTrue('abc' in r) + self.assertTrue(isinstance(r['abc'], encoded_str_type)) + del r['abc'] + + for k, v in r.items(): + self.assertTrue(isinstance(k, encoded_str_type)) + self.assertTrue(isinstance(v, encoded_str_type)) + def test_set_user_agent_default(self): _junk, conn = c.http_connection('http://www.example.com') req_headers = {} @@ -214,7 +222,6 @@ class TestGetAuth(MockHttpTest): self.assertEqual(token, None) def test_invalid_auth(self): - c.http_connection = self.fake_http_connection(200) self.assertRaises(c.ClientException, c.get_auth, 'http://www.tests.com', 'asdf', 'asdf', auth_version="foo") @@ -227,7 +234,7 @@ class TestGetAuth(MockHttpTest): self.assertEqual(token, 'someauthtoken') def test_auth_v1_insecure(self): - c.http_connection = self.fake_http_connection(200, auth_v1=True) + c.http_connection = self.fake_http_connection(200, 200, auth_v1=True) url, token = c.get_auth('http://www.test.com/invalid_cert', 'asdf', 'asdf', auth_version='1.0', @@ -235,10 +242,12 @@ class TestGetAuth(MockHttpTest): self.assertEqual(url, 'storageURL') self.assertEqual(token, 'someauthtoken') - self.assertRaises(c.ClientException, c.get_auth, - 'http://www.test.com/invalid_cert', - 'asdf', 'asdf', - auth_version='1.0') + e = self.assertRaises(c.ClientException, c.get_auth, + 'http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') + # TODO: this test is really on validating the mock and not the + # the full plumbing into the requests's 'verify' option + self.assertIn('invalid_certificate', str(e)) def test_auth_v2_with_tenant_name(self): os_options = {'tenant_name': 'asdf'} @@ -479,23 +488,29 @@ class TestGetAccount(MockHttpTest): class TestHeadAccount(MockHttpTest): def test_ok(self): - c.http_connection = self.fake_http_connection(200) - value = c.head_account('http://www.tests.com', 'asdf') - # TODO: Hmm. This doesn't really test too much as it uses a fake that - # always returns the same dict. I guess it "exercises" the code, so - # I'll leave it for now. - self.assertEqual(type(value), dict) + c.http_connection = self.fake_http_connection(200, headers={ + 'x-account-meta-color': 'blue', + }) + resp_headers = c.head_account('http://www.tests.com', 'asdf') + self.assertEqual(resp_headers['x-account-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - self.assertRaises(c.ClientException, c.head_account, - 'http://www.tests.com', 'asdf') - try: - c.head_account('http://www.tests.com', 'asdf') - except c.ClientException as e: - new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(e.__str__()[-89:], new_body) + e = self.assertRaises(c.ClientException, c.head_account, + 'http://www.tests.com', 'asdf') + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(e.__str__()[-89:], new_body) class TestGetContainer(MockHttpTest): @@ -546,16 +561,29 @@ class TestGetContainer(MockHttpTest): class TestHeadContainer(MockHttpTest): + def test_head_ok(self): + fake_conn = self.fake_http_connection( + 200, headers={'x-container-meta-color': 'blue'}) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + resp = c.head_container('https://example.com/v1/AUTH_test', + 'token', 'container') + self.assertEqual(resp['x-container-meta-color'], 'blue') + self.assertRequests([ + ('HEAD', 'https://example.com/v1/AUTH_test/container', '', + {'x-auth-token': 'token'}), + ]) + def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - self.assertRaises(c.ClientException, c.head_container, - 'http://www.test.com', 'asdf', 'asdf', - ) - try: - c.head_container('http://www.test.com', 'asdf', 'asdf') - except c.ClientException as e: - self.assertEqual(e.http_response_content, body) + e = self.assertRaises(c.ClientException, c.head_container, + 'http://www.test.com', 'asdf', 'container') + self.assertRequests([ + ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), + ]) + self.assertEqual(e.http_status, 500) + self.assertEqual(e.http_response_content, body) class TestPutContainer(MockHttpTest): @@ -568,13 +596,12 @@ class TestPutContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - self.assertRaises(c.ClientException, c.put_container, - 'http://www.test.com', 'asdf', 'asdf', - ) - try: - c.put_container('http://www.test.com', 'asdf', 'asdf') - except c.ClientException as e: - self.assertEqual(e.http_response_content, body) + e = self.assertRaises(c.ClientException, c.put_container, + 'http://www.test.com', 'token', 'container') + self.assertEqual(e.http_response_content, body) + self.assertRequests([ + ('PUT', '/container', '', {'x-auth-token': 'token'}), + ]) class TestDeleteContainer(MockHttpTest): @@ -597,26 +624,25 @@ class TestGetObject(MockHttpTest): query_string="hello=20") c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf', query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['parsed_path'].path, '/asdf/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['body'], '') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') def test_request_headers(self): - request_args = {} - - def fake_request(method, url, body=None, headers=None): - request_args['method'] = method - request_args['url'] = url - request_args['body'] = body - request_args['headers'] = headers - return - conn = self.fake_http_connection(200)('http://www.test.com/') - conn[1].request = fake_request + c.http_connection = self.fake_http_connection(200) + conn = c.http_connection('http://www.test.com') headers = {'Range': 'bytes=1-2'} c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', http_conn=conn, headers=headers) - self.assertFalse(request_args['headers'] is None, - "No headers in the request") - self.assertTrue('Range' in request_args['headers'], - "No Range header in the request") - self.assertEqual(request_args['headers']['Range'], 'bytes=1-2') + self.assertRequests([ + ('GET', '/container/object', '', { + 'x-auth-token': 'TOKEN', + 'range': 'bytes=1-2', + }), + ]) class TestHeadObject(MockHttpTest): @@ -682,17 +708,23 @@ class TestPutObject(MockHttpTest): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - self.assertRaises(c.ClientException, c.put_object, *args) - try: - c.put_object(*args) - except c.ClientException as e: - self.assertEqual(e.http_response_content, body) + e = self.assertRaises(c.ClientException, c.put_object, *args) + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('PUT', '/asdf/asdf', 'asdf', {'x-auth-token': 'asdf'}), + ]) def test_query_string(self): c.http_connection = self.fake_http_connection(200, query_string="hello=20") c.put_object('http://www.test.com', 'asdf', 'asdf', 'asdf', query_string="hello=20") + for req in self.iter_request_log(): + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['parsed_path'].path, '/asdf/asdf') + self.assertEqual(req['parsed_path'].query, 'hello=20') + self.assertEqual(req['headers']['x-auth-token'], 'asdf') def test_raw_upload(self): # Raw upload happens when content_length is passed to put_object @@ -801,12 +833,14 @@ class TestPostObject(MockHttpTest): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {}) - self.assertRaises(c.ClientException, c.post_object, *args) - try: - c.post_object(*args) - except c.ClientException as e: - self.assertEqual(e.http_response_content, body) + args = ('http://www.test.com', 'token', 'container', 'obj', {}) + e = self.assertRaises(c.ClientException, c.post_object, *args) + self.assertEqual(e.http_response_content, body) + self.assertRequests([ + ('POST', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) class TestDeleteObject(MockHttpTest): @@ -832,14 +866,112 @@ class TestGetCapabilities(MockHttpTest): def test_ok(self): conn = self.fake_http_connection(200, body='{}') http_conn = conn('http://www.test.com/info') - self.assertEqual(type(c.get_capabilities(http_conn)), dict) - self.assertTrue(http_conn[1].has_been_read) + info = c.get_capabilities(http_conn) + self.assertRequests([ + ('GET', '/info'), + ]) + self.assertEqual(info, {}) + self.assertTrue(http_conn[1].resp.has_been_read) def test_server_error(self): conn = self.fake_http_connection(500) http_conn = conn('http://www.test.com/info') self.assertRaises(c.ClientException, c.get_capabilities, http_conn) + def test_conn_get_capabilities_with_auth(self): + auth_headers = { + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_test' + } + auth_v1_response = StubResponse(headers=auth_headers) + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=json.dumps(stub_info)) + fake_conn = self.fake_http_connection(auth_v1_response, info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', '/auth/v1.0'), + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_os_auth(self): + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_test') + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=json.dumps(stub_info)) + fake_conn = self.fake_http_connection(info_response) + + os_options = {'project_id': 'test'} + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_url_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=json.dumps(stub_info)) + fake_conn = self.fake_http_connection(info_response) + + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key') + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities( + 'http://other-storage.example.com/info') + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://other-storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_preauthurl_param(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=json.dumps(stub_info)) + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + conn = c.Connection('http://auth.example.com/auth/v1.0', + 'user', 'key', preauthurl=storage_url) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + + def test_conn_get_capabilities_with_os_options(self): + stub_info = {'swift': {'fake': True}} + info_response = StubResponse(body=json.dumps(stub_info)) + fake_conn = self.fake_http_connection(info_response) + + storage_url = 'http://storage.example.com/v1/AUTH_test' + os_options = { + 'project_id': 'test', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://keystone.example.com/v3.0', + 'user', 'key', os_options=os_options, + auth_version=3) + with mock.patch('swiftclient.client.http_connection', + new=fake_conn): + info = conn.get_capabilities() + self.assertEqual(info, stub_info) + self.assertRequests([ + ('GET', 'http://storage.example.com/info'), + ]) + class TestHTTPConnection(MockHttpTest): @@ -883,12 +1015,39 @@ class TestConnection(MockHttpTest): args = {'preauthtoken': 'atoken123', 'preauthurl': 'http://www.test.com:8080/v1/AUTH_123456'} conn = c.Connection(**args) - self.assertEqual(type(conn), c.Connection) + self.assertEqual(conn.url, args['preauthurl']) + self.assertEqual(conn.token, args['preauthtoken']) + + def test_instance_kwargs_os_token(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'os_options': { + 'object_storage_url': storage_url, + 'auth_token': token, + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) + + def test_instance_kwargs_token_precedence(self): + storage_url = 'http://storage.example.com/v1/AUTH_test' + token = 'token' + args = { + 'preauthurl': storage_url, + 'preauthtoken': token, + 'os_options': { + 'auth_token': 'less-specific-token', + 'object_storage_url': 'less-specific-storage-url', + } + } + conn = c.Connection(**args) + self.assertEqual(conn.url, storage_url) + self.assertEqual(conn.token, token) def test_storage_url_override(self): static_url = 'http://overridden.storage.url' - c.http_connection = self.fake_http_connection( - 200, body='[]', storage_url=static_url) conn = c.Connection('http://auth.url/', 'some_user', 'some_key', os_options={ 'object_storage_url': static_url}) @@ -910,7 +1069,15 @@ class TestConnection(MockHttpTest): mock_get_auth.return_value = ('http://auth.storage.url', 'tToken') for method, args in method_signatures: + c.http_connection = self.fake_http_connection( + 200, body='[]', storage_url=static_url) method(*args) + self.assertEqual(len(self.request_log), 1) + for request in self.iter_request_log(): + self.assertEqual(request['parsed_path'].netloc, + 'overridden.storage.url') + self.assertEqual(request['headers']['x-auth-token'], + 'tToken') def test_get_capabilities(self): conn = c.Connection() @@ -927,35 +1094,46 @@ class TestConnection(MockHttpTest): self.assertEqual(parsed.netloc, 'storage.test.com') def test_retry(self): - c.http_connection = self.fake_http_connection(500) - def quick_sleep(*args): pass c.sleep = quick_sleep conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + code_iter = [500] * (conn.retries + 1) + c.http_connection = self.fake_http_connection(*code_iter) + self.assertRaises(c.ClientException, conn.head_account) self.assertEqual(conn.attempts, conn.retries + 1) def test_retry_on_ratelimit(self): - c.http_connection = self.fake_http_connection(498) def quick_sleep(*args): pass c.sleep = quick_sleep # test retries - conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', retry_on_ratelimit=True) - self.assertRaises(c.ClientException, conn.head_account) + code_iter = [200] + [498] * (conn.retries + 1) + auth_resp_headers = { + 'x-auth-token': 'asdf', + 'x-storage-url': 'http://storage/v1/test', + } + c.http_connection = self.fake_http_connection( + *code_iter, headers=auth_resp_headers) + e = self.assertRaises(c.ClientException, conn.head_account) + self.assertIn('Account HEAD failed', str(e)) self.assertEqual(conn.attempts, conn.retries + 1) # test default no-retry - conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - self.assertRaises(c.ClientException, conn.head_account) + c.http_connection = self.fake_http_connection( + 200, 498, + headers=auth_resp_headers) + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') + e = self.assertRaises(c.ClientException, conn.head_account) + self.assertIn('Account HEAD failed', str(e)) self.assertEqual(conn.attempts, 1) def test_resp_read_on_server_error(self): - c.http_connection = self.fake_http_connection(500) conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) def get_auth(*args, **kwargs): @@ -978,25 +1156,28 @@ class TestConnection(MockHttpTest): ) for method, args in method_signatures: + c.http_connection = self.fake_http_connection(500) self.assertRaises(c.ClientException, method, *args) - try: - self.assertTrue(conn.http_conn[1].has_been_read) - except AssertionError: + requests = list(self.iter_request_log()) + self.assertEqual(len(requests), 1) + for req in requests: msg = '%s did not read resp on server error' % method.__name__ - self.fail(msg) - except Exception as e: - raise e.__class__("%s - %s" % (method.__name__, e)) + self.assertTrue(req['resp'].has_been_read, msg) def test_reauth(self): - c.http_connection = self.fake_http_connection(401) + c.http_connection = self.fake_http_connection(401, 200) def get_auth(*args, **kwargs): + # this mock, and by extension this test are not + # represenative of the unit under test. The real get_auth + # method will always return the os_option dict's + # object_storage_url which will be overridden by the + # preauthurl paramater to Connection if it is provided. return 'http://www.new.com', 'new' def swap_sleep(*args): self.swap_sleep_called = True c.get_auth = get_auth - c.http_connection = self.fake_http_connection(200) c.sleep = swap_sleep self.swap_sleep_called = False @@ -1016,6 +1197,129 @@ class TestConnection(MockHttpTest): self.assertEqual(conn.url, 'http://www.new.com') self.assertEqual(conn.token, 'new') + def test_reauth_preauth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl='http://storage.example.com/v1/AUTH_test', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(401, auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_reauth_os_preauth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': 'http://storage.example.com/v1/AUTH_test', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone(os_preauth_options) + fake_conn = self.fake_http_connection(401, 200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'expired'}), + ('HEAD', '/v1/AUTH_test', '', {'x-auth-token': 'token'}), + ]) + + def test_preauth_token_with_no_storage_url_requires_auth(self): + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthtoken='expired') + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_user', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_os_preauth_token_with_no_storage_url_requires_auth(self): + os_preauth_options = { + 'tenant_name': 'demo', + 'auth_token': 'expired', + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + storage_url = 'http://storage.example.com/v1/AUTH_user' + fake_keystone = fake_get_auth_keystone(storage_url=storage_url) + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_user', '', {'x-auth-token': 'token'}), + ]) + + def test_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + conn = c.Connection( + 'http://auth.example.com', 'user', 'password', + preauthurl=storage_url) + auth_v1_response = StubResponse(200, headers={ + 'x-auth-token': 'post_token', + 'x-storage-url': 'http://storage.example.com/v1/AUTH_post_url', + }) + fake_conn = self.fake_http_connection(auth_v1_response, 200) + with mock.patch.multiple('swiftclient.client', + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('GET', 'http://auth.example.com', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'password'}), + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + def test_os_preauth_url_trumps_auth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': storage_url, + } + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2) + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + def test_reset_stream(self): class LocalContents(object): @@ -1243,30 +1547,16 @@ class TestLogging(MockHttpTest): 'http://www.test.com', 'asdf', 'asdf', 'asdf') def test_get_error(self): - body = 'c' * 65 - conn = self.fake_http_connection( - 404, body=body)('http://www.test.com/') - request_args = {} - - def fake_request(method, url, body=None, headers=None): - request_args['method'] = method - request_args['url'] = url - request_args['body'] = body - request_args['headers'] = headers - return - conn[1].request = fake_request - headers = {'Range': 'bytes=1-2'} - self.assertRaises( - c.ClientException, - c.get_object, - 'url_is_irrelevant', 'TOKEN', 'container', 'object', - http_conn=conn, headers=headers) + c.http_connection = self.fake_http_connection(404) + e = self.assertRaises(c.ClientException, c.get_object, + 'http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(e.http_status, 404) class TestCloseConnection(MockHttpTest): def test_close_none(self): - c.http_connection = self.fake_http_connection(200) + c.http_connection = self.fake_http_connection() conn = c.Connection('http://www.test.com', 'asdf', 'asdf') self.assertEqual(conn.http_conn, None) conn.close() @@ -1274,15 +1564,12 @@ class TestCloseConnection(MockHttpTest): def test_close_ok(self): url = 'http://www.test.com' - c.http_connection = self.fake_http_connection(200) conn = c.Connection(url, 'asdf', 'asdf') self.assertEqual(conn.http_conn, None) - conn.http_conn = c.http_connection(url) self.assertEqual(type(conn.http_conn), tuple) self.assertEqual(len(conn.http_conn), 2) http_conn_obj = conn.http_conn[1] - self.assertEqual(http_conn_obj.isclosed(), False) + self.assertIsInstance(http_conn_obj, c.HTTPConnection) + self.assertFalse(hasattr(http_conn_obj, 'close')) conn.close() - self.assertEqual(http_conn_obj.isclosed(), True) - self.assertEqual(conn.http_conn, None) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index c149abf..9d8aacc 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -12,22 +12,37 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools +import sys from requests import RequestException +from requests.structures import CaseInsensitiveDict from time import sleep +import unittest import testtools +import mock +import six from six.moves import reload_module +from six.moves.urllib.parse import urlparse, ParseResult from swiftclient import client as c +from swiftclient import shell as s -def fake_get_auth_keystone(os_options, exc=None, **kwargs): +def fake_get_auth_keystone(expected_os_options=None, exc=None, + storage_url='http://url/', token='token', + **kwargs): def fake_get_auth_keystone(auth_url, user, key, actual_os_options, **actual_kwargs): if exc: raise exc('test') - if actual_os_options != os_options: + # TODO: some way to require auth_url, user and key? + if expected_os_options and actual_os_options != expected_os_options: return "", None + if 'required_kwargs' in kwargs: + for k, v in kwargs['required_kwargs'].items(): + if v != actual_kwargs.get(k): + return "", None if auth_url.startswith("https") and \ auth_url.endswith("invalid-certificate") and \ @@ -40,20 +55,36 @@ def fake_get_auth_keystone(os_options, exc=None, **kwargs): actual_kwargs['cacert'] is None: from swiftclient import client as c raise c.ClientException("unverified-certificate") - if 'required_kwargs' in kwargs: - for k, v in kwargs['required_kwargs'].items(): - if v != actual_kwargs.get(k): - return "", None - return "http://url/", "token" + return storage_url, token return fake_get_auth_keystone +class StubResponse(object): + """ + Placeholder structure for use with fake_http_connect's code_iter to modify + response attributes (status, body, headers) on a per-request basis. + """ + + def __init__(self, status=200, body='', headers=None): + self.status = status + self.body = body + self.headers = headers or {} + + def fake_http_connect(*code_iter, **kwargs): + """ + Generate a callable which yields a series of stubbed responses. Because + swiftclient will reuse an HTTP connection across pipelined requests it is + not always the case that this fake is used strictly for mocking an HTTP + connection, but rather each HTTP response (i.e. each call to requests + get_response). + """ class FakeConn(object): - def __init__(self, status, etag=None, body='', timestamp='1'): + def __init__(self, status, etag=None, body='', timestamp='1', + headers=None): self.status = status self.reason = 'Fake' self.host = '1.2.3.4' @@ -64,6 +95,7 @@ def fake_http_connect(*code_iter, **kwargs): self.body = body self.timestamp = timestamp self._is_closed = True + self.headers = headers or {} def connect(self): self._is_closed = False @@ -87,6 +119,8 @@ def fake_http_connect(*code_iter, **kwargs): return FakeConn(100) def getheaders(self): + if self.headers: + return self.headers.items() headers = {'content-length': len(self.body), 'content-type': 'x-application/test', 'x-timestamp': self.timestamp, @@ -149,15 +183,20 @@ def fake_http_connect(*code_iter, **kwargs): if 'give_connect' in kwargs: kwargs['give_connect'](*args, **ckwargs) status = next(code_iter) - etag = next(etag_iter) - timestamp = next(timestamps_iter) - if status <= 0: + if isinstance(status, StubResponse): + fake_conn = FakeConn(status.status, body=status.body, + headers=status.headers) + else: + etag = next(etag_iter) + timestamp = next(timestamps_iter) + fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), + timestamp=timestamp) + if fake_conn.status <= 0: raise RequestException() - fake_conn = FakeConn(status, etag, body=kwargs.get('body', ''), - timestamp=timestamp) fake_conn.connect() return fake_conn + connect.code_iter = code_iter return connect @@ -165,10 +204,14 @@ class MockHttpTest(testtools.TestCase): def setUp(self): super(MockHttpTest, self).setUp() + self.fake_connect = None + self.request_log = [] def fake_http_connection(*args, **kwargs): + self.validateMockedRequestsConsumed() + self.request_log = [] + self.fake_connect = fake_http_connect(*args, **kwargs) _orig_http_connection = c.http_connection - return_read = kwargs.get('return_read') query_string = kwargs.get('query_string') storage_url = kwargs.get('storage_url') auth_token = kwargs.get('auth_token') @@ -180,9 +223,28 @@ class MockHttpTest(testtools.TestCase): self.assertEqual(storage_url, url) parsed, _conn = _orig_http_connection(url, proxy=proxy) - conn = fake_http_connect(*args, **kwargs)() + + class RequestsWrapper(object): + pass + conn = RequestsWrapper() def request(method, url, *args, **kwargs): + try: + conn.resp = self.fake_connect() + except StopIteration: + self.fail('Unexpected %s request for %s' % ( + method, url)) + self.request_log.append((parsed, method, url, args, + kwargs, conn.resp)) + conn.host = conn.resp.host + conn.isclosed = conn.resp.isclosed + conn.resp.has_been_read = False + _orig_read = conn.resp.read + + def read(*args, **kwargs): + conn.resp.has_been_read = True + return _orig_read(*args, **kwargs) + conn.resp.read = read if auth_token: headers = args[1] self.assertTrue('X-Auth-Token' in headers) @@ -193,23 +255,169 @@ class MockHttpTest(testtools.TestCase): if url.endswith('invalid_cert') and not insecure: from swiftclient import client as c raise c.ClientException("invalid_certificate") - elif exc: + if exc: raise exc - return + return conn.resp conn.request = request - conn.has_been_read = False - _orig_read = conn.read - - def read(*args, **kwargs): - conn.has_been_read = True - return _orig_read(*args, **kwargs) - conn.read = return_read or read + def getresponse(): + return conn.resp + conn.getresponse = getresponse return parsed, conn return wrapper self.fake_http_connection = fake_http_connection + def iter_request_log(self): + for parsed, method, path, args, kwargs, resp in self.request_log: + parts = parsed._asdict() + parts['path'] = path + full_path = ParseResult(**parts).geturl() + args = list(args) + log = dict(zip(('body', 'headers'), args)) + log.update({ + 'method': method, + 'full_path': full_path, + 'parsed_path': urlparse(full_path), + 'path': path, + 'headers': CaseInsensitiveDict(log.get('headers')), + 'resp': resp, + 'status': resp.status, + }) + yield log + + orig_assertEqual = unittest.TestCase.assertEqual + + def assertRequests(self, expected_requests): + """ + Make sure some requests were made like you expected, provide a list of + expected requests, typically in the form of [(method, path), ...] + """ + real_requests = self.iter_request_log() + for expected in expected_requests: + method, path = expected[:2] + real_request = next(real_requests) + if urlparse(path).scheme: + match_path = real_request['full_path'] + else: + match_path = real_request['path'] + self.assertEqual((method, path), (real_request['method'], + match_path)) + if len(expected) > 2: + body = expected[2] + real_request['expected'] = body + err_msg = 'Body mismatch for %(method)s %(path)s, ' \ + 'expected %(expected)r, and got %(body)r' % real_request + self.orig_assertEqual(body, real_request['body'], err_msg) + + if len(expected) > 3: + headers = expected[3] + for key, value in headers.items(): + real_request['key'] = key + real_request['expected_value'] = value + real_request['value'] = real_request['headers'].get(key) + err_msg = ( + 'Header mismatch on %(key)r, ' + 'expected %(expected_value)r and got %(value)r ' + 'for %(method)s %(path)s %(headers)r' % real_request) + self.orig_assertEqual(value, real_request['value'], + err_msg) + + def validateMockedRequestsConsumed(self): + if not self.fake_connect: + return + unused_responses = list(self.fake_connect.code_iter) + if unused_responses: + self.fail('Unused responses %r' % (unused_responses,)) + def tearDown(self): + self.validateMockedRequestsConsumed() super(MockHttpTest, self).tearDown() + # TODO: this nuke from orbit clean up seems to be encouraging + # un-hygienic mocking on the swiftclient.client module; which may lead + # to some unfortunate test order dependency bugs by way of the broken + # window theory if any other modules are similarly patched reload_module(c) + + +class CaptureStream(object): + + def __init__(self, stream): + self.stream = stream + self._capture = six.StringIO() + self.streams = [self.stream, self._capture] + + def write(self, *args, **kwargs): + for stream in self.streams: + stream.write(*args, **kwargs) + + def writelines(self, *args, **kwargs): + for stream in self.streams: + stream.writelines(*args, **kwargs) + + def getvalue(self): + return self._capture.getvalue() + + def clear(self): + self._capture.truncate(0) + self._capture.seek(0) + + +class CaptureOutput(object): + + def __init__(self, suppress_systemexit=False): + self._out = CaptureStream(sys.stdout) + self._err = CaptureStream(sys.stderr) + self.patchers = [] + + WrappedOutputManager = functools.partial(s.OutputManager, + print_stream=self._out, + error_stream=self._err) + + if suppress_systemexit: + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager.get_error_count', + return_value=0) + ] + + self.patchers += [ + mock.patch('swiftclient.shell.OutputManager', + WrappedOutputManager), + mock.patch('sys.stdout', self._out), + mock.patch('sys.stderr', self._err), + ] + + def __enter__(self): + for patcher in self.patchers: + patcher.start() + return self + + def __exit__(self, *args, **kwargs): + for patcher in self.patchers: + patcher.stop() + + @property + def out(self): + return self._out.getvalue() + + @property + def err(self): + return self._err.getvalue() + + def clear(self): + self._out.clear() + self._err.clear() + + # act like the string captured by stdout + + def __str__(self): + return self.out + + def __len__(self): + return len(self.out) + + def __eq__(self, other): + return self.out == other + + def __getattr__(self, name): + return getattr(self.out, name) |