diff options
| -rw-r--r-- | .mailmap | 3 | ||||
| -rwxr-xr-x | .manpages | 18 | ||||
| -rw-r--r-- | AUTHORS | 13 | ||||
| -rw-r--r-- | ChangeLog | 42 | ||||
| -rw-r--r-- | doc/source/apis.rst | 137 | ||||
| -rw-r--r-- | swiftclient/client.py | 82 | ||||
| -rw-r--r-- | swiftclient/service.py | 2 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 2 | ||||
| -rw-r--r-- | test-requirements.txt | 2 | ||||
| -rw-r--r-- | tests/functional/test_swiftclient.py | 4 | ||||
| -rw-r--r-- | tests/unit/test_command_helpers.py | 4 | ||||
| -rw-r--r-- | tests/unit/test_multithreading.py | 6 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 108 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 37 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 185 | ||||
| -rw-r--r-- | tests/unit/test_utils.py | 14 | ||||
| -rw-r--r-- | tests/unit/utils.py | 3 | ||||
| -rw-r--r-- | tox.ini | 4 |
18 files changed, 458 insertions, 208 deletions
@@ -86,3 +86,6 @@ Stanislaw Pitucha <stanislaw.pitucha@hpe.com> <stanislaw.pitucha@hp.com> Mahati Chamarthy <mahati.chamarthy@gmail.com> Peter Lisak <peter.lisak@firma.seznam.cz> Doug Hellmann <doug@doughellmann.com> <doug.hellmann@dreamhost.com> +Ondrej Novy <ondrej.novy@firma.seznam.cz> +James Nzomo <james@tdt.rocks> <kazikubwa@gmail.com> +Alessandro Pilotti <ap@pilotti.it> <apilotti@cloudbasesolutions.com> diff --git a/.manpages b/.manpages new file mode 100755 index 0000000..69fcfc7 --- /dev/null +++ b/.manpages @@ -0,0 +1,18 @@ +#!/bin/sh + +RET=0 +for MAN in doc/manpages/* ; do + OUTPUT=$(LC_ALL=en_US.UTF-8 MANROFFSEQ='' MANWIDTH=80 man --warnings -E UTF-8 -l \ + -Tutf8 -Z "$MAN" 2>&1 >/dev/null) + if [ -n "$OUTPUT" ] ; then + RET=1 + echo "$MAN:" + echo "$OUTPUT" + fi +done + +if [ "$RET" -eq "0" ] ; then + echo "All manpages are fine" +fi + +exit "$RET" @@ -10,11 +10,14 @@ Clint Byrum (clint@fewbar.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Mahati Chamarthy (mahati.chamarthy@gmail.com) +Chaozhe.Chen (chaozhe.chen@easystack.cn) Ray Chen (oldsharp@163.com) +Li Cheng (shcli@cn.ibm.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) Alistair Coles (alistair.coles@hpe.com) Ian Cordasco (ian.cordasco@rackspace.com) Nick Craig-Wood (nick@craig-wood.com) +Thiago da Silva (thiago@redhat.com) Sean Dague (sean@dague.net) Zack M. Davis (zdavis@swiftstack.com) John Dickinson (me@not.mn) @@ -38,7 +41,7 @@ Charles Hsu (charles0126@gmail.com) Kun Huang (gareth@unitedstack.com) Matthieu Huin (mhu@enovance.com) Andreas Jaeger (aj@suse.de) -OpenStack Jenkins (jenkins@openstack.org) +Jude Job (judeopenstack@gmail.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) Leah Klearman (lklrmn@gmail.com) Jaivish Kothari (jaivish.kothari@nectechnologies.in) @@ -52,6 +55,7 @@ Peter Lisak (peter.lisak@firma.seznam.cz) Feng Liu (mefengliu23@gmail.com) Jing Liuqing (jing.liuqing@99cloud.net) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +Pratik Mallya (pratik.mallya@gmail.com) Steve Martinelli (stevemar@ca.ibm.com) Juan J. Martinez (juan@memset.com) Donagh McCabe (donagh.mccabe@hpe.com) @@ -59,13 +63,14 @@ Ben McCann (ben@benmccann.com) Andy McCrae (andy.mccrae@gmail.com) Stuart McLaren (stuart.mclaren@hpe.com) Samuel Merritt (sam@swiftstack.com) +Min Min Ren (rminmin@cn.ibm.com) Jola Mirecka (jola.mirecka@hp.com) Hiroshi Miura (miurahr@nttdata.co.jp) Sam Morrison (sorrison@gmail.com) Dirk Mueller (dirk@dmllr.de) Zhenguo Niu (zhenguo@unitedstack.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) -Alessandro Pilotti (apilotti@cloudbasesolutions.com) +James Nzomo (james@tdt.rocks) Alessandro Pilotti (ap@pilotti.it) Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Dan Prince (dprince@redhat.com) @@ -77,6 +82,7 @@ Mark Seger (mark.seger@hpe.com) Chuck Short (chuck.short@canonical.com) David Shrewsbury (shrewsbury.dave@gmail.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Alexandra Settle (alexandra.settle@rackspace.com) Jeremy Stanley (fungi@yuggoth.org) Victor Stinner (victor.stinner@enovance.com) Jiří Suchomel (jsuchome@suse.cz) @@ -103,3 +109,6 @@ tanlin (lin.tan@intel.com) yangxurong (yangxurong@huawei.com) yuxcer (yuxcer@126.com) zhang-jinnan (ben.os@99cloud.net) +hgangwx (hgangwx@cn.ibm.com) +shu-mutou (shu-mutou@rf.jp.nec.com) +SaiKiran (saikiranveeravarapu@gmail.com) @@ -1,3 +1,45 @@ +3.0.0 +----- + +* Python 2.6 and Python 3.3 support has been removed. Currently + supported and tested versions of Python are Python 2.7 and Python 3.4. + +* Do not reveal sensitive headers in swiftclient log messages by default. + This is controlled by the client.logger_settings dictionary. Setting the + `redact_sensitive_headers` key to False prevents the information hiding. If + the value is True (the default), the `reveal_sensitive_prefix` controls + the maximum length of any sensitive header value logged. The default is + 16 to match the default in Swift. + +* Object downloads that fail partway through will now retry with a Range + request to read the rest of the object. + +* Object uploads will be retried if the source supports seek/tell or has a + reset() method. + +* Delete requests will use the cluster's bulk delete feature, if available, + for requests that would require a lot of individual deletes. + +* The delete CLI option now accepts a --prefix option to delete objects that + start with the given prefix (similar to the same-named option for list). + +* Add support for the auth-version to be specified using + --os-identity-api-version or OS_IDENTITY_API_VERSION + for compatibility with other openstack client command + line options. + +* --debug and --info command-line options now work anywhere in the command. + +* Objects can now be uploaded to pseudo-directories with the CLI. + +* Fixed an issue with uploading a large object that includes a unicode path. + +* swiftclient can now auth against Keystone using only a project (tenant) + and a token. This is useful when the client doesn't have access to the + password for a user but otherwise has been granted access. + +* Various other minor bug fixes and improvements. + 2.7.0 ----- diff --git a/doc/source/apis.rst b/doc/source/apis.rst index 1a8e8f7..935b4a4 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -1,18 +1,18 @@ -============ -Introduction -============ +====================== +python-swiftclient API +====================== -The python-swiftclient includes two levels of API; a low level client API that -provides simple python wrappers around the various authentication mechanisms -and the individual HTTP requests, and a high level service API that provides +The python-swiftclient includes two levels of API. A low level client API that +provides simple python wrappers around the various authentication mechanisms, +the individual HTTP requests, and a high level service API that provides methods for performing common operations in parallel on a thread pool. This document aims to provide guidance for choosing between these APIs and examples of usage for the service API. ------------------------- + Important Considerations ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ This section covers some important considerations, helpful hints, and things to avoid when integrating an object store into your workflow. @@ -20,27 +20,30 @@ to avoid when integrating an object store into your workflow. An Object Store is not a filesystem ----------------------------------- -It cannot be stressed enough that your usage of the object store should reflect -the proper use case, and not treat the storage like a filesystem. There are 2 -main restrictions to bear in mind here when designing your use of the object +.. important:: + + It cannot be stressed enough that your usage of the object store should reflect + the use case, and not treat the storage like a filesystem. + +There are 2 main restrictions to bear in mind here when designing your use of the object store: - * Objects cannot be renamed due to the way in which objects are stored and - references by the object store. This usually requires multiple copies of - the data to be moved between physical storage devices. - As a result, a move operation is not provided. If the user wants to move an - object they must re-upload to the new location and delete the - original. - * Objects cannot be modified. Objects are stored in multiple locations and are - checked for integrity based on the ``MD5 sum`` calculated during upload. - Object creation is a 1-shot event, and in order to modify the contents of an - object the entire new contents must be re-uploaded. In certain special cases - it is possible to work around this restriction using large objects, but no - general file-like access is available to modify a stored object. - ------------------------------- +#. Objects cannot be renamed due to the way in which objects are stored and + references by the object store. This usually requires multiple copies of + the data to be moved between physical storage devices. + As a result, a move operation is not provided. If the user wants to move an + object they must re-upload to the new location and delete the + original. +#. Objects cannot be modified. Objects are stored in multiple locations and are + checked for integrity based on the ``MD5 sum`` calculated during upload. + Object creation is a 1-shot event, and in order to modify the contents of an + object the entire new contents must be re-uploaded. In certain special cases + it is possible to work around this restriction using large objects, but no + general file-like access is available to modify a stored object. + + The swiftclient.Connection API ------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A low level API that provides methods for authentication and methods that correspond to the individual REST API calls described in the swift @@ -48,12 +51,12 @@ documentation. For usage details see the client docs: :mod:`swiftclient.client`. --------------------------------- + The swiftclient.SwiftService API --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A higher level API aimed at allowing developers an easy way to perform multiple -operations asynchronously using a configurable thread pool. Docs for each +operations asynchronously using a configurable thread pool. Documentation for each service method call can be found here: :mod:`swiftclient.service`. Configuration @@ -74,7 +77,7 @@ passed to the ``SwiftService`` during initialisation. The options available in this dictionary are described below, along with their defaults: Options -~~~~~~~ +^^^^^^^ ``retries``: ``5`` The number of times that the library should attempt to retry HTTP @@ -190,14 +193,14 @@ for an optional dictionary to override those specified at init time, and the appropriate docstrings show which options modify each method's behaviour. Authentication --------------- +~~~~~~~~~~~~~~ This section covers the various options for authenticating with a swift object store. The combinations of options required for each authentication version are detailed below. Version 1.0 Auth -~~~~~~~~~~~~~~~~ +---------------- ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` @@ -208,8 +211,8 @@ Version 1.0 Auth ``key``: ``environ.get('ST_KEY')`` -Version 2.0 & 3.0 Auth -~~~~~~~~~~~~~~~~~~~~~~ +Version 2.0 and 3.0 Auth +------------------------ ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` @@ -233,7 +236,7 @@ having options from different auth versions can cause unexpected behaviour. authorization fails. Operation Return Values ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ Each operation provided by the service API may raise a ``SwiftError`` or ``ClientException`` for any call that fails completely (or a call which @@ -291,7 +294,7 @@ All the possible ``action`` values are detailed below: ] Stat ----- +~~~~ Stat can be called against an account, a container, or a list of objects to get account stats, container stats or information about the given objects. In @@ -368,7 +371,7 @@ operation was not successful, and will include the keys below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``stat`` to retrieve the headers for a given list of objects in a container using 20 threads. The code creates a @@ -396,7 +399,7 @@ mapping from object name to headers. ) List ----- +~~~~ List can be called against an account or a container to retrieve the containers or objects contained within them. Each call returns an iterator that returns @@ -453,7 +456,7 @@ dictionary as described below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``list`` to list all items in a container that are over 10MiB in size: @@ -482,7 +485,7 @@ container that are over 10MiB in size: output_manager.error(e.value) Post ----- +~~~~ Post can be called against an account, container or list of objects in order to update the metadata attached to the given items. Each element of the object list @@ -494,7 +497,7 @@ an iterator over the results generated for each object post is returned. If the given container or account does not exist, the ``post`` method will raise a ``SwiftError``. -When a string is given for the object name, the options +.. When a string is given for the object name, the options Successful metadata update results are dictionaries as described below: @@ -510,34 +513,31 @@ Successful metadata update results are dictionaries as described below: } .. note:: + Updating user metadata keys will not only add any specified keys, but will also remove user metadata that has previously been set. This means that each time user metadata is updated, the complete set of desired key-value pairs must be specified. -Example -~~~~~~~ -.. Do we want to hide this section until it is complete? -TBD +.. Example +.. ------- -Download --------- +.. TBD -.. Do we want to hide this section until it is complete? +.. Download +.. ~~~~~~~~ -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- -.. Do we want to hide this section until it is complete? - -TBD +.. TBD Upload ------- +~~~~~~ Upload is always called against an account and container and with a list of objects to upload. Each element of the object list may be a plain string @@ -622,7 +622,7 @@ below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``upload`` to upload all files and folders in ``/tmp``, and renaming each object by replacing ``/tmp`` in the @@ -689,31 +689,30 @@ object or directory marker names with ``temporary-objects``: except SwiftError as e: out_manager.error(e.value) -Delete ------- - +.. Delete +.. ~~~~~~ .. Do we want to hide this section until it is complete? -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- .. Do we want to hide this section until it is complete? -TBD +.. TBD -Capabilities ------------- +.. Capabilities +.. ~~~~~~~~~~~~ .. Do we want to hide this section until it is complete? -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- .. Do we want to hide this section until it is complete? -TBD +.. TBD diff --git a/swiftclient/client.py b/swiftclient/client.py index 58fff7b..e5d564d 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -72,6 +72,69 @@ if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) +#: Default behaviour is to redact header values known to contain secrets, +#: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars +#: may be revealed. +#: +#: To disable, set the value of ``redact_sensitive_headers`` to ``False``. +#: +#: When header redaction is enabled, ``reveal_sensitive_prefix`` configures the +#: maximum length of any sensitive header data sent to the logs. If the header +#: is less than twice this length, only ``int(len(value)/2)`` chars will be +#: logged; if it is less than 15 chars long, even less will be logged. +logger_settings = { + 'redact_sensitive_headers': True, + 'reveal_sensitive_prefix': 16 +} +#: A list of sensitive headers to redact in logs. Note that when extending this +#: list, the header names must be added in all lower case. +LOGGER_SENSITIVE_HEADERS = [ + 'x-auth-token', 'x-auth-key', 'x-service-token', 'x-storage-token', + 'x-account-meta-temp-url-key', 'x-account-meta-temp-url-key-2', + 'x-container-meta-temp-url-key', 'x-container-meta-temp-url-key-2', + 'set-cookie' +] + + +def safe_value(name, value): + """ + Only show up to logger_settings['reveal_sensitive_prefix'] characters + from a sensitive header. + + :param name: Header name + :param value: Header value + :return: Safe header value + """ + if name.lower() in LOGGER_SENSITIVE_HEADERS: + prefix_length = logger_settings.get('reveal_sensitive_prefix', 16) + prefix_length = int( + min(prefix_length, (len(value) ** 2) / 32, len(value) / 2) + ) + redacted_value = value[0:prefix_length] + return redacted_value + '...' + return value + + +def scrub_headers(headers): + """ + Redact header values that can contain sensitive information that + should not be logged. + + :param headers: Either a dict or an iterable of two-element tuples + :return: Safe dictionary of headers with sensitive information removed + """ + if isinstance(headers, dict): + headers = headers.items() + headers = [ + (parse_header_string(key), parse_header_string(val)) + for (key, val) in headers + ] + if not logger_settings.get('redact_sensitive_headers', True): + return dict(headers) + if logger_settings.get('reveal_sensitive_prefix', 16) < 0: + logger_settings['reveal_sensitive_prefix'] = 16 + return {key: safe_value(key, val) for (key, val) in headers} + def http_log(args, kwargs, resp, body): if not logger.isEnabledFor(logging.INFO): @@ -87,8 +150,9 @@ def http_log(args, kwargs, resp, body): else: string_parts.append(' %s' % element) if 'headers' in kwargs: - for element in kwargs['headers']: - header = ' -H "%s: %s"' % (element, kwargs['headers'][element]) + headers = scrub_headers(kwargs['headers']) + for element in headers: + header = ' -H "%s: %s"' % (element, headers[element]) string_parts.append(header) # log response as debug if good, or info if error @@ -99,12 +163,14 @@ def http_log(args, kwargs, resp, body): log_method("REQ: %s", "".join(string_parts)) log_method("RESP STATUS: %s %s", resp.status, resp.reason) - log_method("RESP HEADERS: %s", resp.getheaders()) + log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders())) if body: log_method("RESP BODY: %s", body) def parse_header_string(data): + if not isinstance(data, (six.text_type, six.binary_type)): + data = str(data) if six.PY2: if isinstance(data, six.text_type): # Under Python2 requests only returns binary_type, but if we get @@ -386,11 +452,11 @@ def get_auth_1_0(url, user, key, snet, **kwargs): parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, timeout=timeout) method = 'GET' - conn.request(method, parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) + headers = {'X-Auth-User': user, 'X-Auth-Key': key} + conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {}, resp, body) + http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') # There is a side-effect on current Rackspace 1.0 server where a @@ -769,7 +835,7 @@ def get_container(url, token, container, marker=None, limit=None, if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, - service_token, headers=headers) + service_token=service_token, headers=headers) listing = rv[1] while listing: if not delimiter: @@ -778,7 +844,7 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn, service_token, + http_conn, service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) diff --git a/swiftclient/service.py b/swiftclient/service.py index 5fa2870..f253ec8 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1004,7 +1004,7 @@ class SwiftService(object): raise raise SwiftError('Account not found', exc=err) - elif not objects: + elif objects is None: if '/' in container: raise SwiftError('\'/\' in container name', container=container) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4b444a9..15be20a 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -33,6 +33,7 @@ from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version +from swiftclient.client import logger_settings as client_logger_settings from swiftclient.service import SwiftService, SwiftError, \ SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ @@ -1107,6 +1108,7 @@ def parse_args(parser, args, enforce_requires=True): if options.debug: logging.basicConfig(level=logging.DEBUG) logging.getLogger('iso8601').setLevel(logging.WARNING) + client_logger_settings['redact_sensitive_headers'] = False elif options.info: logging.basicConfig(level=logging.INFO) diff --git a/test-requirements.txt b/test-requirements.txt index 7f7e405..044f7c3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,8 @@ hacking>=0.10.0,<0.11 coverage>=3.6 -discover mock>=1.2 oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 -testtools>=0.9.34 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 5f9e271..7a77c07 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -14,7 +14,7 @@ # limitations under the License. import os -import testtools +import unittest import time from io import BytesIO @@ -23,7 +23,7 @@ from six.moves import configparser import swiftclient -class TestFunctional(testtools.TestCase): +class TestFunctional(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestFunctional, self).__init__(*args, **kwargs) diff --git a/tests/unit/test_command_helpers.py b/tests/unit/test_command_helpers.py index d9d7efa..24684ae 100644 --- a/tests/unit/test_command_helpers.py +++ b/tests/unit/test_command_helpers.py @@ -15,13 +15,13 @@ import mock from six import StringIO -import testtools +import unittest from swiftclient import command_helpers as h from swiftclient.multithreading import OutputManager -class TestStatHelpers(testtools.TestCase): +class TestStatHelpers(unittest.TestCase): def setUp(self): super(TestStatHelpers, self).setUp() diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 76758b6..8944d48 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys -import testtools +import unittest import threading import six @@ -25,7 +25,7 @@ from swiftclient import multithreading as mt from .utils import CaptureStream -class ThreadTestCase(testtools.TestCase): +class ThreadTestCase(unittest.TestCase): def setUp(self): super(ThreadTestCase, self).setUp() self.got_items = Queue() @@ -163,7 +163,7 @@ class TestConnectionThreadPoolExecutor(ThreadTestCase): ) -class TestOutputManager(testtools.TestCase): +class TestOutputManager(unittest.TestCase): def test_instantiation(self): output_manager = mt.OutputManager() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 3fbe987..418ee85 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -18,7 +18,7 @@ import mock import os import six import tempfile -import testtools +import unittest import time from concurrent.futures import Future @@ -49,7 +49,7 @@ else: import builtins -class TestSwiftPostObject(testtools.TestCase): +class TestSwiftPostObject(unittest.TestCase): def setUp(self): super(TestSwiftPostObject, self).setUp() @@ -69,7 +69,7 @@ class TestSwiftPostObject(testtools.TestCase): self.assertRaises(SwiftError, self.spo, 1) -class TestSwiftReader(testtools.TestCase): +class TestSwiftReader(unittest.TestCase): def setUp(self): super(TestSwiftReader, self).setUp() @@ -152,25 +152,7 @@ class TestSwiftReader(testtools.TestCase): '97ac82a5b825239e782d0339e2d7b910') -class _TestServiceBase(testtools.TestCase): - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertIsInstance(a, dict, - 'First argument is not a dictionary') - self.assertIsInstance(b, dict, - 'Second argument is not a dictionary') - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) - +class _TestServiceBase(unittest.TestCase): def _get_mock_connection(self, attempts=2): m = Mock(spec=Connection) type(m).attempts = PropertyMock(return_value=attempts) @@ -223,8 +205,8 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) def test_delete_segment_exception(self): mock_q = Queue() @@ -246,8 +228,8 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -268,7 +250,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_exception(self): mock_q = Queue() @@ -294,7 +276,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -321,7 +303,7 @@ class TestServiceDelete(_TestServiceBase): query_string='multipart-manifest=delete', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_dlo_support(self): mock_q = Queue() @@ -352,7 +334,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn, 'test_c', 'test_o', self.opts, mock_q ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) expected = [ mock.call('test_c', 'test_o', query_string=None, response_dict={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), @@ -372,7 +354,7 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() @@ -394,13 +376,13 @@ class TestServiceDelete(_TestServiceBase): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) -class TestSwiftError(testtools.TestCase): +class TestSwiftError(unittest.TestCase): def test_is_exception(self): se = SwiftError(5) @@ -430,7 +412,7 @@ class TestSwiftError(testtools.TestCase): self.assertEqual(str(se), '5 container:con object:obj segment:seg') -class TestServiceUtils(testtools.TestCase): +class TestServiceUtils(unittest.TestCase): def setUp(self): super(TestServiceUtils, self).setUp() @@ -525,7 +507,7 @@ class TestServiceUtils(testtools.TestCase): mock_headers) -class TestSwiftUploadObject(testtools.TestCase): +class TestSwiftUploadObject(unittest.TestCase): def setUp(self): self.suo = swiftclient.service.SwiftUploadObject @@ -614,7 +596,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_account_job( mock_conn, self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -635,7 +617,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_account_job( mock_conn, long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_account_exception(self): @@ -657,7 +639,7 @@ class TestServiceList(_TestServiceBase): mock_conn.get_account.assert_called_once_with( marker='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_container(self): @@ -680,7 +662,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_container_job( mock_conn, 'test_c', self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -702,7 +684,7 @@ class TestServiceList(_TestServiceBase): SwiftService._list_container_job( mock_conn, 'test_c', long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_container_exception(self): @@ -726,7 +708,7 @@ class TestServiceList(_TestServiceBase): mock_conn.get_container.assert_called_once_with( 'test_c', marker='', delimiter='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) @mock.patch('swiftclient.service.get_conn') @@ -805,7 +787,7 @@ class TestServiceList(_TestServiceBase): self.assertEqual(observed_listing, expected_listing) -class TestService(testtools.TestCase): +class TestService(unittest.TestCase): def test_upload_with_bad_segment_size(self): for bad in ('ten', '1234X', '100.3'): @@ -913,7 +895,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', '', @@ -960,7 +942,7 @@ class TestServiceUpload(_TestServiceBase): options={'segment_container': None, 'checksum': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', @@ -1098,7 +1080,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1155,7 +1137,7 @@ class TestServiceUpload(_TestServiceBase): self.assertEqual(mtime, expected_mtime) del r['headers']['x-object-meta-mtime'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1559,7 +1541,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_with_mtime(self): mock_conn = self._get_mock_connection() @@ -1605,7 +1587,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_bad_mtime(self): mock_conn = self._get_mock_connection() @@ -1650,7 +1632,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() @@ -1670,7 +1652,7 @@ class TestServiceDownload(_TestServiceBase): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download(self): service = SwiftService() @@ -1690,6 +1672,22 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') + @mock.patch('swiftclient.service.interruptable_as_completed') + @mock.patch('swiftclient.service.SwiftService._download_container') + @mock.patch('swiftclient.service.SwiftService._download_object_job') + def test_download_with_objects_empty(self, mock_down_obj, + mock_down_cont, mock_as_comp): + fake_future = Future() + fake_future.set_result(1) + mock_as_comp.return_value = [fake_future] + service = SwiftService() + next(service.download('c', [], self.opts), None) + mock_down_obj.assert_not_called() + mock_down_cont.assert_not_called() + + next(service.download('c', options=self.opts), None) + self.assertEqual(True, mock_down_cont.called) + def test_download_with_output_dir(self): service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: @@ -1814,7 +1812,7 @@ class TestServiceDownload(_TestServiceBase): 'header': {}, 'yes_all': False, 'skip_identical': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1876,7 +1874,7 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1959,7 +1957,7 @@ class TestServiceDownload(_TestServiceBase): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', @@ -2025,7 +2023,7 @@ class TestServiceDownload(_TestServiceBase): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_container.mock_calls, [ mock.call('test_c_segments', @@ -2116,7 +2114,7 @@ class TestServiceDownload(_TestServiceBase): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index ddb40f1..e3ab0bf 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -20,9 +20,8 @@ import logging import mock import os import tempfile -import testtools +import unittest import textwrap -from testtools import ExpectedException import six @@ -106,7 +105,7 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): @mock.patch.dict(os.environ, mocked_os_environ) -class TestShell(testtools.TestCase): +class TestShell(unittest.TestCase): def setUp(self): super(TestShell, self).setUp() tmpfile = tempfile.NamedTemporaryFile(delete=False) @@ -1077,7 +1076,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException('bad auth') with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1089,7 +1088,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException('test', http_status=404) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'Account not found\n') @@ -1108,7 +1107,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException('bad auth') with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1126,7 +1125,7 @@ class TestShell(testtools.TestCase): argv = ["", "post", "conta/iner"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('WARNING: / in')) @@ -1166,7 +1165,7 @@ class TestShell(testtools.TestCase): swiftclient.ClientException("bad auth") with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1175,7 +1174,7 @@ class TestShell(testtools.TestCase): argv = ["", "post", "container", "object", "bad_arg"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') @@ -1236,49 +1235,49 @@ class TestShell(testtools.TestCase): _check_expected(mock_swift, 12345) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): # Test invalid states argv = ["", "upload", "-S", "1234X", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K1234", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") def test_negative_upload_segment_size(self): with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40M", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40G", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() -class TestSubcommandHelp(testtools.TestCase): +class TestSubcommandHelp(unittest.TestCase): def test_subcommand_help(self): for command in swiftclient.shell.commands: @@ -1299,7 +1298,7 @@ class TestSubcommandHelp(testtools.TestCase): @mock.patch.dict(os.environ, mocked_os_environ) -class TestDebugAndInfoOptions(testtools.TestCase): +class TestDebugAndInfoOptions(unittest.TestCase): @mock.patch('logging.basicConfig') @mock.patch('swiftclient.service.Connection') def test_option_after_posarg(self, connection, mock_logging): @@ -1330,7 +1329,7 @@ class TestDebugAndInfoOptions(testtools.TestCase): % (mock_logging.call_args_list, argv)) -class TestBase(testtools.TestCase): +class TestBase(unittest.TestCase): """ Provide some common methods to subclasses """ diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index a272465..65830f5 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -18,7 +18,7 @@ import mock import six import socket import string -import testtools +import unittest import warnings import tempfile from hashlib import md5 @@ -34,7 +34,7 @@ import swiftclient.utils import swiftclient -class TestClientException(testtools.TestCase): +class TestClientException(unittest.TestCase): def test_is_exception(self): self.assertTrue(issubclass(c.ClientException, Exception)) @@ -251,12 +251,12 @@ class TestGetAuth(MockHttpTest): self.assertEqual(url, 'storageURL') self.assertEqual(token, 'someauthtoken') - e = self.assertRaises(c.ClientException, c.get_auth, - 'http://www.test.com/invalid_cert', - 'asdf', 'asdf', auth_version='1.0') + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') # TODO: this test is really on validating the mock and not the # the full plumbing into the requests's 'verify' option - self.assertIn('invalid_certificate', str(e)) + self.assertIn('invalid_certificate', str(exc_context.exception)) def test_auth_v1_timeout(self): # this test has some overlap with @@ -583,8 +583,9 @@ class TestHeadAccount(MockHttpTest): def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_account, - 'http://www.tests.com', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -617,17 +618,17 @@ class TestPostAccount(MockHttpTest): def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.post_account, - 'http://www.tests.com', 'asdf', {}) - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_status, 500) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) self.assertRequests([ ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) ]) # TODO: this is a fairly brittle test of the __repr__ on the # ClientException which should probably be in a targeted test new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(e.__str__()[-89:], new_body) + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) class TestGetContainer(MockHttpTest): @@ -741,8 +742,9 @@ class TestHeadContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_container, - 'http://www.test.com', 'asdf', 'container') + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception self.assertRequests([ ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), ]) @@ -765,9 +767,9 @@ class TestPutContainer(MockHttpTest): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.put_container, - 'http://www.test.com', 'token', 'container') - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -987,7 +989,7 @@ class TestPutObject(MockHttpTest): mock_file) text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, - 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'} + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -1024,7 +1026,9 @@ class TestPutObject(MockHttpTest): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, c.put_object, *args) + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -1203,13 +1207,16 @@ class TestPostObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! args = ('http://www.test.com', 'token', 'container', 'obj', - {'X-Object-Meta-Test': 'mymeta'}) + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) c.post_object(*args) self.assertRequests([ ('POST', '/container/obj', '', { 'x-auth-token': 'token', - 'X-Object-Meta-Test': 'mymeta'}), + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), ]) def test_unicode_ok(self): @@ -1221,7 +1228,7 @@ class TestPostObject(MockHttpTest): text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', - 'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', 'X-Object-Meta-Header-not-encoded': text, b'X-Object-Meta-Header-encoded': 'value'} @@ -1244,8 +1251,9 @@ class TestPostObject(MockHttpTest): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'token', 'container', 'obj', {}) - e = self.assertRaises(c.ClientException, c.post_object, *args) - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) self.assertRequests([ ('POST', 'http://www.test.com/container/obj', '', { 'x-auth-token': 'token', @@ -1399,17 +1407,23 @@ class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): url = u'www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) url = u'://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) url = u'blah://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) @@ -1576,8 +1590,9 @@ class TestConnection(MockHttpTest): } c.http_connection = self.fake_http_connection( *code_iter, headers=auth_resp_headers) - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, conn.retries + 1) # test default no-retry @@ -1585,8 +1600,9 @@ class TestConnection(MockHttpTest): 200, 498, headers=auth_resp_headers) conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) def test_resp_read_on_server_error(self): @@ -2184,9 +2200,86 @@ class TestLogging(MockHttpTest): def test_get_error(self): c.http_connection = self.fake_http_connection(404) - e = self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(e.http_status, 404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) + + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) class TestCloseConnection(MockHttpTest): @@ -2382,6 +2475,28 @@ class TestServiceToken(MockHttpTest): actual['full_path']) self.assertEqual(conn.attempts, 1) + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + def test_service_token_head_container(self): with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fe50f55..aae466c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import testtools +import unittest import mock import six import tempfile @@ -22,7 +22,7 @@ from hashlib import md5 from swiftclient import utils as u -class TestConfigTrueValue(testtools.TestCase): +class TestConfigTrueValue(unittest.TestCase): def test_TRUE_VALUES(self): for v in u.TRUE_VALUES: @@ -37,7 +37,7 @@ class TestConfigTrueValue(testtools.TestCase): self.assertIs(u.config_true_value(False), False) -class TestPrtBytes(testtools.TestCase): +class TestPrtBytes(unittest.TestCase): def test_zero_bytes(self): bytes_ = 0 @@ -119,7 +119,7 @@ class TestPrtBytes(testtools.TestCase): self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) -class TestTempURL(testtools.TestCase): +class TestTempURL(unittest.TestCase): def setUp(self): super(TestTempURL, self).setUp() @@ -164,7 +164,7 @@ class TestTempURL(testtools.TestCase): self.method) -class TestReadableToIterable(testtools.TestCase): +class TestReadableToIterable(unittest.TestCase): def test_iter(self): chunk_size = 4 @@ -216,7 +216,7 @@ class TestReadableToIterable(testtools.TestCase): self.assertEqual(actual_md5sum, data.get_md5sum()) -class TestLengthWrapper(testtools.TestCase): +class TestLengthWrapper(unittest.TestCase): def test_stringio(self): contents = six.StringIO(u'a' * 50 + u'b' * 50) @@ -292,7 +292,7 @@ class TestLengthWrapper(testtools.TestCase): self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) -class TestGroupers(testtools.TestCase): +class TestGroupers(unittest.TestCase): def test_n_at_a_time(self): result = list(u.n_at_a_time(range(100), 9)) self.assertEqual([9] * 11 + [1], list(map(len, result))) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f8f5e90..1bfa8da 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -18,7 +18,6 @@ from requests import RequestException from requests.structures import CaseInsensitiveDict from time import sleep import unittest -import testtools import mock import six from six.moves import reload_module @@ -189,7 +188,7 @@ def fake_http_connect(*code_iter, **kwargs): return connect -class MockHttpTest(testtools.TestCase): +class MockHttpTest(unittest.TestCase): def setUp(self): super(MockHttpTest, self).setUp() @@ -6,7 +6,9 @@ skipsdist = True [testenv] usedevelop = True install_command = pip install -U {opts} {packages} -setenv = VIRTUAL_ENV={envdir} +setenv = + LANG=en_US.utf8 + VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt |