diff options
-rw-r--r-- | doc/manpages/swift.1 | 31 | ||||
-rw-r--r-- | doc/source/cli.rst | 33 | ||||
-rw-r--r-- | swiftclient/client.py | 6 | ||||
-rw-r--r-- | swiftclient/service.py | 43 | ||||
-rwxr-xr-x | swiftclient/shell.py | 68 | ||||
-rw-r--r-- | swiftclient/utils.py | 73 | ||||
-rw-r--r-- | tests/unit/test_service.py | 34 | ||||
-rw-r--r-- | tests/unit/test_shell.py | 193 | ||||
-rw-r--r-- | tests/unit/test_swiftclient.py | 94 | ||||
-rw-r--r-- | tests/unit/test_utils.py | 87 |
10 files changed, 518 insertions, 144 deletions
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index bbf5280..1f288d6 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -130,14 +130,33 @@ programs, such as jq. capabilities \-\-json .RE -\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR +\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fItime\fR \fIpath\fR \fIkey\fR .RS 4 Generates a temporary URL allowing unauthenticated access to the Swift object -at the given path, using the given HTTP method, for the given number of -seconds, using the given TempURL key. With the optional \-\-prefix\-based option a -prefix-based URL is generated. If optional \-\-absolute argument is -provided, seconds is instead interpreted as a Unix timestamp at which the URL -should expire. \fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s) +at the given path, using the given HTTP method, for the given time, +using the given TempURL key. + +The time can be specified either as an integer +denoting the amount of seconds the temporary URL is valid, or as an ISO 8601 +timestamp in one of following formats: Complete date: YYYY\-MM\-DD (eg 1997\-07\-16), +complete date plus hours, minutes and seconds: YYYY\-MM\-DDThh:mm:ss +(eg 1997\-07\-16T19:20:30) or complete date plus hours, minutes and seconds with +UTC designator: YYYY\-MM\-DDThh:mm:ssZ (eg 1997\-07\-16T19:20:30Z). Be aware that +if you do not use the latter format, the timestamp is generated using your locale +timezone. If the first format is used, the time part used will equal to 00:00:00. + +With the \-\-prefix\-based option a +prefix-based URL is generated. + +The option \-\-iso8601 provides ISO 8601 UTC timestamps +instead of Unix timestamps inside the generated URL. + +If optional \-\-absolute argument is +provided and the time argument is specified in seconds, the seconds are +interpreted as a Unix timestamp at which the URL +should expire. + +\fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s) /v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key \-\-absolute .RE diff --git a/doc/source/cli.rst b/doc/source/cli.rst index b107448..8d80d1b 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -228,18 +228,39 @@ Capabilities Tempurl ------- - ``tempurl [command-options] [method] [seconds] [path] [key]`` + ``tempurl [command-options] [method] [time] [path] [key]`` Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to - allow for this temporary URL that is usually 'GET' or 'PUT'. ``seconds`` option sets - the amount of time in seconds the temporary URL will be valid for; or, if ``--absolute`` - is passed, the Unix timestamp when the temporary URL will expire. ``path`` option sets - the full path to the Swift object. Example: ``/v1/AUTH_account/c/o``. ``key`` option is + allow for this temporary URL that is usually ``GET` or ``PUT``. ``time`` option sets + the amount of time the temporary URL will be valid for. + ``time`` can be specified as an integer, denoting the number of seconds + from now on until the URL shall be valid; or, if ``--absolute`` + is passed, the Unix timestamp when the temporary URL will expire. + But beyond that, ``time`` can also be specified as an ISO 8601 timestamp + in one of following formats: + + i) Complete date: YYYY-MM-DD (eg 1997-07-16) + + ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (eg 1997-07-16T19:20:30) + + iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (eg 1997-07-16T19:20:30Z) + + Please be aware that if you don't provide the UTC designator (i.e., Z) + the timestamp is generated using your local timezone. If only a date is + specified, the time part used will equal to ``00:00:00``. + + ``path`` option sets the full path to the Swift object. + Example: ``/v1/AUTH_account/c/o``. ``key`` option is the secret temporary URL key set on the Swift cluster. To set a key, run ``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not forget to append the desired objectname at the end of the path portion (and before the - query portion) before sharing the URL. + query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the + URL by using the ``--iso8601`` option. Auth ---- diff --git a/swiftclient/client.py b/swiftclient/client.py index 80b6eda..80bc4a3 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1286,8 +1286,10 @@ def put_object(url, token=None, container=None, name=None, contents=None, if content_type is not None: headers['Content-Type'] = content_type elif 'Content-Type' not in headers: - # python-requests sets application/x-www-form-urlencoded otherwise - headers['Content-Type'] = '' + if StrictVersion(requests.__version__) < StrictVersion('2.4.0'): + # python-requests sets application/x-www-form-urlencoded otherwise + # if using python3. + headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' diff --git a/swiftclient/service.py b/swiftclient/service.py index c5f815f..b9b843e 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -43,7 +43,8 @@ from swiftclient.command_helpers import ( ) from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, - parse_api_response, report_traceback, n_groups, split_request_headers + parse_api_response, report_traceback, n_groups, split_request_headers, + n_at_a_time ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -1997,7 +1998,6 @@ class SwiftService(object): } for d in segment_results ]) - put_headers['x-static-large-object'] = 'true' mr = {} conn.put_object( container, obj, manifest_data, @@ -2165,11 +2165,15 @@ class SwiftService(object): rq = Queue() obj_dels = {} - if self._should_bulk_delete(objects): - for obj_slice in n_groups( - objects, self._options['object_dd_threads']): - self._bulk_delete(container, obj_slice, options, - obj_dels) + bulk_page_size = self._bulk_delete_page_size(objects) + if bulk_page_size > 1: + page_at_a_time = n_at_a_time(objects, bulk_page_size) + for page_slice in page_at_a_time: + for obj_slice in n_groups( + page_slice, + self._options['object_dd_threads']): + self._bulk_delete(container, obj_slice, options, + obj_dels) else: self._per_item_delete(container, objects, options, obj_dels, rq) @@ -2222,23 +2226,36 @@ class SwiftService(object): and not res['success']): cancelled = True - def _should_bulk_delete(self, objects): - if len(objects) < 2 * self._options['object_dd_threads']: + def _bulk_delete_page_size(self, objects): + ''' + Given the iterable 'objects', will return how many items should be + deleted at a time. + + :param objects: An iterable that supports 'len()' + :returns: The bulk delete page size (i.e. the max number of + objects that can be bulk deleted at once, as reported by + the cluster). If bulk delete is disabled, return 1 + ''' + if len(objects) <= 2 * self._options['object_dd_threads']: # Not many objects; may as well delete one-by-one - return False + return 1 try: cap_result = self.capabilities() if not cap_result['success']: # This shouldn't actually happen, but just in case we start # being more nuanced about our capabilities result... - return False + return 1 except ClientException: # Old swift, presumably; assume no bulk middleware - return False + return 1 swift_info = cap_result['capabilities'] - return 'bulk_delete' in swift_info + if 'bulk_delete' in swift_info: + return swift_info['bulk_delete'].get( + 'max_deletes_per_request', 10000) + else: + return 1 def _per_item_delete(self, container, objects, options, rdict, rq): for obj in objects: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4a8a356..841ed6e 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1224,8 +1224,8 @@ def st_auth(parser, args, thread_manager): print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) -st_tempurl_options = '''[--absolute] [--prefix-based] - <method> <seconds> <path> <key>''' +st_tempurl_options = '''[--absolute] [--prefix-based] [--iso8601] + <method> <time> <path> <key>''' st_tempurl_help = ''' @@ -1234,9 +1234,35 @@ Generates a temporary URL for a Swift object. Positional arguments: <method> An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - <seconds> The amount of time in seconds the temporary URL will be - valid for; or, if --absolute is passed, the Unix - timestamp when the temporary URL will expire. + <time> The amount of time the temporary URL will be + valid. The time can be specified in two ways: + an integer representing the time in seconds or an + ISO 8601 timestamp in a specific format. + If --absolute is passed and time + is an integer, the seconds are intepreted as the Unix + timestamp when the temporary URL will expire. The ISO + 8601 timestamp can be specified in one of following + formats: + + i) Complete date: YYYY-MM-DD (eg 1997-07-16) + + ii) Complete date plus hours, minutes and seconds: + + YYYY-MM-DDThh:mm:ss + + (eg 1997-07-16T19:20:30) + + iii) Complete date plus hours, minutes and seconds with + UTC designator: + + YYYY-MM-DDThh:mm:ssZ + + (eg 1997-07-16T19:20:30Z) + + Please be aware that if you don't provide the UTC + designator (i.e., Z) the timestamp is generated using + your local timezone. If only a date is specified, + the time part used will equal to 00:00:00. <path> The full path or storage URL to the Swift object. Example: /v1/AUTH_account/c/o or: http://saio:8080/v1/AUTH_account/c/o @@ -1245,10 +1271,14 @@ Positional arguments: "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' Optional arguments: - --absolute Interpret the <seconds> positional argument as a Unix + --absolute Interpret the <time> positional argument as a Unix timestamp rather than a number of seconds in the - future. - --prefix-based If present, a prefix-based tempURL will be generated. + future. If an ISO 8601 timestamp is passed for <time>, + this argument is ignored. + --prefix-based If present, a prefix-based temporary URL will be + generated. + --iso8601 If present, the generated temporary URL will contain an + ISO 8601 UTC timestamp instead of a Unix timestamp. '''.strip('\n') @@ -1256,14 +1286,21 @@ def st_tempurl(parser, args, thread_manager): parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, - help=("If present, seconds argument will be interpreted as a Unix " - "timestamp representing when the tempURL should expire, rather " - "than an offset from the current time"), + help=("If present, and time argument is an integer, " + "time argument will be interpreted as a Unix " + "timestamp representing when the temporary URL should expire, " + "rather than an offset from the current time."), ) parser.add_argument( '--prefix-based', action='store_true', default=False, - help=("If present, a prefix-based tempURL will be generated."), + help=("If present, a prefix-based temporary URL will be generated."), + ) + parser.add_argument( + '--iso8601', action='store_true', + default=False, + help=("If present, the temporary URL will contain an ISO 8601 UTC " + "timestamp instead of a Unix timestamp."), ) (options, args) = parse_args(parser, args) @@ -1272,7 +1309,7 @@ def st_tempurl(parser, args, thread_manager): thread_manager.error('Usage: %s tempurl %s\n%s', BASENAME, st_tempurl_options, st_tempurl_help) return - method, seconds, path, key = args[:4] + method, timestamp, path, key = args[:4] parsed = urlparse(path) @@ -1281,9 +1318,10 @@ def st_tempurl(parser, args, thread_manager): 'tempurl specified, possibly an error' % method.upper()) try: - path = generate_temp_url(parsed.path, seconds, key, method, + path = generate_temp_url(parsed.path, timestamp, key, method, absolute=options['absolute_expiry'], - prefix=options['prefix_based'],) + iso8601=options['iso8601'], + prefix=options['prefix_based']) except ValueError as err: thread_manager.error(err) return diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 08cd7b2..8afcde9 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" +from calendar import timegm import collections import gzip import hashlib @@ -25,6 +26,10 @@ import traceback TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e' +EXPIRES_ISO8601_FORMAT = '%Y-%m-%dT%H:%M:%SZ' +SHORT_EXPIRES_ISO8601_FORMAT = '%Y-%m-%d' +TIME_ERRMSG = ('time must either be a whole number or in specific ' + 'ISO 8601 format.') def config_true_value(value): @@ -64,39 +69,65 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False): + prefix=False, iso8601=False): """Generates a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object or prefix if a prefix-based temporary URL should be generated. Example: /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. - :param seconds: If absolute is False then this specifies the amount of time - in seconds for which the temporary URL will be valid. If absolute is - True then this specifies an absolute time at which the temporary URL - will expire. + :param seconds: time in seconds or ISO 8601 timestamp. + If absolute is False and this is the string representation of an + integer, then this specifies the amount of time in seconds for which + the temporary URL will be valid. + If absolute is True then this specifies an absolute time at which the + temporary URL will expire. :param key: The secret temporary URL key set on the Swift cluster. To set a key, run 'swift post -m "Temp-URL-Key: <substitute tempurl key here>"' :param method: A HTTP method, typically either GET or PUT, to allow for this temporary URL. - :param absolute: if True then the seconds parameter is interpreted as an - absolute Unix time, otherwise seconds is interpreted as a relative time - offset from current time. + :param absolute: if True then the seconds parameter is interpreted as a + Unix timestamp, if seconds represents an integer. :param prefix: if True then a prefix-based temporary URL will be generated. - :raises ValueError: if seconds is not a whole number or path is not to - an object. + :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp + instead of a UNIX timestamp will be created. + :raises ValueError: if timestamp or path is not in valid format. :return: the path portion of a temporary URL """ try: - seconds = float(seconds) - if not seconds.is_integer(): - raise ValueError() - seconds = int(seconds) - if seconds < 0: - raise ValueError() + try: + timestamp = float(seconds) + except ValueError: + formats = ( + EXPIRES_ISO8601_FORMAT, + EXPIRES_ISO8601_FORMAT[:-1], + SHORT_EXPIRES_ISO8601_FORMAT) + for f in formats: + try: + t = time.strptime(seconds, f) + except ValueError: + t = None + else: + if f == EXPIRES_ISO8601_FORMAT: + timestamp = timegm(t) + else: + # Use local time if UTC designator is missing. + timestamp = int(time.mktime(t)) + + absolute = True + break + + if t is None: + raise ValueError() + else: + if not timestamp.is_integer(): + raise ValueError() + timestamp = int(timestamp) + if timestamp < 0: + raise ValueError() except ValueError: - raise ValueError('seconds must be a whole number') + raise ValueError(TIME_ERRMSG) if isinstance(path, six.binary_type): try: @@ -121,9 +152,9 @@ def generate_temp_url(path, seconds, key, method, absolute=False, 'possibly an error', method.upper()) if not absolute: - expiration = int(time.time() + seconds) + expiration = int(time.time() + timestamp) else: - expiration = seconds + expiration = timestamp hmac_body = u'\n'.join([method.upper(), str(expiration), ('prefix:' if prefix else '') + path_for_body]) @@ -132,6 +163,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False, key = key.encode('utf-8') sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() + if iso8601: + expiration = time.strftime( + EXPIRES_ISO8601_FORMAT, time.gmtime(expiration)) + temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) if prefix: diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index af32d84..b759e6b 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -492,6 +492,40 @@ class TestServiceDelete(_TestServiceBase): self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) + @mock.patch.object(swiftclient.service.SwiftService, 'capabilities', + lambda *a: {'action': 'capabilities', + 'timestamp': time.time(), + 'success': True, + 'capabilities': { + 'bulk_delete': + {'max_deletes_per_request': 10}} + }) + def test_bulk_delete_page_size(self): + # make a list of 100 objects + obj_list = ['x%02d' % i for i in range(100)] + errors = [] + + # _bulk_delete_page_size uses 2x the number of threads to determine + # if if there are "many" object to delete or not + + # format is: [(thread_count, expected result), ...] + obj_threads_exp = [ + (10, 10), # something small + (49, 10), # just under the bounds + (50, 1), # cutover point + (51, 1), # just over bounds + (100, 1), # something big + ] + for thread_count, exp in obj_threads_exp: + s = SwiftService(options={'object_dd_threads': thread_count}) + res = s._bulk_delete_page_size(obj_list) + if res != exp: + msg = 'failed for thread_count %d: got %r expected %r' % \ + (thread_count, res, exp) + errors.append(msg) + if errors: + self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) + class TestSwiftError(unittest.TestCase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index f7e9c1a..db96df7 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -23,6 +23,7 @@ import os import tempfile import unittest import textwrap +from time import localtime, mktime, strftime, strptime from requests.packages.urllib3.exceptions import InsecureRequestWarning import six @@ -36,7 +37,9 @@ from os.path import basename, dirname from .utils import ( CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, FakeKeystone, StubResponse, MockHttpTest) -from swiftclient.utils import EMPTY_ETAG +from swiftclient.utils import ( + EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, + SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) if six.PY2: @@ -714,7 +717,6 @@ class TestShell(unittest.TestCase): mock.ANY, headers={ 'x-object-meta-mtime': mock.ANY, - 'x-static-large-object': 'true' }, query_string='multipart-manifest=put', response_dict={}) @@ -903,8 +905,8 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 0) @mock.patch('swiftclient.service.Connection') def test_delete_bad_threads(self, mock_connection): mock_connection.return_value.get_container.return_value = (None, []) @@ -934,8 +936,8 @@ class TestShell(unittest.TestCase): check_good(["--object-threads", "1"]) check_good(["--container-threads", "1"]) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_delete_account(self, connection): connection.return_value.get_account.side_effect = [ @@ -971,8 +973,8 @@ class TestShell(unittest.TestCase): mock.call('container2', response_dict={}, headers={}), mock.call('empty_container', response_dict={}, headers={})]) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: True) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) @mock.patch('swiftclient.service.Connection') def test_delete_bulk_account(self, connection): connection.return_value.get_account.side_effect = [ @@ -1085,8 +1087,80 @@ class TestShell(unittest.TestCase): self.assertEqual(connection.return_value.get_capabilities.mock_calls, [mock.call(None)]) # only one /info request - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities_and_pages(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 2, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + # check that each bulk call was only called with 2 objects + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_delete_container(self, connection): connection.return_value.get_container.side_effect = [ @@ -1103,8 +1177,8 @@ class TestShell(unittest.TestCase): 'container', 'object', query_string=None, response_dict={}, headers={}) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_delete_container_headers(self, connection): connection.return_value.get_container.side_effect = [ @@ -1122,8 +1196,8 @@ class TestShell(unittest.TestCase): 'container', 'object', query_string=None, response_dict={}, headers={'Skip-Middleware': 'Test'}) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: True) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) @mock.patch('swiftclient.service.Connection') def test_delete_bulk_container(self, connection): connection.return_value.get_container.side_effect = [ @@ -1176,8 +1250,8 @@ class TestShell(unittest.TestCase): self.assertTrue(out.out.find( 't\u00e9st_c [after 2 attempts]') >= 0, out) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_delete_per_object(self, connection): argv = ["", "delete", "container", "object"] @@ -1188,8 +1262,8 @@ class TestShell(unittest.TestCase): 'container', 'object', query_string=None, response_dict={}, headers={}) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: True) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) @mock.patch('swiftclient.service.Connection') def test_delete_bulk_object(self, connection): argv = ["", "delete", "container", "object"] @@ -1550,7 +1624,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - prefix=False) + iso8601=False, prefix=False) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -1559,7 +1633,28 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - prefix=True) + iso8601=False, prefix=True) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_in(self, temp_url): + dates = ('1970-01-01T00:01:00Z', '1970-01-01T00:01:00', + '1970-01-01') + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/AUTH_account/c/", + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_iso8601_out(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--iso8601"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + iso8601=True, prefix=False) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1568,7 +1663,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - prefix=False) + iso8601=False, prefix=False) def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1595,6 +1690,42 @@ class TestShell(unittest.TestCase): "&temp_url_prefix=\n" % sig) self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix", '--iso8601'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expires = '1970-01-01T00:01:00Z' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=%s" + "&temp_url_prefix=\n" % (sig, expires)) + self.assertEqual(expected, output.out) + + dates = ("1970-01-01T00:01:00Z", + strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) + for d in dates: + argv = ["", "tempurl", "GET", d, "/v1/a/c/o", + "secret_key"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + ts = str(int( + mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) + + argv = ["", "tempurl", "GET", ts, "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + expected = output.out + + argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -1614,7 +1745,17 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for path %r' % - (expected, output.err, bad_path)) + (expected, output.err, '/v1/a/c')) + + expected = TIME_ERRMSG + '\n' + for bad_time in ('not_an_int', '-1', '2015-05', '2015-05-01T01:00'): + argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for time %r' % + (expected, output.err, bad_time)) @mock.patch('swiftclient.service.Connection') def test_capabilities(self, connection): @@ -2714,8 +2855,8 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): return status return on_request - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_upload_bad_threads(self, mock_connection): mock_connection.return_value.put_object.return_value = EMPTY_ETAG @@ -2897,8 +3038,8 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.assertIn(expected_err, out.err) self.assertEqual('', out) - @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', - lambda *a: False) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') def test_download_bad_threads(self, mock_connection): mock_connection.return_value.get_object.return_value = [{}, ''] diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index d4a704e..5384ac7 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -14,6 +14,7 @@ # limitations under the License. import gzip +import json import logging import mock import six @@ -83,7 +84,7 @@ class TestClientException(unittest.TestCase): class MockHttpResponse(object): - def __init__(self, status=0, headers=None, verify=False, need_items=None): + def __init__(self, status=0, headers=None, verify=False): self.status = status self.status_code = status self.reason = "OK" @@ -92,7 +93,6 @@ class MockHttpResponse(object): self.verify = verify self.md5sum = md5() self.headers = {'etag': '"%s"' % EMPTY_ETAG} - self.need_items = need_items if headers: self.headers.update(headers) self.closed = False @@ -119,9 +119,7 @@ class MockHttpResponse(object): return self.headers.get(name, default) def getheaders(self): - if self.need_items: - return dict(self.headers).items() - return dict(self.headers) + return dict(self.headers).items() def fake_response(self): return self @@ -1165,6 +1163,7 @@ class TestHeadObject(MockHttpTest): class TestPutObject(MockHttpTest): + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) @@ -1222,6 +1221,7 @@ class TestPutObject(MockHttpTest): self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, UserWarning)) + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_server_error(self): body = 'c' * 60 headers = {'foo': 'bar'} @@ -1236,7 +1236,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', 'content-type': ''}), + 'x-auth-token': 'asdf', + 'content-type': ''}), ]) def test_query_string(self): @@ -1377,7 +1378,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - def test_no_content_type(self): + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_no_content_type_old_requests(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response @@ -1387,6 +1389,17 @@ class TestPutObject(MockHttpTest): request_header = resp.requests_params['headers'] self.assertEqual(request_header['content-type'], b'') + @mock.patch('swiftclient.requests.__version__', '2.4.0') + def test_no_content_type_new_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertNotIn('content-type', request_header) + def test_content_type_in_headers(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) @@ -2319,7 +2332,7 @@ class TestConnection(MockHttpTest): return 'header' def getheaders(self): - return {"key1": "value1", "key2": "value2"} + return [('key1', 'value1'), ('key2', 'value2')] def read(self, *args, **kwargs): return '' @@ -2569,6 +2582,33 @@ class TestLogging(MockHttpTest): c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') self.assertEqual(exc_context.exception.http_status, 404) + def test_content_encoding_gzip_body_is_logged_decoded(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + data = {"test": u"\u2603"} + decoded_body = json.dumps(data).encode('utf-8') + gz.write(decoded_body) + gz.close() + # stub a gzip encoded body + body = buf.getvalue() + headers = {'content-encoding': 'gzip'} + # ... and make a content-encoding gzip error response + stub_response = StubResponse(500, body, headers) + with mock.patch('swiftclient.client.logger.info') as mock_log: + # ... if the client gets such a response + c.http_connection = self.fake_http_connection(stub_response) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 500) + # it will log the decoded body + self.assertEqual([ + mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + '-X GET -H "X-Auth-Token: ..."'), + mock.call('RESP STATUS: %s %s', 500, 'Fake'), + mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), + mock.call('RESP BODY: %s', decoded_body) + ], mock_log.mock_calls) + def test_redact_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' @@ -2611,44 +2651,6 @@ class TestLogging(MockHttpTest): self.assertNotIn(unicode_token_value, output) self.assertNotIn(set_cookie_value, output) - def test_logging_body(self): - with mock.patch('swiftclient.client.logger.debug') as mock_log: - token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' - token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') - unicode_token_encoded = unicode_token_value.encode('utf8') - set_cookie_value = 'X-Auth-Token=%s' % token_value - set_cookie_encoded = set_cookie_value.encode('utf8') - buf = six.BytesIO() - gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) - gz.close() - c.http_log( - ['GET'], - {'headers': { - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded - }}, - MockHttpResponse( - status=200, - headers={ - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded, - 'content-encoding': 'gzip', - 'Etag': b'mock_etag', - 'Set-Cookie': set_cookie_encoded - }, - need_items=True, - ), - buf.getvalue(), - ) - self.assertEqual( - mock.call( - 'RESP BODY: %s', u'{"test": "\u2603"}'.encode('utf8')), - mock_log.mock_calls[3]) - def test_show_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c5961e8..adead00 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -18,6 +18,7 @@ import unittest import mock import six import tempfile +from time import gmtime, localtime, mktime, strftime, strptime from hashlib import md5, sha1 from swiftclient import utils as u @@ -151,6 +152,67 @@ class TestTempURL(unittest.TestCase): self.assertIsInstance(url, type(self.url)) @mock.patch('hmac.HMAC') + def test_generate_temp_url_iso8601_argument(self, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method) + self.assertEqual(url, self.expected_url) + + # Don't care about absolute arg. + url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', + self.key, self.method, absolute=True) + self.assertEqual(url, self.expected_url) + + lt = localtime() + expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) + lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) + + if not isinstance(self.expected_url, six.string_types): + expected_url = self.expected_url.replace( + b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) + else: + expected_url = self.expected_url.replace( + '1400003600', str(int(mktime(lt)))) + url = u.generate_temp_url(self.url, expires, + self.key, self.method) + self.assertEqual(url, expected_url) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, + iso8601=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) + if not isinstance(self.url, six.string_types): + self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) + else: + self.assertTrue(url.endswith(expires)) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') @mock.patch('time.time', return_value=1400000000) def test_generate_temp_url_prefix(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' @@ -198,31 +260,34 @@ class TestTempURL(unittest.TestCase): absolute=True) self.assertEqual(url, expected_url) - def test_generate_temp_url_bad_seconds(self): + def test_generate_temp_url_bad_time(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 'not_an_int', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'seconds must be a whole number') + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, -1, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'seconds must be a whole number') + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, 1.1, self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'seconds must be a whole number') + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '-1', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'seconds must be a whole number') + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(self.url, '1.1', self.key, self.method) - self.assertEqual(exc_manager.exception.args[0], - 'seconds must be a whole number') + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url(self.url, '2015-05', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) + + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url( + self.url, '2015-05-01T01:00', self.key, self.method) + self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG) def test_generate_temp_url_bad_path(self): with self.assertRaises(ValueError) as exc_manager: |