summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/manpages/swift.131
-rw-r--r--doc/source/cli.rst33
-rw-r--r--swiftclient/client.py6
-rw-r--r--swiftclient/service.py43
-rwxr-xr-xswiftclient/shell.py68
-rw-r--r--swiftclient/utils.py73
-rw-r--r--tests/unit/test_service.py34
-rw-r--r--tests/unit/test_shell.py193
-rw-r--r--tests/unit/test_swiftclient.py94
-rw-r--r--tests/unit/test_utils.py87
10 files changed, 518 insertions, 144 deletions
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1
index bbf5280..1f288d6 100644
--- a/doc/manpages/swift.1
+++ b/doc/manpages/swift.1
@@ -130,14 +130,33 @@ programs, such as jq.
capabilities \-\-json
.RE
-\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR
+\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fItime\fR \fIpath\fR \fIkey\fR
.RS 4
Generates a temporary URL allowing unauthenticated access to the Swift object
-at the given path, using the given HTTP method, for the given number of
-seconds, using the given TempURL key. With the optional \-\-prefix\-based option a
-prefix-based URL is generated. If optional \-\-absolute argument is
-provided, seconds is instead interpreted as a Unix timestamp at which the URL
-should expire. \fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s)
+at the given path, using the given HTTP method, for the given time,
+using the given TempURL key.
+
+The time can be specified either as an integer
+denoting the amount of seconds the temporary URL is valid, or as an ISO 8601
+timestamp in one of following formats: Complete date: YYYY\-MM\-DD (eg 1997\-07\-16),
+complete date plus hours, minutes and seconds: YYYY\-MM\-DDThh:mm:ss
+(eg 1997\-07\-16T19:20:30) or complete date plus hours, minutes and seconds with
+UTC designator: YYYY\-MM\-DDThh:mm:ssZ (eg 1997\-07\-16T19:20:30Z). Be aware that
+if you do not use the latter format, the timestamp is generated using your locale
+timezone. If the first format is used, the time part used will equal to 00:00:00.
+
+With the \-\-prefix\-based option a
+prefix-based URL is generated.
+
+The option \-\-iso8601 provides ISO 8601 UTC timestamps
+instead of Unix timestamps inside the generated URL.
+
+If optional \-\-absolute argument is
+provided and the time argument is specified in seconds, the seconds are
+interpreted as a Unix timestamp at which the URL
+should expire.
+
+\fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s)
/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key \-\-absolute
.RE
diff --git a/doc/source/cli.rst b/doc/source/cli.rst
index b107448..8d80d1b 100644
--- a/doc/source/cli.rst
+++ b/doc/source/cli.rst
@@ -228,18 +228,39 @@ Capabilities
Tempurl
-------
- ``tempurl [command-options] [method] [seconds] [path] [key]``
+ ``tempurl [command-options] [method] [time] [path] [key]``
Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to
- allow for this temporary URL that is usually 'GET' or 'PUT'. ``seconds`` option sets
- the amount of time in seconds the temporary URL will be valid for; or, if ``--absolute``
- is passed, the Unix timestamp when the temporary URL will expire. ``path`` option sets
- the full path to the Swift object. Example: ``/v1/AUTH_account/c/o``. ``key`` option is
+ allow for this temporary URL that is usually ``GET` or ``PUT``. ``time`` option sets
+ the amount of time the temporary URL will be valid for.
+ ``time`` can be specified as an integer, denoting the number of seconds
+ from now on until the URL shall be valid; or, if ``--absolute``
+ is passed, the Unix timestamp when the temporary URL will expire.
+ But beyond that, ``time`` can also be specified as an ISO 8601 timestamp
+ in one of following formats:
+
+ i) Complete date: YYYY-MM-DD (eg 1997-07-16)
+
+ ii) Complete date plus hours, minutes and seconds:
+ YYYY-MM-DDThh:mm:ss
+ (eg 1997-07-16T19:20:30)
+
+ iii) Complete date plus hours, minutes and seconds with UTC designator:
+ YYYY-MM-DDThh:mm:ssZ
+ (eg 1997-07-16T19:20:30Z)
+
+ Please be aware that if you don't provide the UTC designator (i.e., Z)
+ the timestamp is generated using your local timezone. If only a date is
+ specified, the time part used will equal to ``00:00:00``.
+
+ ``path`` option sets the full path to the Swift object.
+ Example: ``/v1/AUTH_account/c/o``. ``key`` option is
the secret temporary URL key set on the Swift cluster. To set a key, run
``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary
URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not
forget to append the desired objectname at the end of the path portion (and before the
- query portion) before sharing the URL.
+ query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the
+ URL by using the ``--iso8601`` option.
Auth
----
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 80b6eda..80bc4a3 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -1286,8 +1286,10 @@ def put_object(url, token=None, container=None, name=None, contents=None,
if content_type is not None:
headers['Content-Type'] = content_type
elif 'Content-Type' not in headers:
- # python-requests sets application/x-www-form-urlencoded otherwise
- headers['Content-Type'] = ''
+ if StrictVersion(requests.__version__) < StrictVersion('2.4.0'):
+ # python-requests sets application/x-www-form-urlencoded otherwise
+ # if using python3.
+ headers['Content-Type'] = ''
if not contents:
headers['Content-Length'] = '0'
diff --git a/swiftclient/service.py b/swiftclient/service.py
index c5f815f..b9b843e 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -43,7 +43,8 @@ from swiftclient.command_helpers import (
)
from swiftclient.utils import (
config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG,
- parse_api_response, report_traceback, n_groups, split_request_headers
+ parse_api_response, report_traceback, n_groups, split_request_headers,
+ n_at_a_time
)
from swiftclient.exceptions import ClientException
from swiftclient.multithreading import MultiThreadingManager
@@ -1997,7 +1998,6 @@ class SwiftService(object):
} for d in segment_results
])
- put_headers['x-static-large-object'] = 'true'
mr = {}
conn.put_object(
container, obj, manifest_data,
@@ -2165,11 +2165,15 @@ class SwiftService(object):
rq = Queue()
obj_dels = {}
- if self._should_bulk_delete(objects):
- for obj_slice in n_groups(
- objects, self._options['object_dd_threads']):
- self._bulk_delete(container, obj_slice, options,
- obj_dels)
+ bulk_page_size = self._bulk_delete_page_size(objects)
+ if bulk_page_size > 1:
+ page_at_a_time = n_at_a_time(objects, bulk_page_size)
+ for page_slice in page_at_a_time:
+ for obj_slice in n_groups(
+ page_slice,
+ self._options['object_dd_threads']):
+ self._bulk_delete(container, obj_slice, options,
+ obj_dels)
else:
self._per_item_delete(container, objects, options,
obj_dels, rq)
@@ -2222,23 +2226,36 @@ class SwiftService(object):
and not res['success']):
cancelled = True
- def _should_bulk_delete(self, objects):
- if len(objects) < 2 * self._options['object_dd_threads']:
+ def _bulk_delete_page_size(self, objects):
+ '''
+ Given the iterable 'objects', will return how many items should be
+ deleted at a time.
+
+ :param objects: An iterable that supports 'len()'
+ :returns: The bulk delete page size (i.e. the max number of
+ objects that can be bulk deleted at once, as reported by
+ the cluster). If bulk delete is disabled, return 1
+ '''
+ if len(objects) <= 2 * self._options['object_dd_threads']:
# Not many objects; may as well delete one-by-one
- return False
+ return 1
try:
cap_result = self.capabilities()
if not cap_result['success']:
# This shouldn't actually happen, but just in case we start
# being more nuanced about our capabilities result...
- return False
+ return 1
except ClientException:
# Old swift, presumably; assume no bulk middleware
- return False
+ return 1
swift_info = cap_result['capabilities']
- return 'bulk_delete' in swift_info
+ if 'bulk_delete' in swift_info:
+ return swift_info['bulk_delete'].get(
+ 'max_deletes_per_request', 10000)
+ else:
+ return 1
def _per_item_delete(self, container, objects, options, rdict, rq):
for obj in objects:
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 4a8a356..841ed6e 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -1224,8 +1224,8 @@ def st_auth(parser, args, thread_manager):
print('export OS_AUTH_TOKEN=%s' % sh_quote(token))
-st_tempurl_options = '''[--absolute] [--prefix-based]
- <method> <seconds> <path> <key>'''
+st_tempurl_options = '''[--absolute] [--prefix-based] [--iso8601]
+ <method> <time> <path> <key>'''
st_tempurl_help = '''
@@ -1234,9 +1234,35 @@ Generates a temporary URL for a Swift object.
Positional arguments:
<method> An HTTP method to allow for this temporary URL.
Usually 'GET' or 'PUT'.
- <seconds> The amount of time in seconds the temporary URL will be
- valid for; or, if --absolute is passed, the Unix
- timestamp when the temporary URL will expire.
+ <time> The amount of time the temporary URL will be
+ valid. The time can be specified in two ways:
+ an integer representing the time in seconds or an
+ ISO 8601 timestamp in a specific format.
+ If --absolute is passed and time
+ is an integer, the seconds are intepreted as the Unix
+ timestamp when the temporary URL will expire. The ISO
+ 8601 timestamp can be specified in one of following
+ formats:
+
+ i) Complete date: YYYY-MM-DD (eg 1997-07-16)
+
+ ii) Complete date plus hours, minutes and seconds:
+
+ YYYY-MM-DDThh:mm:ss
+
+ (eg 1997-07-16T19:20:30)
+
+ iii) Complete date plus hours, minutes and seconds with
+ UTC designator:
+
+ YYYY-MM-DDThh:mm:ssZ
+
+ (eg 1997-07-16T19:20:30Z)
+
+ Please be aware that if you don't provide the UTC
+ designator (i.e., Z) the timestamp is generated using
+ your local timezone. If only a date is specified,
+ the time part used will equal to 00:00:00.
<path> The full path or storage URL to the Swift object.
Example: /v1/AUTH_account/c/o
or: http://saio:8080/v1/AUTH_account/c/o
@@ -1245,10 +1271,14 @@ Positional arguments:
"Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\'
Optional arguments:
- --absolute Interpret the <seconds> positional argument as a Unix
+ --absolute Interpret the <time> positional argument as a Unix
timestamp rather than a number of seconds in the
- future.
- --prefix-based If present, a prefix-based tempURL will be generated.
+ future. If an ISO 8601 timestamp is passed for <time>,
+ this argument is ignored.
+ --prefix-based If present, a prefix-based temporary URL will be
+ generated.
+ --iso8601 If present, the generated temporary URL will contain an
+ ISO 8601 UTC timestamp instead of a Unix timestamp.
'''.strip('\n')
@@ -1256,14 +1286,21 @@ def st_tempurl(parser, args, thread_manager):
parser.add_argument(
'--absolute', action='store_true',
dest='absolute_expiry', default=False,
- help=("If present, seconds argument will be interpreted as a Unix "
- "timestamp representing when the tempURL should expire, rather "
- "than an offset from the current time"),
+ help=("If present, and time argument is an integer, "
+ "time argument will be interpreted as a Unix "
+ "timestamp representing when the temporary URL should expire, "
+ "rather than an offset from the current time."),
)
parser.add_argument(
'--prefix-based', action='store_true',
default=False,
- help=("If present, a prefix-based tempURL will be generated."),
+ help=("If present, a prefix-based temporary URL will be generated."),
+ )
+ parser.add_argument(
+ '--iso8601', action='store_true',
+ default=False,
+ help=("If present, the temporary URL will contain an ISO 8601 UTC "
+ "timestamp instead of a Unix timestamp."),
)
(options, args) = parse_args(parser, args)
@@ -1272,7 +1309,7 @@ def st_tempurl(parser, args, thread_manager):
thread_manager.error('Usage: %s tempurl %s\n%s', BASENAME,
st_tempurl_options, st_tempurl_help)
return
- method, seconds, path, key = args[:4]
+ method, timestamp, path, key = args[:4]
parsed = urlparse(path)
@@ -1281,9 +1318,10 @@ def st_tempurl(parser, args, thread_manager):
'tempurl specified, possibly an error' %
method.upper())
try:
- path = generate_temp_url(parsed.path, seconds, key, method,
+ path = generate_temp_url(parsed.path, timestamp, key, method,
absolute=options['absolute_expiry'],
- prefix=options['prefix_based'],)
+ iso8601=options['iso8601'],
+ prefix=options['prefix_based'])
except ValueError as err:
thread_manager.error(err)
return
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index 08cd7b2..8afcde9 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Miscellaneous utility functions for use with Swift."""
+from calendar import timegm
import collections
import gzip
import hashlib
@@ -25,6 +26,10 @@ import traceback
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e'
+EXPIRES_ISO8601_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+SHORT_EXPIRES_ISO8601_FORMAT = '%Y-%m-%d'
+TIME_ERRMSG = ('time must either be a whole number or in specific '
+ 'ISO 8601 format.')
def config_true_value(value):
@@ -64,39 +69,65 @@ def prt_bytes(num_bytes, human_flag):
def generate_temp_url(path, seconds, key, method, absolute=False,
- prefix=False):
+ prefix=False, iso8601=False):
"""Generates a temporary URL that gives unauthenticated access to the
Swift object.
:param path: The full path to the Swift object or prefix if
a prefix-based temporary URL should be generated. Example:
/v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix.
- :param seconds: If absolute is False then this specifies the amount of time
- in seconds for which the temporary URL will be valid. If absolute is
- True then this specifies an absolute time at which the temporary URL
- will expire.
+ :param seconds: time in seconds or ISO 8601 timestamp.
+ If absolute is False and this is the string representation of an
+ integer, then this specifies the amount of time in seconds for which
+ the temporary URL will be valid.
+ If absolute is True then this specifies an absolute time at which the
+ temporary URL will expire.
:param key: The secret temporary URL key set on the Swift
cluster. To set a key, run 'swift post -m
"Temp-URL-Key: <substitute tempurl key here>"'
:param method: A HTTP method, typically either GET or PUT, to allow
for this temporary URL.
- :param absolute: if True then the seconds parameter is interpreted as an
- absolute Unix time, otherwise seconds is interpreted as a relative time
- offset from current time.
+ :param absolute: if True then the seconds parameter is interpreted as a
+ Unix timestamp, if seconds represents an integer.
:param prefix: if True then a prefix-based temporary URL will be generated.
- :raises ValueError: if seconds is not a whole number or path is not to
- an object.
+ :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp
+ instead of a UNIX timestamp will be created.
+ :raises ValueError: if timestamp or path is not in valid format.
:return: the path portion of a temporary URL
"""
try:
- seconds = float(seconds)
- if not seconds.is_integer():
- raise ValueError()
- seconds = int(seconds)
- if seconds < 0:
- raise ValueError()
+ try:
+ timestamp = float(seconds)
+ except ValueError:
+ formats = (
+ EXPIRES_ISO8601_FORMAT,
+ EXPIRES_ISO8601_FORMAT[:-1],
+ SHORT_EXPIRES_ISO8601_FORMAT)
+ for f in formats:
+ try:
+ t = time.strptime(seconds, f)
+ except ValueError:
+ t = None
+ else:
+ if f == EXPIRES_ISO8601_FORMAT:
+ timestamp = timegm(t)
+ else:
+ # Use local time if UTC designator is missing.
+ timestamp = int(time.mktime(t))
+
+ absolute = True
+ break
+
+ if t is None:
+ raise ValueError()
+ else:
+ if not timestamp.is_integer():
+ raise ValueError()
+ timestamp = int(timestamp)
+ if timestamp < 0:
+ raise ValueError()
except ValueError:
- raise ValueError('seconds must be a whole number')
+ raise ValueError(TIME_ERRMSG)
if isinstance(path, six.binary_type):
try:
@@ -121,9 +152,9 @@ def generate_temp_url(path, seconds, key, method, absolute=False,
'possibly an error', method.upper())
if not absolute:
- expiration = int(time.time() + seconds)
+ expiration = int(time.time() + timestamp)
else:
- expiration = seconds
+ expiration = timestamp
hmac_body = u'\n'.join([method.upper(), str(expiration),
('prefix:' if prefix else '') + path_for_body])
@@ -132,6 +163,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False,
key = key.encode('utf-8')
sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest()
+ if iso8601:
+ expiration = time.strftime(
+ EXPIRES_ISO8601_FORMAT, time.gmtime(expiration))
+
temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format(
path=path_for_body, sig=sig, exp=expiration)
if prefix:
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index af32d84..b759e6b 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -492,6 +492,40 @@ class TestServiceDelete(_TestServiceBase):
self.assertLessEqual(r['error_timestamp'], after)
self.assertIn('Traceback', r['traceback'])
+ @mock.patch.object(swiftclient.service.SwiftService, 'capabilities',
+ lambda *a: {'action': 'capabilities',
+ 'timestamp': time.time(),
+ 'success': True,
+ 'capabilities': {
+ 'bulk_delete':
+ {'max_deletes_per_request': 10}}
+ })
+ def test_bulk_delete_page_size(self):
+ # make a list of 100 objects
+ obj_list = ['x%02d' % i for i in range(100)]
+ errors = []
+
+ # _bulk_delete_page_size uses 2x the number of threads to determine
+ # if if there are "many" object to delete or not
+
+ # format is: [(thread_count, expected result), ...]
+ obj_threads_exp = [
+ (10, 10), # something small
+ (49, 10), # just under the bounds
+ (50, 1), # cutover point
+ (51, 1), # just over bounds
+ (100, 1), # something big
+ ]
+ for thread_count, exp in obj_threads_exp:
+ s = SwiftService(options={'object_dd_threads': thread_count})
+ res = s._bulk_delete_page_size(obj_list)
+ if res != exp:
+ msg = 'failed for thread_count %d: got %r expected %r' % \
+ (thread_count, res, exp)
+ errors.append(msg)
+ if errors:
+ self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors))
+
class TestSwiftError(unittest.TestCase):
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index f7e9c1a..db96df7 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -23,6 +23,7 @@ import os
import tempfile
import unittest
import textwrap
+from time import localtime, mktime, strftime, strptime
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import six
@@ -36,7 +37,9 @@ from os.path import basename, dirname
from .utils import (
CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client,
FakeKeystone, StubResponse, MockHttpTest)
-from swiftclient.utils import EMPTY_ETAG
+from swiftclient.utils import (
+ EMPTY_ETAG, EXPIRES_ISO8601_FORMAT,
+ SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG)
if six.PY2:
@@ -714,7 +717,6 @@ class TestShell(unittest.TestCase):
mock.ANY,
headers={
'x-object-meta-mtime': mock.ANY,
- 'x-static-large-object': 'true'
},
query_string='multipart-manifest=put',
response_dict={})
@@ -903,8 +905,8 @@ class TestShell(unittest.TestCase):
'x-object-meta-mtime': mock.ANY},
response_dict={})
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 0)
@mock.patch('swiftclient.service.Connection')
def test_delete_bad_threads(self, mock_connection):
mock_connection.return_value.get_container.return_value = (None, [])
@@ -934,8 +936,8 @@ class TestShell(unittest.TestCase):
check_good(["--object-threads", "1"])
check_good(["--container-threads", "1"])
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_delete_account(self, connection):
connection.return_value.get_account.side_effect = [
@@ -971,8 +973,8 @@ class TestShell(unittest.TestCase):
mock.call('container2', response_dict={}, headers={}),
mock.call('empty_container', response_dict={}, headers={})])
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: True)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 10)
@mock.patch('swiftclient.service.Connection')
def test_delete_bulk_account(self, connection):
connection.return_value.get_account.side_effect = [
@@ -1085,8 +1087,80 @@ class TestShell(unittest.TestCase):
self.assertEqual(connection.return_value.get_capabilities.mock_calls,
[mock.call(None)]) # only one /info request
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch('swiftclient.service.Connection')
+ def test_delete_bulk_account_with_capabilities_and_pages(self, connection):
+ connection.return_value.get_capabilities.return_value = {
+ 'bulk_delete': {
+ 'max_deletes_per_request': 2,
+ 'max_failed_deletes': 1000,
+ },
+ }
+ connection.return_value.get_account.side_effect = [
+ [None, [{'name': 'container'}]],
+ [None, [{'name': 'container2'}]],
+ [None, [{'name': 'empty_container'}]],
+ [None, []],
+ ]
+ connection.return_value.get_container.side_effect = [
+ [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'},
+ {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]],
+ [None, []],
+ [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'},
+ {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]],
+ [None, []],
+ [None, []],
+ ]
+ connection.return_value.attempts = 0
+ argv = ["", "delete", "--all", "--object-threads", "1"]
+ connection.return_value.post_account.return_value = {}, (
+ b'{"Number Not Found": 0, "Response Status": "200 OK", '
+ b'"Errors": [], "Number Deleted": 1, "Response Body": ""}')
+ swiftclient.shell.main(argv)
+ # check that each bulk call was only called with 2 objects
+ self.assertEqual(
+ connection.return_value.post_account.mock_calls, [
+ mock.call(query_string='bulk-delete',
+ data=b''.join([
+ b'/container/object\n',
+ b'/container/obj%C3%A9ct2\n',
+ ]),
+ headers={'Content-Type': 'text/plain',
+ 'Accept': 'application/json'},
+ response_dict={}),
+ mock.call(query_string='bulk-delete',
+ data=b''.join([
+ b'/container/z_object\n',
+ b'/container/z_obj%C3%A9ct2\n'
+ ]),
+ headers={'Content-Type': 'text/plain',
+ 'Accept': 'application/json'},
+ response_dict={}),
+ mock.call(query_string='bulk-delete',
+ data=b''.join([
+ b'/container2/object\n',
+ b'/container2/obj%C3%A9ct2\n',
+ ]),
+ headers={'Content-Type': 'text/plain',
+ 'Accept': 'application/json'},
+ response_dict={}),
+ mock.call(query_string='bulk-delete',
+ data=b''.join([
+ b'/container2/z_object\n',
+ b'/container2/z_obj%C3%A9ct2\n'
+ ]),
+ headers={'Content-Type': 'text/plain',
+ 'Accept': 'application/json'},
+ response_dict={})])
+ self.assertEqual(
+ connection.return_value.delete_container.mock_calls, [
+ mock.call('container', response_dict={}, headers={}),
+ mock.call('container2', response_dict={}, headers={}),
+ mock.call('empty_container', response_dict={}, headers={})])
+ self.assertEqual(connection.return_value.get_capabilities.mock_calls,
+ [mock.call(None)]) # only one /info request
+
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_delete_container(self, connection):
connection.return_value.get_container.side_effect = [
@@ -1103,8 +1177,8 @@ class TestShell(unittest.TestCase):
'container', 'object', query_string=None, response_dict={},
headers={})
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_delete_container_headers(self, connection):
connection.return_value.get_container.side_effect = [
@@ -1122,8 +1196,8 @@ class TestShell(unittest.TestCase):
'container', 'object', query_string=None, response_dict={},
headers={'Skip-Middleware': 'Test'})
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: True)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 10)
@mock.patch('swiftclient.service.Connection')
def test_delete_bulk_container(self, connection):
connection.return_value.get_container.side_effect = [
@@ -1176,8 +1250,8 @@ class TestShell(unittest.TestCase):
self.assertTrue(out.out.find(
't\u00e9st_c [after 2 attempts]') >= 0, out)
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_delete_per_object(self, connection):
argv = ["", "delete", "container", "object"]
@@ -1188,8 +1262,8 @@ class TestShell(unittest.TestCase):
'container', 'object', query_string=None, response_dict={},
headers={})
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: True)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 10)
@mock.patch('swiftclient.service.Connection')
def test_delete_bulk_object(self, connection):
argv = ["", "delete", "container", "object"]
@@ -1550,7 +1624,7 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
temp_url.assert_called_with(
'/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False,
- prefix=False)
+ iso8601=False, prefix=False)
@mock.patch('swiftclient.shell.generate_temp_url', return_value='')
def test_temp_url_prefix_based(self, temp_url):
@@ -1559,7 +1633,28 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
temp_url.assert_called_with(
'/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False,
- prefix=True)
+ iso8601=False, prefix=True)
+
+ @mock.patch('swiftclient.shell.generate_temp_url', return_value='')
+ def test_temp_url_iso8601_in(self, temp_url):
+ dates = ('1970-01-01T00:01:00Z', '1970-01-01T00:01:00',
+ '1970-01-01')
+ for d in dates:
+ argv = ["", "tempurl", "GET", d, "/v1/AUTH_account/c/",
+ "secret_key"]
+ swiftclient.shell.main(argv)
+ temp_url.assert_called_with(
+ '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False,
+ iso8601=False, prefix=False)
+
+ @mock.patch('swiftclient.shell.generate_temp_url', return_value='')
+ def test_temp_url_iso8601_out(self, temp_url):
+ argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/",
+ "secret_key", "--iso8601"]
+ swiftclient.shell.main(argv)
+ temp_url.assert_called_with(
+ '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False,
+ iso8601=True, prefix=False)
@mock.patch('swiftclient.shell.generate_temp_url', return_value='')
def test_absolute_expiry_temp_url(self, temp_url):
@@ -1568,7 +1663,7 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
temp_url.assert_called_with(
'/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True,
- prefix=False)
+ iso8601=False, prefix=False)
def test_temp_url_output(self):
argv = ["", "tempurl", "GET", "60", "/v1/a/c/o",
@@ -1595,6 +1690,42 @@ class TestShell(unittest.TestCase):
"&temp_url_prefix=\n" % sig)
self.assertEqual(expected, output.out)
+ argv = ["", "tempurl", "GET", "60", "/v1/a/c/",
+ "secret_key", "--absolute", "--prefix", '--iso8601']
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349'
+ expires = '1970-01-01T00:01:00Z'
+ expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=%s"
+ "&temp_url_prefix=\n" % (sig, expires))
+ self.assertEqual(expected, output.out)
+
+ dates = ("1970-01-01T00:01:00Z",
+ strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60)))
+ for d in dates:
+ argv = ["", "tempurl", "GET", d, "/v1/a/c/o",
+ "secret_key"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3"
+ expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig
+ self.assertEqual(expected, output.out)
+
+ ts = str(int(
+ mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT))))
+
+ argv = ["", "tempurl", "GET", ts, "/v1/a/c/",
+ "secret_key", "--absolute"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ expected = output.out
+
+ argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/",
+ "secret_key", "--absolute"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ self.assertEqual(expected, output.out)
+
def test_temp_url_error_output(self):
expected = 'path must be full path to an object e.g. /v1/a/c/o\n'
for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o',
@@ -1614,7 +1745,17 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
self.assertEqual(expected, output.err,
'Expected %r but got %r for path %r' %
- (expected, output.err, bad_path))
+ (expected, output.err, '/v1/a/c'))
+
+ expected = TIME_ERRMSG + '\n'
+ for bad_time in ('not_an_int', '-1', '2015-05', '2015-05-01T01:00'):
+ argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o',
+ "secret_key", "--absolute"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ self.assertEqual(expected, output.err,
+ 'Expected %r but got %r for time %r' %
+ (expected, output.err, bad_time))
@mock.patch('swiftclient.service.Connection')
def test_capabilities(self, connection):
@@ -2714,8 +2855,8 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest):
return status
return on_request
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_upload_bad_threads(self, mock_connection):
mock_connection.return_value.put_object.return_value = EMPTY_ETAG
@@ -2897,8 +3038,8 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest):
self.assertIn(expected_err, out.err)
self.assertEqual('', out)
- @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete',
- lambda *a: False)
+ @mock.patch.object(swiftclient.service.SwiftService,
+ '_bulk_delete_page_size', lambda *a: 1)
@mock.patch('swiftclient.service.Connection')
def test_download_bad_threads(self, mock_connection):
mock_connection.return_value.get_object.return_value = [{}, '']
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index d4a704e..5384ac7 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -14,6 +14,7 @@
# limitations under the License.
import gzip
+import json
import logging
import mock
import six
@@ -83,7 +84,7 @@ class TestClientException(unittest.TestCase):
class MockHttpResponse(object):
- def __init__(self, status=0, headers=None, verify=False, need_items=None):
+ def __init__(self, status=0, headers=None, verify=False):
self.status = status
self.status_code = status
self.reason = "OK"
@@ -92,7 +93,6 @@ class MockHttpResponse(object):
self.verify = verify
self.md5sum = md5()
self.headers = {'etag': '"%s"' % EMPTY_ETAG}
- self.need_items = need_items
if headers:
self.headers.update(headers)
self.closed = False
@@ -119,9 +119,7 @@ class MockHttpResponse(object):
return self.headers.get(name, default)
def getheaders(self):
- if self.need_items:
- return dict(self.headers).items()
- return dict(self.headers)
+ return dict(self.headers).items()
def fake_response(self):
return self
@@ -1165,6 +1163,7 @@ class TestHeadObject(MockHttpTest):
class TestPutObject(MockHttpTest):
+ @mock.patch('swiftclient.requests.__version__', '2.2.0')
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4)
@@ -1222,6 +1221,7 @@ class TestPutObject(MockHttpTest):
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[-1].category, UserWarning))
+ @mock.patch('swiftclient.requests.__version__', '2.2.0')
def test_server_error(self):
body = 'c' * 60
headers = {'foo': 'bar'}
@@ -1236,7 +1236,8 @@ class TestPutObject(MockHttpTest):
self.assertEqual(e.http_status, 500)
self.assertRequests([
('PUT', '/asdf/asdf', 'asdf', {
- 'x-auth-token': 'asdf', 'content-type': ''}),
+ 'x-auth-token': 'asdf',
+ 'content-type': ''}),
])
def test_query_string(self):
@@ -1377,7 +1378,8 @@ class TestPutObject(MockHttpTest):
self.assertEqual(request_header['etag'], b'1234-5678')
self.assertEqual(request_header['content-type'], b'text/plain')
- def test_no_content_type(self):
+ @mock.patch('swiftclient.requests.__version__', '2.2.0')
+ def test_no_content_type_old_requests(self):
conn = c.http_connection(u'http://www.test.com/')
resp = MockHttpResponse(status=200)
conn[1].getresponse = resp.fake_response
@@ -1387,6 +1389,17 @@ class TestPutObject(MockHttpTest):
request_header = resp.requests_params['headers']
self.assertEqual(request_header['content-type'], b'')
+ @mock.patch('swiftclient.requests.__version__', '2.4.0')
+ def test_no_content_type_new_requests(self):
+ conn = c.http_connection(u'http://www.test.com/')
+ resp = MockHttpResponse(status=200)
+ conn[1].getresponse = resp.fake_response
+ conn[1]._request = resp._fake_request
+
+ c.put_object(url='http://www.test.com', http_conn=conn)
+ request_header = resp.requests_params['headers']
+ self.assertNotIn('content-type', request_header)
+
def test_content_type_in_headers(self):
conn = c.http_connection(u'http://www.test.com/')
resp = MockHttpResponse(status=200)
@@ -2319,7 +2332,7 @@ class TestConnection(MockHttpTest):
return 'header'
def getheaders(self):
- return {"key1": "value1", "key2": "value2"}
+ return [('key1', 'value1'), ('key2', 'value2')]
def read(self, *args, **kwargs):
return ''
@@ -2569,6 +2582,33 @@ class TestLogging(MockHttpTest):
c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf')
self.assertEqual(exc_context.exception.http_status, 404)
+ def test_content_encoding_gzip_body_is_logged_decoded(self):
+ buf = six.BytesIO()
+ gz = gzip.GzipFile(fileobj=buf, mode='w')
+ data = {"test": u"\u2603"}
+ decoded_body = json.dumps(data).encode('utf-8')
+ gz.write(decoded_body)
+ gz.close()
+ # stub a gzip encoded body
+ body = buf.getvalue()
+ headers = {'content-encoding': 'gzip'}
+ # ... and make a content-encoding gzip error response
+ stub_response = StubResponse(500, body, headers)
+ with mock.patch('swiftclient.client.logger.info') as mock_log:
+ # ... if the client gets such a response
+ c.http_connection = self.fake_http_connection(stub_response)
+ with self.assertRaises(c.ClientException) as exc_context:
+ c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf')
+ self.assertEqual(exc_context.exception.http_status, 500)
+ # it will log the decoded body
+ self.assertEqual([
+ mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf '
+ '-X GET -H "X-Auth-Token: ..."'),
+ mock.call('RESP STATUS: %s %s', 500, 'Fake'),
+ mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}),
+ mock.call('RESP BODY: %s', decoded_body)
+ ], mock_log.mock_calls)
+
def test_redact_token(self):
with mock.patch('swiftclient.client.logger.debug') as mock_log:
token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b'
@@ -2611,44 +2651,6 @@ class TestLogging(MockHttpTest):
self.assertNotIn(unicode_token_value, output)
self.assertNotIn(set_cookie_value, output)
- def test_logging_body(self):
- with mock.patch('swiftclient.client.logger.debug') as mock_log:
- token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b'
- token_encoded = token_value.encode('utf8')
- unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91'
- u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91'
- u'\u5929\u7a7a\u4e2d\u7684\u4e4c')
- unicode_token_encoded = unicode_token_value.encode('utf8')
- set_cookie_value = 'X-Auth-Token=%s' % token_value
- set_cookie_encoded = set_cookie_value.encode('utf8')
- buf = six.BytesIO()
- gz = gzip.GzipFile(fileobj=buf, mode='w')
- gz.write(u'{"test": "\u2603"}'.encode('utf8'))
- gz.close()
- c.http_log(
- ['GET'],
- {'headers': {
- 'X-Auth-Token': token_encoded,
- 'X-Storage-Token': unicode_token_encoded
- }},
- MockHttpResponse(
- status=200,
- headers={
- 'X-Auth-Token': token_encoded,
- 'X-Storage-Token': unicode_token_encoded,
- 'content-encoding': 'gzip',
- 'Etag': b'mock_etag',
- 'Set-Cookie': set_cookie_encoded
- },
- need_items=True,
- ),
- buf.getvalue(),
- )
- self.assertEqual(
- mock.call(
- 'RESP BODY: %s', u'{"test": "\u2603"}'.encode('utf8')),
- mock_log.mock_calls[3])
-
def test_show_token(self):
with mock.patch('swiftclient.client.logger.debug') as mock_log:
token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b'
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index c5961e8..adead00 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -18,6 +18,7 @@ import unittest
import mock
import six
import tempfile
+from time import gmtime, localtime, mktime, strftime, strptime
from hashlib import md5, sha1
from swiftclient import utils as u
@@ -151,6 +152,67 @@ class TestTempURL(unittest.TestCase):
self.assertIsInstance(url, type(self.url))
@mock.patch('hmac.HMAC')
+ def test_generate_temp_url_iso8601_argument(self, hmac_mock):
+ hmac_mock().hexdigest.return_value = 'temp_url_signature'
+ url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z',
+ self.key, self.method)
+ self.assertEqual(url, self.expected_url)
+
+ # Don't care about absolute arg.
+ url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z',
+ self.key, self.method, absolute=True)
+ self.assertEqual(url, self.expected_url)
+
+ lt = localtime()
+ expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt)
+
+ if not isinstance(self.expected_url, six.string_types):
+ expected_url = self.expected_url.replace(
+ b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
+ else:
+ expected_url = self.expected_url.replace(
+ '1400003600', str(int(mktime(lt))))
+ url = u.generate_temp_url(self.url, expires,
+ self.key, self.method)
+ self.assertEqual(url, expected_url)
+
+ expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt)
+ lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT)
+
+ if not isinstance(self.expected_url, six.string_types):
+ expected_url = self.expected_url.replace(
+ b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii'))
+ else:
+ expected_url = self.expected_url.replace(
+ '1400003600', str(int(mktime(lt))))
+ url = u.generate_temp_url(self.url, expires,
+ self.key, self.method)
+ self.assertEqual(url, expected_url)
+
+ @mock.patch('hmac.HMAC')
+ @mock.patch('time.time', return_value=1400000000)
+ def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock):
+ hmac_mock().hexdigest.return_value = 'temp_url_signature'
+ url = u.generate_temp_url(self.url, self.seconds,
+ self.key, self.method,
+ iso8601=True)
+ key = self.key
+ if not isinstance(key, six.binary_type):
+ key = key.encode('utf-8')
+
+ expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600))
+ if not isinstance(self.url, six.string_types):
+ self.assertTrue(url.endswith(bytes(expires, 'utf-8')))
+ else:
+ self.assertTrue(url.endswith(expires))
+ self.assertEqual(hmac_mock.mock_calls, [
+ mock.call(),
+ mock.call(key, self.expected_body, sha1),
+ mock.call().hexdigest(),
+ ])
+ self.assertIsInstance(url, type(self.url))
+
+ @mock.patch('hmac.HMAC')
@mock.patch('time.time', return_value=1400000000)
def test_generate_temp_url_prefix(self, time_mock, hmac_mock):
hmac_mock().hexdigest.return_value = 'temp_url_signature'
@@ -198,31 +260,34 @@ class TestTempURL(unittest.TestCase):
absolute=True)
self.assertEqual(url, expected_url)
- def test_generate_temp_url_bad_seconds(self):
+ def test_generate_temp_url_bad_time(self):
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
- self.assertEqual(exc_manager.exception.args[0],
- 'seconds must be a whole number')
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, -1, self.key, self.method)
- self.assertEqual(exc_manager.exception.args[0],
- 'seconds must be a whole number')
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, 1.1, self.key, self.method)
- self.assertEqual(exc_manager.exception.args[0],
- 'seconds must be a whole number')
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '-1', self.key, self.method)
- self.assertEqual(exc_manager.exception.args[0],
- 'seconds must be a whole number')
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
with self.assertRaises(ValueError) as exc_manager:
u.generate_temp_url(self.url, '1.1', self.key, self.method)
- self.assertEqual(exc_manager.exception.args[0],
- 'seconds must be a whole number')
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
+ with self.assertRaises(ValueError) as exc_manager:
+ u.generate_temp_url(self.url, '2015-05', self.key, self.method)
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
+
+ with self.assertRaises(ValueError) as exc_manager:
+ u.generate_temp_url(
+ self.url, '2015-05-01T01:00', self.key, self.method)
+ self.assertEqual(exc_manager.exception.args[0], u.TIME_ERRMSG)
def test_generate_temp_url_bad_path(self):
with self.assertRaises(ValueError) as exc_manager:
View Lorry Controller Status