summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.rst2
-rw-r--r--bindep.txt5
-rw-r--r--doc/manpages/swift.133
-rw-r--r--doc/source/cli.rst3
-rw-r--r--doc/source/client-api.rst2
-rw-r--r--doc/source/conf.py2
-rw-r--r--doc/source/service-api.rst3
-rw-r--r--setup.py6
-rw-r--r--swiftclient/client.py34
-rw-r--r--swiftclient/service.py8
-rwxr-xr-xswiftclient/shell.py36
-rw-r--r--swiftclient/utils.py45
-rw-r--r--tests/unit/test_service.py53
-rw-r--r--tests/unit/test_shell.py62
-rw-r--r--tests/unit/test_swiftclient.py31
-rw-r--r--tests/unit/test_utils.py142
-rw-r--r--tests/unit/utils.py3
17 files changed, 352 insertions, 118 deletions
diff --git a/README.rst b/README.rst
index 3677406..cd5efc3 100644
--- a/README.rst
+++ b/README.rst
@@ -17,7 +17,7 @@ in the `OpenStack wiki`__.
__ http://docs.openstack.org/infra/manual/developers.html
-This code is based on original the client previously included with
+This code is based on the original client previously included with
`OpenStack's Swift`__ The python-swiftclient is licensed under the
Apache License like the rest of OpenStack.
diff --git a/bindep.txt b/bindep.txt
new file mode 100644
index 0000000..c587dda
--- /dev/null
+++ b/bindep.txt
@@ -0,0 +1,5 @@
+# This is a cross-platform list tracking distribution packages needed by tests;
+# see http://docs.openstack.org/infra/bindep/ for additional information.
+
+pypy
+pypy-dev
diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1
index 7c2ee89..5d21c18 100644
--- a/doc/manpages/swift.1
+++ b/doc/manpages/swift.1
@@ -61,8 +61,8 @@ of container or objects being listed. With the \-t or \-\-total option they only
.RS 4
Uploads to the given container the files and directories specified by the
remaining args. The \-c or \-\-changed is an option that will only upload files
-that have changed since the last upload. The \-\-object-name <object\-name> is
-an option that will upload file and name object to <object-name> or upload dir
+that have changed since the last upload. The \-\-object\-name <object\-name> is
+an option that will upload file and name object to <object\-name> or upload dir
and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size>
and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more).
.RE
@@ -99,6 +99,7 @@ container, or a list of objects depending on the args given. For a single
object download, you may use the \-o [\-\-output] <filename> option to
redirect the output to a specific file or if "-" then just redirect to stdout or
with \-\-no-download actually not to write anything to disk.
+The \-\-ignore-checksum is an option that turns off checksum validation.
You can specify optional headers with the repeatable cURL-like option
\-H [\-\-header]. For more details and options see swift download \-\-help.
.RE
@@ -112,23 +113,31 @@ will be deleted as well, unless you specify the \-\-leave\-segments option.
For more details and options see swift delete \-\-help.
.RE
-\fBcapabilities\fR [\fIproxy-url\fR]
+\fBcapabilities\fR [\fIcommand-options\fR] [\fIproxy-url\fR]
.RS 4
-Displays cluster capabilities. The output includes the list of the activated
-Swift middlewares as well as relevant options for each ones. Addtionaly the
-command displays relevant options for the Swift core. If the proxy-url option
-is not provided the storage-url retrieved after authentication is used as
-proxy-url.
+Displays cluster capabilities. If the proxy-url option is not provided the
+storage-url retrieved after authentication is used as proxy-url.
+
+By default, the output includes the list of the activated Swift middlewares as
+well as relevant options for each one. Additionally the command displays
+relevant options for the Swift core.
+
+The \-\-json option will print a json representation of the cluster
+capabilities. This is typically more suitable for consumption by other
+programs, such as jq.
+
+\fBExample\fR: capabilities https://swift.example.com
+ capabilities \-\-json
.RE
\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR
.RS 4
Generates a temporary URL allowing unauthenticated access to the Swift object
at the given path, using the given HTTP method, for the given number of
-seconds, using the given TempURL key. If optional --absolute argument is
+seconds, using the given TempURL key. If optional \-\-absolute argument is
provided, seconds is instead interpreted as a Unix timestamp at which the URL
-should expire. \fBExample\fR: tempurl GET $(date -d "Jan 1 2016" +%s)
-/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key --absolute
+should expire. \fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s)
+/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key \-\-absolute
.RE
\fBauth\fR
@@ -152,7 +161,7 @@ For examples see swift auth \-\-help.
.IP "--os-help Show all OpenStack authentication options"
.PD
.RS 4
-For more options see swift \-\-help and swift \-\-os-help.
+For more options see swift \-\-help and swift \-\-os\-help.
.RE
diff --git a/doc/source/cli.rst b/doc/source/cli.rst
index 76630be..9bb229d 100644
--- a/doc/source/cli.rst
+++ b/doc/source/cli.rst
@@ -186,7 +186,8 @@ Download
container, or a list of objects depending on the arguments given. For a
single object download, you may use the ``-o <filename>`` or ``--output <filename>``
option to redirect the output to a specific file or ``-`` to
- redirect to stdout. You can specify optional headers with the repeatable
+ redirect to stdout. The ``--ignore-checksum`` is an option that turn off
+ checksum validation. You can specify optional headers with the repeatable
cURL-like option ``-H [--header <name:value>]``.
Delete
diff --git a/doc/source/client-api.rst b/doc/source/client-api.rst
index 5677f70..b0bb637 100644
--- a/doc/source/client-api.rst
+++ b/doc/source/client-api.rst
@@ -12,7 +12,7 @@ Authentication
--------------
This section covers the various combinations of kwargs required when creating
-and instance of the ``Connection`` object for communicating with a swift
+an instance of the ``Connection`` object for communicating with a swift
object store. The combinations of options required for each authentication
version are detailed below, but are
just a subset of those that can be used to successfully authenticate. These
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 0b3e7e1..5af77b2 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -111,6 +111,8 @@ pygments_style = 'sphinx'
# documentation.
# html_theme_options = {}
+html_theme_options = {'show_other_versions': True}
+
# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst
index aeb3daa..ea530b0 100644
--- a/doc/source/service-api.rst
+++ b/doc/source/service-api.rst
@@ -207,6 +207,9 @@ Options
Affects uploads, and allows empty 'pseudofolder' objects to be created
when the source of an upload is ``None``.
+ ``checksum``: ``True``
+ Affects uploads and downloads. If set check md5 sum for the transfer.
+
``shuffle``: ``False``
When downloading objects, the default behaviour of the CLI is to shuffle
lists of objects in order to spread the load on storage drives when multiple
diff --git a/setup.py b/setup.py
index 70c2b3f..16a18f6 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,11 @@
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
-import setuptools
+import setuptools, sys
+
+if sys.version_info < (2, 7):
+ sys.exit('Sorry, Python < 2.7 is not supported for'
+ ' python-swiftclient>=3.0')
setuptools.setup(
setup_requires=['pbr'],
diff --git a/swiftclient/client.py b/swiftclient/client.py
index 5b3fa72..602489d 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -827,7 +827,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None,
def get_container(url, token, container, marker=None, limit=None,
prefix=None, delimiter=None, end_marker=None,
path=None, http_conn=None,
- full_listing=False, service_token=None, headers=None):
+ full_listing=False, service_token=None, headers=None,
+ query_string=None):
"""
Get a listing of objects for the container.
@@ -846,6 +847,7 @@ def get_container(url, token, container, marker=None, limit=None,
of 10000 listings
:param service_token: service auth token
:param headers: additional headers to include in the request
+ :param query_string: if set will be appended with '?' to generated path
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
@@ -889,6 +891,8 @@ def get_container(url, token, container, marker=None, limit=None,
qs += '&end_marker=%s' % quote(end_marker)
if path:
qs += '&path=%s' % quote(path)
+ if query_string:
+ qs += '&%s' % query_string.lstrip('?')
if service_token:
headers['X-Service-Token'] = service_token
method = 'GET'
@@ -950,7 +954,7 @@ def head_container(url, token, container, http_conn=None, headers=None,
def put_container(url, token, container, headers=None, http_conn=None,
- response_dict=None, service_token=None):
+ response_dict=None, service_token=None, query_string=None):
"""
Create a container
@@ -963,6 +967,7 @@ def put_container(url, token, container, headers=None, http_conn=None,
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
+ :param query_string: if set will be appended with '?' to generated path
:raises ClientException: HTTP PUT request failed
"""
if http_conn:
@@ -978,6 +983,8 @@ def put_container(url, token, container, headers=None, http_conn=None,
headers['X-Service-Token'] = service_token
if 'content-length' not in (k.lower() for k in headers):
headers['Content-Length'] = '0'
+ if query_string:
+ path += '?' + query_string.lstrip('?')
conn.request(method, path, '', headers)
resp = conn.getresponse()
body = resp.read()
@@ -1031,7 +1038,8 @@ def post_container(url, token, container, headers, http_conn=None,
def delete_container(url, token, container, http_conn=None,
- response_dict=None, service_token=None):
+ response_dict=None, service_token=None,
+ query_string=None):
"""
Delete a container
@@ -1043,6 +1051,7 @@ def delete_container(url, token, container, http_conn=None,
:param response_dict: an optional dictionary into which to place
the response - status, reason and headers
:param service_token: service auth token
+ :param query_string: if set will be appended with '?' to generated path
:raises ClientException: HTTP DELETE request failed
"""
if http_conn:
@@ -1053,6 +1062,8 @@ def delete_container(url, token, container, http_conn=None,
headers = {'X-Auth-Token': token}
if service_token:
headers['X-Service-Token'] = service_token
+ if query_string:
+ path += '?' + query_string.lstrip('?')
method = 'DELETE'
conn.request(method, path, '', headers)
resp = conn.getresponse()
@@ -1689,7 +1700,7 @@ class Connection(object):
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, end_marker=None, path=None,
- full_listing=False, headers=None):
+ full_listing=False, headers=None, query_string=None):
"""Wrapper for :func:`get_container`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
@@ -1697,22 +1708,27 @@ class Connection(object):
return self._retry(None, get_container, container, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
end_marker=end_marker, path=path,
- full_listing=full_listing, headers=headers)
+ full_listing=full_listing, headers=headers,
+ query_string=query_string)
- def put_container(self, container, headers=None, response_dict=None):
+ def put_container(self, container, headers=None, response_dict=None,
+ query_string=None):
"""Wrapper for :func:`put_container`"""
return self._retry(None, put_container, container, headers=headers,
- response_dict=response_dict)
+ response_dict=response_dict,
+ query_string=query_string)
def post_container(self, container, headers, response_dict=None):
"""Wrapper for :func:`post_container`"""
return self._retry(None, post_container, container, headers,
response_dict=response_dict)
- def delete_container(self, container, response_dict=None):
+ def delete_container(self, container, response_dict=None,
+ query_string=None):
"""Wrapper for :func:`delete_container`"""
return self._retry(None, delete_container, container,
- response_dict=response_dict)
+ response_dict=response_dict,
+ query_string=query_string)
def head_object(self, container, obj, headers=None):
"""Wrapper for :func:`head_object`"""
diff --git a/swiftclient/service.py b/swiftclient/service.py
index a813d16..af412d1 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -374,7 +374,7 @@ class _SwiftReader(object):
errors on failures caused by either invalid md5sum or size of the
data read.
"""
- def __init__(self, path, body, headers):
+ def __init__(self, path, body, headers, checksum=True):
self._path = path
self._body = body
self._actual_read = 0
@@ -383,7 +383,7 @@ class _SwiftReader(object):
self._expected_etag = headers.get('etag')
if ('x-object-manifest' not in headers
- and 'x-static-large-object' not in headers):
+ and 'x-static-large-object' not in headers and checksum):
self._actual_md5 = md5()
if 'content-length' in headers:
@@ -1018,6 +1018,7 @@ class SwiftService(object):
'header': [],
'skip_identical': False,
'out_directory': None,
+ 'checksum': True,
'out_file': None,
'remove_prefix': False,
'shuffle' : False
@@ -1173,7 +1174,8 @@ class SwiftService(object):
headers_receipt = time()
- obj_body = _SwiftReader(path, body, headers)
+ obj_body = _SwiftReader(path, body, headers,
+ options.get('checksum', True))
no_file = options['no_download']
if out_file == "-" and not no_file:
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 8da0e7b..be1888d 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -17,6 +17,7 @@
from __future__ import print_function, unicode_literals
import argparse
+import json
import logging
import signal
import socket
@@ -204,7 +205,7 @@ def st_delete(parser, args, output_manager):
st_download_options = '''[--all] [--marker <marker>] [--prefix <prefix>]
[--output <out_file>] [--output-dir <out_directory>]
- [--object-threads <threads>]
+ [--object-threads <threads>] [--ignore-checksum]
[--container-threads <threads>] [--no-download]
[--skip-identical] [--remove-prefix]
[--header <header:value>] [--no-shuffle]
@@ -248,9 +249,10 @@ Optional arguments:
-H, --header <header:value>
Adds a customized request header to the query, like
"Range" or "If-Match". This option may be repeated.
- Example --header "content-type:text/plain"
+ Example: --header "content-type:text/plain"
--skip-identical Skip downloading files that are identical on both
sides.
+ --ignore-checksum Turn off checksum validation for downloads.
--no-shuffle By default, when downloading a complete account or
container, download order is randomised in order to
reduce the load on individual drives when multiple
@@ -309,6 +311,9 @@ def st_download(parser, args, output_manager):
default=False, help='Skip downloading files that are identical on '
'both sides.')
parser.add_argument(
+ '--ignore-checksum', action='store_false', dest='checksum',
+ default=True, help='Turn off checksum validation for downloads.')
+ parser.add_argument(
'--no-shuffle', action='store_false', dest='shuffle',
default=True, help='By default, download order is randomised in order '
'to reduce the load on individual drives when multiple clients are '
@@ -884,7 +889,7 @@ Optional arguments:
Default is 10.
-H, --header <header:value>
Adds a customized request header. This option may be
- repeated. Example -H "content-type:text/plain"
+ repeated. Example: -H "content-type:text/plain"
-H "Content-Length: 4000".
--use-slo When used in conjunction with --segment-size it will
create a Static Large Object instead of the default
@@ -935,7 +940,7 @@ def st_upload(parser, args, output_manager):
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[], help='Set request headers with the syntax header:value. '
- ' This option may be repeated. Example -H "content-type:text/plain" '
+ ' This option may be repeated. Example: -H "content-type:text/plain" '
'-H "Content-Length: 4000"')
parser.add_argument(
'--use-slo', action='store_true', default=False,
@@ -1090,13 +1095,16 @@ def st_upload(parser, args, output_manager):
output_manager.error(e.value)
-st_capabilities_options = "[<proxy_url>]"
+st_capabilities_options = "[--json] [<proxy_url>]"
st_info_options = st_capabilities_options
st_capabilities_help = '''
Retrieve capability of the proxy.
Optional positional arguments:
<proxy_url> Proxy URL of the cluster to retrieve capabilities.
+
+Optional arguments:
+ --json Print the cluster capabilities in JSON format.
'''.strip('\n')
st_info_help = st_capabilities_help
@@ -1112,6 +1120,8 @@ def st_capabilities(parser, args, output_manager):
key=lambda x: x[0]):
output_manager.print_msg(" %s: %s" % (key, value))
+ parser.add_argument('--json', action='store_true',
+ help='print capability information in json')
(options, args) = parse_args(parser, args)
if args and len(args) > 2:
output_manager.error('Usage: %s capabilities %s\n%s',
@@ -1129,9 +1139,14 @@ def st_capabilities(parser, args, output_manager):
capabilities_result = swift.capabilities()
capabilities = capabilities_result['capabilities']
- _print_compo_cap('Core', {'swift': capabilities['swift']})
- del capabilities['swift']
- _print_compo_cap('Additional middleware', capabilities)
+ if options['json']:
+ output_manager.print_msg(
+ json.dumps(capabilities, sort_keys=True, indent=2))
+ else:
+ capabilities = dict(capabilities)
+ _print_compo_cap('Core', {'swift': capabilities['swift']})
+ del capabilities['swift']
+ _print_compo_cap('Additional middleware', capabilities)
except SwiftError as e:
output_manager.error(e.value)
@@ -1624,6 +1639,11 @@ Examples:
with OutputManager() as output:
parser.usage = globals()['st_%s_help' % args[0]]
+ if options['insecure']:
+ import requests
+ from requests.packages.urllib3.exceptions import \
+ InsecureRequestWarning
+ requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
try:
globals()['st_%s' % args[0]](parser, argv[1:], output)
except ClientException as err:
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index 0abaed6..10687bf 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -78,15 +78,20 @@ def generate_temp_url(path, seconds, key, method, absolute=False):
:raises: TypeError if seconds is not an integer
:return: the path portion of a temporary URL
"""
- if seconds < 0:
- raise ValueError('seconds must be a positive integer')
try:
- if not absolute:
- expiration = int(time.time() + seconds)
- else:
- expiration = int(seconds)
- except TypeError:
+ seconds = int(seconds)
+ except ValueError:
raise TypeError('seconds must be an integer')
+ if seconds < 0:
+ raise ValueError('seconds must be a positive integer')
+
+ if isinstance(path, six.binary_type):
+ try:
+ path_for_body = path.decode('utf-8')
+ except UnicodeDecodeError:
+ raise ValueError('path must be representable as UTF-8')
+ else:
+ path_for_body = path
standard_methods = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
if method.upper() not in standard_methods:
@@ -94,18 +99,24 @@ def generate_temp_url(path, seconds, key, method, absolute=False):
logger.warning('Non default HTTP method %s for tempurl specified, '
'possibly an error', method.upper())
- hmac_body = '\n'.join([method.upper(), str(expiration), path])
+ if not absolute:
+ expiration = int(time.time() + seconds)
+ else:
+ expiration = seconds
+ hmac_body = u'\n'.join([method.upper(), str(expiration), path_for_body])
# Encode to UTF-8 for py3 compatibility
- sig = hmac.new(key.encode(),
- hmac_body.encode(),
- hashlib.sha1).hexdigest()
-
- return ('{path}?temp_url_sig='
- '{sig}&temp_url_expires={exp}'.format(
- path=path,
- sig=sig,
- exp=expiration))
+ if not isinstance(key, six.binary_type):
+ key = key.encode('utf-8')
+ sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest()
+
+ temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format(
+ path=path_for_body, sig=sig, exp=expiration)
+ # Have return type match path from caller
+ if isinstance(path, six.binary_type):
+ return temp_url.encode('utf-8')
+ else:
+ return temp_url
def parse_api_response(headers, body):
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index 5b04c44..d306fdb 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -59,7 +59,7 @@ class TestSwiftPostObject(unittest.TestCase):
spo = self.spo('obj_name')
self.assertEqual(spo.object_name, 'obj_name')
- self.assertEqual(spo.options, None)
+ self.assertIsNone(spo.options)
def test_create_with_invalid_name(self):
# empty strings are not allowed as names
@@ -117,10 +117,10 @@ class TestSwiftReader(unittest.TestCase):
self.assertEqual(sr._path, 'path')
self.assertEqual(sr._body, 'body')
- self.assertEqual(sr._content_length, None)
- self.assertEqual(sr._expected_etag, None)
+ self.assertIsNone(sr._content_length)
+ self.assertIsNone(sr._expected_etag)
- self.assertNotEqual(sr._actual_md5, None)
+ self.assertIsNotNone(sr._actual_md5)
self.assertIs(type(sr._actual_md5), self.md5_type)
def test_create_with_large_object_headers(self):
@@ -128,16 +128,25 @@ class TestSwiftReader(unittest.TestCase):
sr = self.sr('path', 'body', {'x-object-manifest': 'test'})
self.assertEqual(sr._path, 'path')
self.assertEqual(sr._body, 'body')
- self.assertEqual(sr._content_length, None)
- self.assertEqual(sr._expected_etag, None)
- self.assertEqual(sr._actual_md5, None)
+ self.assertIsNone(sr._content_length)
+ self.assertIsNone(sr._expected_etag)
+ self.assertIsNone(sr._actual_md5)
sr = self.sr('path', 'body', {'x-static-large-object': 'test'})
self.assertEqual(sr._path, 'path')
self.assertEqual(sr._body, 'body')
- self.assertEqual(sr._content_length, None)
- self.assertEqual(sr._expected_etag, None)
- self.assertEqual(sr._actual_md5, None)
+ self.assertIsNone(sr._content_length)
+ self.assertIsNone(sr._expected_etag)
+ self.assertIsNone(sr._actual_md5)
+
+ def test_create_with_ignore_checksum(self):
+ # md5 should not be initialized if checksum is False
+ sr = self.sr('path', 'body', {}, False)
+ self.assertEqual(sr._path, 'path')
+ self.assertEqual(sr._body, 'body')
+ self.assertIsNone(sr._content_length)
+ self.assertIsNone(sr._expected_etag)
+ self.assertIsNone(sr._actual_md5)
def test_create_with_content_length(self):
sr = self.sr('path', 'body', {'content-length': 5})
@@ -145,9 +154,9 @@ class TestSwiftReader(unittest.TestCase):
self.assertEqual(sr._path, 'path')
self.assertEqual(sr._body, 'body')
self.assertEqual(sr._content_length, 5)
- self.assertEqual(sr._expected_etag, None)
+ self.assertIsNone(sr._expected_etag)
- self.assertNotEqual(sr._actual_md5, None)
+ self.assertIsNotNone(sr._actual_md5)
self.assertIs(type(sr._actual_md5), self.md5_type)
# Check Contentlength raises error if it isn't an integer
@@ -428,10 +437,10 @@ class TestSwiftError(unittest.TestCase):
se = SwiftError(5)
self.assertEqual(se.value, 5)
- self.assertEqual(se.container, None)
- self.assertEqual(se.obj, None)
- self.assertEqual(se.segment, None)
- self.assertEqual(se.exception, None)
+ self.assertIsNone(se.container)
+ self.assertIsNone(se.obj)
+ self.assertIsNone(se.segment)
+ self.assertIsNone(se.exception)
self.assertEqual(str(se), '5')
@@ -553,12 +562,12 @@ class TestSwiftUploadObject(unittest.TestCase):
suo = self.suo('source')
self.assertEqual(suo.source, 'source')
self.assertEqual(suo.object_name, 'source')
- self.assertEqual(suo.options, None)
+ self.assertIsNone(suo.options)
suo = self.suo('source', 'obj_name')
self.assertEqual(suo.source, 'source')
self.assertEqual(suo.object_name, 'obj_name')
- self.assertEqual(suo.options, None)
+ self.assertIsNone(suo.options)
suo = self.suo('source', 'obj_name', {'opt': '123'})
self.assertEqual(suo.source, 'source')
@@ -577,7 +586,7 @@ class TestSwiftUploadObject(unittest.TestCase):
suo = self.suo(mock_file, 'obj_name')
self.assertEqual(suo.source, mock_file)
self.assertEqual(suo.object_name, 'obj_name')
- self.assertEqual(suo.options, None)
+ self.assertIsNone(suo.options)
suo = self.suo(mock_file, 'obj_name', {'opt': '123'})
self.assertEqual(suo.source, mock_file)
@@ -586,9 +595,9 @@ class TestSwiftUploadObject(unittest.TestCase):
def test_create_with_no_source(self):
suo = self.suo(None, 'obj_name')
- self.assertEqual(suo.source, None)
+ self.assertIsNone(suo.source)
self.assertEqual(suo.object_name, 'obj_name')
- self.assertEqual(suo.options, None)
+ self.assertIsNone(suo.options)
# Check error is raised if source is None without an object name
self.assertRaises(SwiftError, self.suo, None)
@@ -1895,7 +1904,7 @@ class TestServiceDownload(_TestServiceBase):
mock_down_cont.assert_not_called()
next(service.download('c', options=self.opts), None)
- self.assertEqual(True, mock_down_cont.called)
+ self.assertTrue(mock_down_cont.called)
def test_download_with_output_dir(self):
with mock.patch('swiftclient.service.Connection') as mock_conn:
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index 407076b..c0fcc41 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -16,6 +16,7 @@ from __future__ import unicode_literals
from genericpath import getmtime
import hashlib
+import json
import logging
import mock
import os
@@ -23,7 +24,7 @@ import tempfile
import unittest
import textwrap
-
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
import six
import swiftclient
@@ -367,6 +368,24 @@ class TestShell(unittest.TestCase):
mock_open.assert_called_with('object', 'wb')
self.assertEqual([], makedirs.mock_calls)
+ # Test downloading without md5 checks
+ objcontent = six.BytesIO(b'objcontent')
+ connection.return_value.get_object.side_effect = [
+ ({'content-type': 'text/plain',
+ 'etag': '2cbbfe139a744d6abbe695e17f3c1991'},
+ objcontent)
+ ]
+ with mock.patch(BUILTIN_OPEN) as mock_open, mock.patch(
+ 'swiftclient.service._SwiftReader') as sr:
+ argv = ["", "download", "container", "object", "--ignore-check"]
+ swiftclient.shell.main(argv)
+ connection.return_value.get_object.assert_called_with(
+ 'container', 'object', headers={}, resp_chunk_size=65536,
+ response_dict={})
+ mock_open.assert_called_with('object', 'wb')
+ sr.assert_called_once_with('object', mock.ANY, mock.ANY, False)
+ self.assertEqual([], makedirs.mock_calls)
+
# Test downloading single object to stdout
objcontent = six.BytesIO(b'objcontent')
connection.return_value.get_object.side_effect = [
@@ -1400,6 +1419,21 @@ class TestShell(unittest.TestCase):
swiftclient.shell.main(argv)
connection.return_value.get_capabilities.assert_called_with(None)
+ @mock.patch('swiftclient.service.Connection')
+ def test_capabilities_json(self, connection):
+ capabilities = {
+ 'slo': {'min_segment_size': 1000000},
+ 'some': [{'arbitrary': 'nested'}, {'crazy': 'structure'}],
+ 'swift': {'version': '2.5.0'}}
+
+ connection.return_value.get_capabilities.return_value = capabilities
+ argv = ["", "capabilities", "--json"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ expected = json.dumps(capabilities, sort_keys=True, indent=2) + '\n'
+ self.assertEqual(expected, output.out)
+ connection.return_value.get_capabilities.assert_called_with(None)
+
def test_human_readable_upload_segment_size(self):
def _check_expected(x, expected):
actual = x.call_args_list[-1][1]["options"]["segment_size"]
@@ -2032,15 +2066,23 @@ class TestKeystoneOptions(MockHttpTest):
auth_token=token)
with mock.patch('swiftclient.client._import_keystone_client',
- _make_fake_import_keystone_client(fake_ks)):
- with mock.patch('swiftclient.client.http_connection', fake_conn):
- with mock.patch.dict(os.environ, env, clear=True):
- try:
- swiftclient.shell.main(args)
- except SystemExit as e:
- self.fail('Unexpected SystemExit: %s' % e)
- except SwiftError as err:
- self.fail('Unexpected SwiftError: %s' % err)
+ _make_fake_import_keystone_client(fake_ks)), \
+ mock.patch('swiftclient.client.http_connection', fake_conn), \
+ mock.patch.dict(os.environ, env, clear=True), \
+ mock.patch('requests.packages.urllib3.disable_warnings') as \
+ mock_disable_warnings:
+ try:
+ swiftclient.shell.main(args)
+ except SystemExit as e:
+ self.fail('Unexpected SystemExit: %s' % e)
+ except SwiftError as err:
+ self.fail('Unexpected SwiftError: %s' % err)
+
+ if 'insecure' in flags:
+ self.assertEqual([mock.call(InsecureRequestWarning)],
+ mock_disable_warnings.mock_calls)
+ else:
+ self.assertEqual([], mock_disable_warnings.mock_calls)
if no_auth:
# check that keystone client was not used and terminate tests
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index cc6a4ac..4e4c9f4 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -772,6 +772,16 @@ class TestGetContainer(MockHttpTest):
}),
])
+ def test_query_string(self):
+ c.http_connection = self.fake_http_connection(
+ 200, query_string="format=json&hello=20", body=b'[]')
+ c.get_container('http://www.test.com', 'asdf', 'asdf',
+ query_string="hello=20")
+ self.assertRequests([
+ ('GET', '/asdf?format=json&hello=20', '', {
+ 'x-auth-token': 'asdf'}),
+ ])
+
class TestHeadContainer(MockHttpTest):
@@ -831,6 +841,17 @@ class TestPutContainer(MockHttpTest):
'content-length': '0'}),
])
+ def test_query_string(self):
+ c.http_connection = self.fake_http_connection(200,
+ query_string="hello=20")
+ c.put_container('http://www.test.com', 'asdf', 'asdf',
+ query_string="hello=20")
+ for req in self.iter_request_log():
+ self.assertEqual(req['method'], 'PUT')
+ self.assertEqual(req['parsed_path'].path, '/asdf')
+ self.assertEqual(req['parsed_path'].query, 'hello=20')
+ self.assertEqual(req['headers']['x-auth-token'], 'asdf')
+
class TestDeleteContainer(MockHttpTest):
@@ -843,6 +864,16 @@ class TestDeleteContainer(MockHttpTest):
'x-auth-token': 'token'}),
])
+ def test_query_string(self):
+ c.http_connection = self.fake_http_connection(200,
+ query_string="hello=20")
+ c.delete_container('http://www.test.com', 'token', 'container',
+ query_string="hello=20")
+ self.assertRequests([
+ ('DELETE', 'http://www.test.com/container?hello=20', '', {
+ 'x-auth-token': 'token'})
+ ])
+
class TestGetObject(MockHttpTest):
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index aae466c..0f210a3 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -17,7 +17,7 @@ import unittest
import mock
import six
import tempfile
-from hashlib import md5
+from hashlib import md5, sha1
from swiftclient import utils as u
@@ -120,48 +120,124 @@ class TestPrtBytes(unittest.TestCase):
class TestTempURL(unittest.TestCase):
-
- def setUp(self):
- super(TestTempURL, self).setUp()
- self.url = '/v1/AUTH_account/c/o'
- self.seconds = 3600
- self.key = 'correcthorsebatterystaple'
- self.method = 'GET'
-
- @mock.patch('hmac.HMAC.hexdigest', return_value='temp_url_signature')
+ url = '/v1/AUTH_account/c/o'
+ seconds = 3600
+ key = 'correcthorsebatterystaple'
+ method = 'GET'
+ expected_url = url + ('?temp_url_sig=temp_url_signature'
+ '&temp_url_expires=1400003600')
+ expected_body = '\n'.join([
+ method,
+ '1400003600',
+ url,
+ ]).encode('utf-8')
+
+ @mock.patch('hmac.HMAC')
@mock.patch('time.time', return_value=1400000000)
def test_generate_temp_url(self, time_mock, hmac_mock):
- expected_url = (
- '/v1/AUTH_account/c/o?'
- 'temp_url_sig=temp_url_signature&'
- 'temp_url_expires=1400003600')
- url = u.generate_temp_url(self.url, self.seconds, self.key,
- self.method)
- self.assertEqual(url, expected_url)
+ hmac_mock().hexdigest.return_value = 'temp_url_signature'
+ url = u.generate_temp_url(self.url, self.seconds,
+ self.key, self.method)
+ key = self.key
+ if not isinstance(key, six.binary_type):
+ key = key.encode('utf-8')
+ self.assertEqual(url, self.expected_url)
+ self.assertEqual(hmac_mock.mock_calls, [
+ mock.call(),
+ mock.call(key, self.expected_body, sha1),
+ mock.call().hexdigest(),
+ ])
+ self.assertIsInstance(url, type(self.url))
+
+ def test_generate_temp_url_invalid_path(self):
+ with self.assertRaises(ValueError) as exc_manager:
+ u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key,
+ self.method)
+ self.assertEqual(exc_manager.exception.args[0],
+ 'path must be representable as UTF-8')
@mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature")
def test_generate_absolute_expiry_temp_url(self, hmac_mock):
- expected_url = ('/v1/AUTH_account/c/o?'
- 'temp_url_sig=temp_url_signature&'
- 'temp_url_expires=2146636800')
+ if isinstance(self.expected_url, six.binary_type):
+ expected_url = self.expected_url.replace(
+ b'1400003600', b'2146636800')
+ else:
+ expected_url = self.expected_url.replace(
+ u'1400003600', u'2146636800')
url = u.generate_temp_url(self.url, 2146636800, self.key, self.method,
absolute=True)
self.assertEqual(url, expected_url)
def test_generate_temp_url_bad_seconds(self):
- self.assertRaises(TypeError,
- u.generate_temp_url,
- self.url,
- 'not_an_int',
- self.key,
- self.method)
-
- self.assertRaises(ValueError,
- u.generate_temp_url,
- self.url,
- -1,
- self.key,
- self.method)
+ with self.assertRaises(TypeError) as exc_manager:
+ u.generate_temp_url(self.url, 'not_an_int', self.key, self.method)
+ self.assertEqual(exc_manager.exception.args[0],
+ 'seconds must be an integer')
+
+ with self.assertRaises(ValueError) as exc_manager:
+ u.generate_temp_url(self.url, -1, self.key, self.method)
+ self.assertEqual(exc_manager.exception.args[0],
+ 'seconds must be a positive integer')
+
+
+class TestTempURLUnicodePathAndKey(TestTempURL):
+ url = u'/v1/\u00e4/c/\u00f3'
+ key = u'k\u00e9y'
+ expected_url = (u'%s?temp_url_sig=temp_url_signature'
+ u'&temp_url_expires=1400003600') % url
+ expected_body = u'\n'.join([
+ u'GET',
+ u'1400003600',
+ url,
+ ]).encode('utf-8')
+
+
+class TestTempURLUnicodePathBytesKey(TestTempURL):
+ url = u'/v1/\u00e4/c/\u00f3'
+ key = u'k\u00e9y'.encode('utf-8')
+ expected_url = (u'%s?temp_url_sig=temp_url_signature'
+ u'&temp_url_expires=1400003600') % url
+ expected_body = '\n'.join([
+ u'GET',
+ u'1400003600',
+ url,
+ ]).encode('utf-8')
+
+
+class TestTempURLBytesPathUnicodeKey(TestTempURL):
+ url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
+ key = u'k\u00e9y'
+ expected_url = url + (b'?temp_url_sig=temp_url_signature'
+ b'&temp_url_expires=1400003600')
+ expected_body = b'\n'.join([
+ b'GET',
+ b'1400003600',
+ url,
+ ])
+
+
+class TestTempURLBytesPathAndKey(TestTempURL):
+ url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
+ key = u'k\u00e9y'.encode('utf-8')
+ expected_url = url + (b'?temp_url_sig=temp_url_signature'
+ b'&temp_url_expires=1400003600')
+ expected_body = b'\n'.join([
+ b'GET',
+ b'1400003600',
+ url,
+ ])
+
+
+class TestTempURLBytesPathAndNonUtf8Key(TestTempURL):
+ url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8')
+ key = b'k\xffy'
+ expected_url = url + (b'?temp_url_sig=temp_url_signature'
+ b'&temp_url_expires=1400003600')
+ expected_body = b'\n'.join([
+ b'GET',
+ b'1400003600',
+ url,
+ ])
class TestReadableToIterable(unittest.TestCase):
diff --git a/tests/unit/utils.py b/tests/unit/utils.py
index d04583f..c05146e 100644
--- a/tests/unit/utils.py
+++ b/tests/unit/utils.py
@@ -494,6 +494,9 @@ class CaptureOutput(object):
def __eq__(self, other):
return self.out == other
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
def __getattr__(self, name):
return getattr(self.out, name)
View Lorry Controller Status