summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--AUTHORS3
-rw-r--r--ChangeLog12
-rw-r--r--doc/requirements.txt2
-rw-r--r--doc/source/cli/index.rst2
-rw-r--r--doc/source/conf.py9
-rw-r--r--lower-constraints.txt4
-rw-r--r--releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml9
-rw-r--r--releasenotes/source/conf.py12
-rw-r--r--swiftclient/client.py6
-rw-r--r--swiftclient/service.py6
-rwxr-xr-xswiftclient/shell.py21
-rw-r--r--swiftclient/utils.py22
-rw-r--r--tests/functional/test_swiftclient.py17
-rw-r--r--tests/unit/test_service.py15
-rw-r--r--tests/unit/test_shell.py49
-rw-r--r--tests/unit/test_utils.py39
-rw-r--r--tests/unit/utils.py16
17 files changed, 204 insertions, 40 deletions
diff --git a/AUTHORS b/AUTHORS
index 1fcf65d..16dfcf9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -52,6 +52,7 @@ Hiroshi Miura (miurahr@nttdata.co.jp)
howardlee (lihongweibj@inspur.com)
Hu Bing (hubingsh@cn.ibm.com)
Ian Cordasco (ian.cordasco@rackspace.com)
+jacky06 (zhang.min@99cloud.net)
Jaivish Kothari (jaivish.kothari@nectechnologies.in)
Jakub Krajcovic (jakub.krajcovic@gmail.com)
James Nzomo (james@tdt.rocks)
@@ -99,6 +100,7 @@ Ondrej Novy (ondrej.novy@firma.seznam.cz)
Pallavi (pallavi.s@nectechnologies.in)
Paul Belanger (pabelanger@redhat.com)
Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br)
+pengyuesheng (pengyuesheng@gohighsec.com)
Pete Zaitcev (zaitcev@kotori.zaitcev.us)
Peter Lisak (peter.lisak@firma.seznam.cz)
Petr Kovar (pkovar@redhat.com)
@@ -147,6 +149,7 @@ Vitaly Gridnev (vgridnev@mirantis.com)
Vu Cong Tuan (tuanvc@vn.fujitsu.com)
wangqi (wang.qi@99cloud.net)
wangxiyuan (wangxiyuan@huawei.com)
+wangzhenyu (wangzy@fiberhome.com)
Wu Wenxiang (wu.wenxiang@99cloud.net)
wu.chunyang (wu.chunyang@99cloud.net)
YangLei (yanglyy@cn.ibm.com)
diff --git a/ChangeLog b/ChangeLog
index 6f6bf8b..253a3cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+3.8.0
+-----
+
+* Added a new `--json` option to `swift list`.
+
+* Fixed an issue introduced in 3.5.0 where re-uploading an SLO with
+ the same size, mtime, and segment size would delete all of the
+ just-uploaded segments.
+
+* Various other minor bug fixes and improvements.
+
+
3.7.0
-----
diff --git a/doc/requirements.txt b/doc/requirements.txt
index d8f432e..6cdad2a 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -2,4 +2,4 @@ keystoneauth1>=3.4.0 # Apache-2.0
sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD
sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD
reno>=2.5.0 # Apache-2.0
-openstackdocstheme>=1.18.1 # Apache-2.0
+openstackdocstheme>=1.20.0 # Apache-2.0
diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst
index d6841d5..1762989 100644
--- a/doc/source/cli/index.rst
+++ b/doc/source/cli/index.rst
@@ -443,7 +443,7 @@ swift post
.. code-block:: console
- Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to]
+ Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>]
[--sync-key <sync-key>] [--meta <name:value>]
[--header <header>]
[<container> [<object>]]
diff --git a/doc/source/conf.py b/doc/source/conf.py
index f56b643..85dd81e 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -53,17 +53,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
-project = u'Swiftclient'
copyright = u'2013-2016 OpenStack, LLC.'
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-import swiftclient.version
-release = swiftclient.version.version_string
-version = swiftclient.version.version_string
-
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None
diff --git a/lower-constraints.txt b/lower-constraints.txt
index ab45e39..88d2865 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -20,11 +20,11 @@ MarkupSafe==1.0
mccabe==0.2.1
mock==1.2.0
netaddr==0.7.10
-openstackdocstheme==1.18.1
+openstackdocstheme==1.20.0
oslo.config==1.2.0
pbr==2.0.0
pep8==1.5.7
-PrettyTable==0.7
+PrettyTable==0.7.1
pyflakes==0.8.1
Pygments==2.2.0
python-keystoneclient==0.7.0
diff --git a/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml b/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml
new file mode 100644
index 0000000..85ae2c0
--- /dev/null
+++ b/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml
@@ -0,0 +1,9 @@
+---
+features:
+ - |
+ Added a new ``--json`` option to ``swift list``.
+fixes:
+ - |
+ Fixed an issue introduced in 3.5.0 where re-uploading an SLO with
+ the same size, mtime, and segment size would delete all of the
+ just-uploaded segments.
diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py
index b27aa96..c71f41d 100644
--- a/releasenotes/source/conf.py
+++ b/releasenotes/source/conf.py
@@ -65,15 +65,8 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
-project = u'Swift Client Release Notes'
copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year
-# Release notes are version independent.
-# The short X.Y version.
-version = ''
-# The full version, including alpha/beta/rc tags.
-release = ''
-
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
@@ -173,11 +166,6 @@ html_theme = 'openstackdocs'
#
# html_extra_path = []
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-# html_last_updated_fmt = '%b %d, %Y'
-html_last_updated_fmt = '%Y-%m-%d %H:%M'
-
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#
diff --git a/swiftclient/client.py b/swiftclient/client.py
index f071182..4be2e2d 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -74,8 +74,10 @@ except ImportError:
pass
# requests version 1.2.3 try to encode headers in ascii, preventing
-# utf-8 encoded header to be 'prepared'
-if StrictVersion(requests.__version__) < StrictVersion('2.0.0'):
+# utf-8 encoded header to be 'prepared'. This also affects all
+# (or at least most) versions of requests on py3
+if StrictVersion(requests.__version__) < StrictVersion('2.0.0') \
+ or not six.PY2:
from requests.structures import CaseInsensitiveDict
def prepare_unicode_headers(self, headers):
diff --git a/swiftclient/service.py b/swiftclient/service.py
index 06de091..5292dc5 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -2070,7 +2070,8 @@ class SwiftService(object):
'status': 'skipped-changed'
})
return res
- if not options['leave_segments']:
+ if not options['leave_segments'] and not headers.get(
+ 'content-location'):
old_manifest = headers.get('x-object-manifest')
if is_slo:
old_slo_manifest_paths.extend(
@@ -2515,7 +2516,8 @@ class SwiftService(object):
if not options['leave_segments']:
try:
headers = conn.head_object(container, obj,
- headers=_headers)
+ headers=_headers,
+ query_string='symlink=get')
old_manifest = headers.get('x-object-manifest')
if config_true_value(headers.get('x-static-large-object')):
query_string = 'multipart-manifest=delete'
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index 0459533..cc4f325 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -33,7 +33,8 @@ from sys import argv as sys_argv, exit, stderr, stdin
from time import gmtime, strftime
from swiftclient import RequestException
-from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes
+from swiftclient.utils import config_true_value, generate_temp_url, \
+ prt_bytes, JSONableIterable
from swiftclient.multithreading import OutputManager
from swiftclient.exceptions import ClientException
from swiftclient import __version__ as client_version
@@ -578,6 +579,8 @@ def st_list(parser, args, output_manager, return_parser=False):
help='Roll up items with the given delimiter. For containers '
'only. See OpenStack Swift API documentation for '
'what this means.')
+ parser.add_argument('-j', '--json', action='store_true',
+ help='print listing information in json')
parser.add_argument(
'-H', '--header', action='append', dest='header',
default=[],
@@ -616,6 +619,20 @@ def st_list(parser, args, output_manager, return_parser=False):
else:
stats_parts_gen = swift.list(container=container)
+ if options.get('json', False):
+ def listing(stats_parts_gen=stats_parts_gen):
+ for stats in stats_parts_gen:
+ if stats["success"]:
+ for item in stats['listing']:
+ yield item
+ else:
+ raise stats["error"]
+
+ json.dump(
+ JSONableIterable(listing()), output_manager.print_stream,
+ sort_keys=True, indent=2)
+ output_manager.print_msg('')
+ return
for stats in stats_parts_gen:
if stats["success"]:
_print_stats(options, stats, human)
@@ -709,7 +726,7 @@ def st_stat(parser, args, output_manager, return_parser=False):
output_manager.error(e.value)
-st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to]
+st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>]
[--sync-key <sync-key>] [--meta <name:value>]
[--header <header>]
[<container> [<object>]]
diff --git a/swiftclient/utils.py b/swiftclient/utils.py
index 2b208b9..9e43237 100644
--- a/swiftclient/utils.py
+++ b/swiftclient/utils.py
@@ -403,3 +403,25 @@ def normalize_manifest_path(path):
if path.startswith('/'):
return path[1:]
return path
+
+
+class JSONableIterable(list):
+ def __init__(self, iterable):
+ self._iterable = iter(iterable)
+ try:
+ self._peeked = next(self._iterable)
+ self._has_items = True
+ except StopIteration:
+ self._peeked = None
+ self._has_items = False
+
+ def __bool__(self):
+ return self._has_items
+
+ __nonzero__ = __bool__
+
+ def __iter__(self):
+ if self._has_items:
+ yield self._peeked
+ for item in self._iterable:
+ yield item
diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py
index bae3044..9a74c63 100644
--- a/tests/functional/test_swiftclient.py
+++ b/tests/functional/test_swiftclient.py
@@ -18,6 +18,7 @@ import unittest
import time
from io import BytesIO
+import six
from six.moves import configparser
import swiftclient
@@ -446,6 +447,22 @@ class TestFunctional(unittest.TestCase):
self.assertEqual('45.67', headers.get('x-object-meta-float'))
self.assertEqual('False', headers.get('x-object-meta-bool'))
+ def test_post_object_unicode_header_name(self):
+ self.conn.post_object(self.containername,
+ self.objectname,
+ {u'x-object-meta-\U0001f44d': u'\U0001f44d'})
+
+ # Note that we can't actually read this header back on py3; see
+ # https://bugs.python.org/issue37093
+ # We'll have to settle for just testing that the POST doesn't blow up
+ # with a UnicodeDecodeError
+ if six.PY2:
+ headers = self.conn.head_object(
+ self.containername, self.objectname)
+ self.assertIn(u'x-object-meta-\U0001f44d', headers)
+ self.assertEqual(u'\U0001f44d',
+ headers.get(u'x-object-meta-\U0001f44d'))
+
def test_copy_object(self):
self.conn.put_object(
self.containername, self.objectname, self.test_data)
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index 12fbaa0..b760352 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -312,8 +312,8 @@ class TestServiceDelete(_TestServiceBase):
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
- mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
- headers={})
+ mock_conn.head_object.assert_called_once_with(
+ 'test_c', 'test_o', query_string='symlink=get', headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
headers={}
@@ -335,7 +335,8 @@ class TestServiceDelete(_TestServiceBase):
r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q)
mock_conn.head_object.assert_called_once_with(
- 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'})
+ 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'},
+ query_string='symlink=get')
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
headers={'Skip-Middleware': 'Test'}
@@ -362,8 +363,8 @@ class TestServiceDelete(_TestServiceBase):
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
after = time.time()
- mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
- headers={})
+ mock_conn.head_object.assert_called_once_with(
+ 'test_c', 'test_o', query_string='symlink=get', headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o', query_string=None, response_dict={},
headers={}
@@ -389,8 +390,8 @@ class TestServiceDelete(_TestServiceBase):
s = SwiftService()
r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q)
- mock_conn.head_object.assert_called_once_with('test_c', 'test_o',
- headers={})
+ mock_conn.head_object.assert_called_once_with(
+ 'test_c', 'test_o', query_string='symlink=get', headers={})
mock_conn.delete_object.assert_called_once_with(
'test_c', 'test_o',
query_string='multipart-manifest=delete',
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index f729c25..c972281 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -298,6 +298,26 @@ class TestShell(unittest.TestCase):
headers={'Skip-Middleware': 'Test'})])
@mock.patch('swiftclient.service.Connection')
+ def test_list_json(self, connection):
+ connection.return_value.get_account.side_effect = [
+ [None, [{'name': 'container'}]],
+ [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]],
+ [None, []],
+ ]
+
+ argv = ["", "list", "--json"]
+ with CaptureOutput(suppress_systemexit=True) as output:
+ swiftclient.shell.main(argv)
+ calls = [mock.call(marker='', prefix=None, headers={}),
+ mock.call(marker='container', prefix=None, headers={})]
+ connection.return_value.get_account.assert_has_calls(calls)
+
+ listing = [{'name': 'container'},
+ {'name': u'\u263A', 'some-custom-key': 'and value'}]
+ expected = json.dumps(listing, sort_keys=True, indent=2) + '\n'
+ self.assertEqual(output.out, expected)
+
+ @mock.patch('swiftclient.service.Connection')
def test_list_account(self, connection):
# Test account listing
connection.return_value.get_account.side_effect = [
@@ -813,6 +833,35 @@ class TestShell(unittest.TestCase):
)
@mock.patch('swiftclient.service.Connection')
+ def test_upload_over_symlink_to_slo(self, connection):
+ # Upload delete existing segments
+ connection.return_value.head_container.return_value = {
+ 'x-storage-policy': 'one'}
+ connection.return_value.attempts = 0
+ connection.return_value.head_object.side_effect = [
+ {'x-static-large-object': 'true',
+ 'content-location': '/v1/a/c/manifest',
+ 'content-length': '2'},
+ ]
+ connection.return_value.get_object.return_value = (
+ {'content-location': '/v1/a/c/manifest'},
+ b'[{"name": "container1/old_seg1"},'
+ b' {"name": "container2/old_seg2"}]'
+ )
+ connection.return_value.put_object.return_value = EMPTY_ETAG
+ connection.return_value.delete_object.return_value = None
+ argv = ["", "upload", "container", self.tmpfile]
+ swiftclient.shell.main(argv)
+ connection.return_value.put_object.assert_called_with(
+ 'container',
+ self.tmpfile.lstrip('/'),
+ mock.ANY,
+ content_length=0,
+ headers={'x-object-meta-mtime': mock.ANY},
+ response_dict={})
+ self.assertEqual([], connection.return_value.delete_object.mock_calls)
+
+ @mock.patch('swiftclient.service.Connection')
def test_upload_leave_slo_segments(self, connection):
# Test upload overwriting a manifest respects --leave-segments
connection.return_value.head_container.return_value = {
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index e54b90c..97abc44 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -14,6 +14,7 @@
# limitations under the License.
import gzip
+import json
import unittest
import mock
import six
@@ -638,3 +639,41 @@ class TestGetBody(unittest.TestCase):
{'content-encoding': 'gzip'},
buf.getvalue())
self.assertEqual({'test': u'\u2603'}, result)
+
+
+class JSONTracker(object):
+ def __init__(self, data):
+ self.data = data
+ self.calls = []
+
+ def __iter__(self):
+ for item in self.data:
+ self.calls.append(('read', item))
+ yield item
+
+ def write(self, s):
+ self.calls.append(('write', s))
+
+
+class TestJSONableIterable(unittest.TestCase):
+ def test_json_dump_iterencodes(self):
+ t = JSONTracker([1, 'fish', 2, 'fish'])
+ json.dump(u.JSONableIterable(t), t)
+ self.assertEqual(t.calls, [
+ ('read', 1),
+ ('write', '[1'),
+ ('read', 'fish'),
+ ('write', ', "fish"'),
+ ('read', 2),
+ ('write', ', 2'),
+ ('read', 'fish'),
+ ('write', ', "fish"'),
+ ('write', ']'),
+ ])
+
+ def test_json_dump_empty_iter(self):
+ t = JSONTracker([])
+ json.dump(u.JSONableIterable(t), t)
+ self.assertEqual(t.calls, [
+ ('write', '[]'),
+ ])
diff --git a/tests/unit/utils.py b/tests/unit/utils.py
index 8081501..025a234 100644
--- a/tests/unit/utils.py
+++ b/tests/unit/utils.py
@@ -20,6 +20,7 @@ from time import sleep
import unittest
import mock
import six
+import os
from six.moves import reload_module
from six.moves.urllib.parse import urlparse, ParseResult
from swiftclient import client as c
@@ -212,7 +213,19 @@ class MockHttpTest(unittest.TestCase):
# won't cover the references to sys.stdout/sys.stderr in
# swiftclient.multithreading
self.capture_output = CaptureOutput()
- self.capture_output.__enter__()
+ if 'SWIFTCLIENT_DEBUG' not in os.environ:
+ self.capture_output.__enter__()
+ self.addCleanup(self.capture_output.__exit__)
+
+ # since we're going to steal all stderr output globally; we should
+ # give the developer an escape hatch or risk scorn
+ def blowup_but_with_the_helpful(*args, **kwargs):
+ raise Exception(
+ "You tried to enter a debugger while stderr is "
+ "patched, you need to set SWIFTCLIENT_DEBUG=1 "
+ "and try again")
+ import pdb
+ pdb.set_trace = blowup_but_with_the_helpful
def fake_http_connection(*args, **kwargs):
self.validateMockedRequestsConsumed()
@@ -391,7 +404,6 @@ class MockHttpTest(unittest.TestCase):
# un-hygienic mocking on the swiftclient.client module; which may lead
# to some unfortunate test order dependency bugs by way of the broken
# window theory if any other modules are similarly patched
- self.capture_output.__exit__()
reload_module(c)
View Lorry Controller Status