diff options
| -rw-r--r-- | .mailmap | 2 | ||||
| -rw-r--r-- | AUTHORS | 17 | ||||
| -rw-r--r-- | ChangeLog | 49 | ||||
| -rw-r--r-- | doc/source/cli.rst | 43 | ||||
| -rw-r--r-- | releasenotes/notes/310-notes-03040158a8683dd8.yaml | 38 | ||||
| -rw-r--r-- | swiftclient/client.py | 15 | ||||
| -rw-r--r-- | swiftclient/utils.py | 5 | ||||
| -rw-r--r-- | tests/functional/test_swiftclient.py | 11 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 28 | ||||
| -rw-r--r-- | tests/unit/test_utils.py | 37 |
10 files changed, 234 insertions, 11 deletions
@@ -89,3 +89,5 @@ Doug Hellmann <doug@doughellmann.com> <doug.hellmann@dreamhost.com> Ondrej Novy <ondrej.novy@firma.seznam.cz> James Nzomo <james@tdt.rocks> <kazikubwa@gmail.com> Alessandro Pilotti <ap@pilotti.it> <apilotti@cloudbasesolutions.com> +Marek Kaleta <marek.kaleta@firma.seznam.cz> <Marek.Kaleta@firma.seznam.cz> +Andreas Jaeger <aj@suse.de> <aj@suse.com> @@ -1,9 +1,12 @@ +Paul Belanger (pabelanger@redhat.com) Christian Berendt (berendt@b1-systems.de) Luis de Bethencourt (luis@debethencourt.com) +Hu Bing (hubingsh@cn.ibm.com) Darrell Bishop (darrell@swiftstack.com) Fabien Boucher (fabien.boucher@enovance.com) Chmouel Boudjnah (chmouel@enovance.com) Clark Boylan (clark.boylan@gmail.com) +Cedric Brandily (zzelle@gmail.com) Chris Buccella (chris.buccella@antallagon.com) Tim Burke (tim.burke@gmail.com) Clint Byrum (clint@fewbar.com) @@ -12,13 +15,13 @@ Sergio Cazzolato (sergio.j.cazzolato@intel.com) Mahati Chamarthy (mahati.chamarthy@gmail.com) Chaozhe.Chen (chaozhe.chen@easystack.cn) Ray Chen (oldsharp@163.com) -Li Cheng (shcli@cn.ibm.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) Alistair Coles (alistair.coles@hpe.com) Ian Cordasco (ian.cordasco@rackspace.com) Nick Craig-Wood (nick@craig-wood.com) Thiago da Silva (thiago@redhat.com) Sean Dague (sean@dague.net) +Julien Danjou (julien@danjou.info) Zack M. Davis (zdavis@swiftstack.com) John Dickinson (me@not.mn) EdLeafe (ed@leafe.com) @@ -32,7 +35,9 @@ Anne Gentle (anne@openstack.org) Clay Gerrard (clay.gerrard@gmail.com) David Goetz (david.goetz@rackspace.com) Thomas Goirand (thomas@goirand.fr) +Sergey Gotliv (sgotliv@redhat.com) Davide Guerri (davide.guerri@hp.com) +Shashirekha Gundur (shashirekha.j.gundur@intel.com) Romain Hardouin (romain_hardouin@yahoo.fr) Steven Hardy (shardy@redhat.com) Doug Hellmann (doug@doughellmann.com) @@ -44,12 +49,15 @@ Andreas Jaeger (aj@suse.de) Jude Job (judeopenstack@gmail.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) Leah Klearman (lklrmn@gmail.com) +Marek Kaleta (marek.kaleta@firma.seznam.cz) Jaivish Kothari (jaivish.kothari@nectechnologies.in) Jakub Krajcovic (jakub.krajcovic@gmail.com) David Kranz (david.kranz@qrclab.com) Sushil Kumar (sushil.kumar2@globallogic.com) Greg Lange (greglange@gmail.com) Alexis Lee (lxsli@hpe.com) +Jamie Lennox (jamielennox@gmail.com) +Cheng Li (shcli@cn.ibm.com) Tong Li (litong01@us.ibm.com) Peter Lisak (peter.lisak@firma.seznam.cz) Feng Liu (mefengliu23@gmail.com) @@ -71,6 +79,7 @@ Dirk Mueller (dirk@dmllr.de) Zhenguo Niu (zhenguo@unitedstack.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) James Nzomo (james@tdt.rocks) +Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Alessandro Pilotti (ap@pilotti.it) Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Dan Prince (dprince@redhat.com) @@ -87,18 +96,22 @@ Jeremy Stanley (fungi@yuggoth.org) Victor Stinner (victor.stinner@enovance.com) Jiří Suchomel (jsuchome@suse.cz) YUZAWA Takahiko (yuzawataka@intellilink.co.jp) +Nandini Tata (nandini.tata.15@gmail.com) Monty Taylor (mordred@inaugust.com) TheSriram (sriram@klusterkloud.com) Tihomir Trifonov (t.trifonov@gmail.com) Dean Troyer (dtroyer@gmail.com) +Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Stanislav Vitkovskiy (stas.vitkovsky@gmail.com) Daniel Wakefield (daniel.wakefield@hp.com) Shane Wang (shane.wang@intel.com) Mark Washenberger (mark.washenberger@rackspace.com) +Andrew Welleck (awellec@us.ibm.com) Wu Wenxiang (wu.wenxiang@99cloud.net) Mike Widman (mwidman@endurancewindpower.com) Joel Wright (joel.wright@sohonet.com) You Yamagata (bi.yamagata@gmail.com) +zheng yin (yin.zheng@easystack.cn) Qiu Yu (qiuyu@ebaysf.com) YangLei (yanglyy@cn.ibm.com) Pete Zaitcev (zaitcev@kotori.zaitcev.us) @@ -112,3 +125,5 @@ zhang-jinnan (ben.os@99cloud.net) hgangwx (hgangwx@cn.ibm.com) shu-mutou (shu-mutou@rf.jp.nec.com) SaiKiran (saikiranveeravarapu@gmail.com) +venkatamahesh (venkatamaheshkotha@gmail.com) +yuyafei (yu.yafei@zte.com.cn) @@ -1,3 +1,52 @@ +3.1.0 +----- + +* Added a copy object method. + +* Arbitrary query strings can now be passed into container functions. + +* Client certificate and key can now be specified via CLI + options (--os-cert/--os-key) or environment variables ($OS_CERT/$OS_KEY). + +* A new CLI option `--ignore-checksum` can be specified to turn off + checksum validation. In the SDK, the new `checksum=True` parameter can + be used for the same purpose. + +* Added --json option to `swift capabilities` / `swift info` + +* Default to v3 auth if we find a (user|project)-domain-(name|id) option. + +* Added a Python version constraint of >= Py27. + +* `client.py` will now retry on a 401 (auth error) even if `retries` is + set to zero. + +* Fixed `swift download` when `marker` was specified. + +* Object segments uploaded via swiftclient are now given the content type + "application/swiftclient-segment". + +* "Directory marker" objects are now given a "application/directory" + content type to match both Swift's `staticweb` feature and other + ecosystem tools. + +* Strip leading/trailing whitespace from headers (otherwise, new versions + of the requests library will raise an InvalidHeader error). Additionally, + header values with standard types (integer, float, or bool) are coerced + to strings before being sent to a socket. + +* Non-python dependencies are now specified in bindep.txt. Currently this + only lists a single dependency for testing (PyPy), but if future + dependencies are added, they will be included in this file. + +* Client exceptions now include response headers. One benefit is that + this allows clients to see transaction IDs without needing to turn on + debug logging. + +* Client connections now accept gzip-encoded responses. + +* Various other minor bug fixes and improvements. + 3.0.0 ----- diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 9bb229d..87020c9 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -225,6 +225,28 @@ Capabilities the ``proxy-url`` option is not provided, the storage URL retrieved after authentication is used as ``proxy-url``. +Tempurl +------- + + ``tempurl [method] [seconds] [path] [key]`` + + Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to + allow for this temporary URL that is usually 'GET' or 'PUT'. ``seconds`` option sets + the amount of time in seconds the temporary URL will be valid for; or, if ``--absolute`` + is passed, the Unix timestamp when the temporary URL will expire. ``path`` option sets + the full path to the Swift object. Example: ``/v1/AUTH_account/c/o``. ``key`` option is + the secret temporary URL key set on the Swift cluster. To set a key, run + ``swift post -m "Temp-URL-Key: <your secret key>"``. + +Auth +---- + + ``auth`` + + Display authentication variables in shell friendly format. Command to run to export storage + url and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. + Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic + authentication: ``swift auth -v -U test:tester -K testing``. Examples ~~~~~~~~ @@ -272,6 +294,15 @@ List the contents of a container: testSwift.txt +Display auth related authentication variables in shell friendly format: + +.. code-block:: bash + + > swift auth + + export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 + export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 + Download an object from a container: .. code-block:: bash @@ -347,3 +378,15 @@ For more information on large objects, see the documentation `here myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 + +Firstly, the key should be set, then generate a temporary URL for a Swift object: + +.. code-block:: bash + + > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" + + > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ + /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 + + /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ + 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/releasenotes/notes/310-notes-03040158a8683dd8.yaml b/releasenotes/notes/310-notes-03040158a8683dd8.yaml new file mode 100644 index 0000000..792b6f5 --- /dev/null +++ b/releasenotes/notes/310-notes-03040158a8683dd8.yaml @@ -0,0 +1,38 @@ +--- +features: + - Added a copy object method. + - Arbitrary query strings can now be passed into container functions. + - > + Client certificate and key can now be specified via CLI + options (--os-cert/--os-key) or environment variables ($OS_CERT/$OS_KEY). + - > + A new CLI option `--ignore-checksum` can be specified to turn off + checksum validation. In the SDK, the new `checksum=True` parameter can + be used for the same purpose. + - Added --json option to `swift capabilities` / `swift info` + - Default to v3 auth if we find a (user|project)-domain-(name|id) option. + - Added a Python version constraint of >= Py27. + - > + `client.py` will now retry on a 401 (auth error) even if `retries` is + set to zero. + - Fixed `swift download` when `marker` was specified. + - Object segments uploaded via swiftclient are now given the content type + "application/swiftclient-segment". + - > + "Directory marker" objects are now given a "application/directory" + content type to match both Swift's `staticweb` feature and other + ecosystem tools. + - > + Strip leading/trailing whitespace from headers (otherwise, new versions + of the requests library will raise an InvalidHeader error). Additionally, + header values with standard types (integer, float, or bool) are coerced + to strings before being sent to a socket. + - > + Non-python dependencies are now specified in bindep.txt. Currently this + only lists a single dependency for testing (PyPy), but if future + dependencies are added, they will be included in this file. + - Client exceptions now include response headers. One benefit is that + this allows clients to see transaction IDs without needing to turn on + debug logging. + - Client connections now accept gzip-encoded responses. + - Various other minor bug fixes and improvements. diff --git a/swiftclient/client.py b/swiftclient/client.py index 988c7d9..ee5a838 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -211,6 +211,13 @@ def quote(value, safe='/'): def encode_utf8(value): + if type(value) in six.integer_types + (float, bool): + # As of requests 2.11.0, headers must be byte- or unicode-strings. + # Convert some known-good types as a convenience for developers. + # Note that we *don't* convert subclasses, as they may have overriddden + # __str__ or __repr__. + # See https://github.com/kennethreitz/requests/pull/3366 for more info + value = str(value) if isinstance(value, six.text_type): value = value.encode('utf8') return value @@ -732,7 +739,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) - headers = {'X-Auth-Token': token} + headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: headers['X-Service-Token'] = service_token method = 'GET' @@ -859,6 +866,7 @@ def get_container(url, token, container, marker=None, limit=None, else: headers = {} headers['X-Auth-Token'] = token + headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -1457,10 +1465,11 @@ def get_capabilities(http_conn): :raises ClientException: HTTP Capabilities GET failed """ parsed, conn = http_conn - conn.request('GET', parsed.path, '') + headers = {'Accept-Encoding': 'gzip'} + conn.request('GET', parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body) + http_log((parsed.geturl(), 'GET',), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response( resp, 'Capabilities GET failed', body) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 10687bf..d394283 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" +import gzip import hashlib import hmac import json @@ -120,6 +121,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False): def parse_api_response(headers, body): + if headers.get('content-encoding') == 'gzip': + with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz: + body = gz.read() + charset = 'utf-8' # Swift *should* be speaking UTF-8, but check content-type just in case content_type = headers.get('content-type', '') diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 6e19abd..0e6a346 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -400,10 +400,19 @@ class TestFunctional(unittest.TestCase): def test_post_object(self): self.conn.post_object(self.containername, self.objectname, - {'x-object-meta-color': 'Something'}) + {'x-object-meta-color': 'Something', + 'x-object-meta-uni': b'\xd8\xaa'.decode('utf8'), + 'x-object-meta-int': 123, + 'x-object-meta-float': 45.67, + 'x-object-meta-bool': False}) headers = self.conn.head_object(self.containername, self.objectname) self.assertEqual('Something', headers.get('x-object-meta-color')) + self.assertEqual(b'\xd8\xaa'.decode('utf-8'), + headers.get('x-object-meta-uni')) + self.assertEqual('123', headers.get('x-object-meta-int')) + self.assertEqual('45.67', headers.get('x-object-meta-float')) + self.assertEqual('False', headers.get('x-object-meta-bool')) def test_copy_object(self): self.conn.put_object( diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 0095447..7d0dd9a 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -185,9 +185,9 @@ class TestHttpHelpers(MockHttpTest): def test_encode_meta_headers(self): headers = {'abc': '123', - u'x-container-meta-\u0394': '123', - u'x-account-meta-\u0394': '123', - u'x-object-meta-\u0394': '123'} + u'x-container-meta-\u0394': 123, + u'x-account-meta-\u0394': 12.3, + u'x-object-meta-\u0394': True} r = swiftclient.encode_meta_headers(headers) @@ -199,6 +199,7 @@ class TestHttpHelpers(MockHttpTest): for k, v in r.items(): self.assertIs(type(k), binary_type) self.assertIs(type(v), binary_type) + self.assertIn(v, (b'123', b'12.3', b'True')) def test_set_user_agent_default(self): _junk, conn = c.http_connection('http://www.example.com') @@ -581,6 +582,7 @@ class TestGetAccount(MockHttpTest): self.assertEqual(value, []) self.assertRequests([ ('GET', '/v1/acct?format=json', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -591,6 +593,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') self.assertRequests([ ('GET', '/v1/acct?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -601,6 +604,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) self.assertRequests([ ('GET', '/v1/acct?format=json&limit=10', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -611,6 +615,7 @@ class TestGetAccount(MockHttpTest): c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') self.assertRequests([ ('GET', '/v1/acct?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -622,6 +627,7 @@ class TestGetAccount(MockHttpTest): end_marker='end_marker') self.assertRequests([ ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -700,6 +706,7 @@ class TestGetContainer(MockHttpTest): self.assertEqual(value, []) self.assertRequests([ ('GET', '/v1/acct/container?format=json', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -711,6 +718,7 @@ class TestGetContainer(MockHttpTest): marker='marker') self.assertRequests([ ('GET', '/v1/acct/container?format=json&marker=marker', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -722,6 +730,7 @@ class TestGetContainer(MockHttpTest): limit=10) self.assertRequests([ ('GET', '/v1/acct/container?format=json&limit=10', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -733,6 +742,7 @@ class TestGetContainer(MockHttpTest): prefix='asdf/') self.assertRequests([ ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -744,6 +754,7 @@ class TestGetContainer(MockHttpTest): delimiter='/') self.assertRequests([ ('GET', '/v1/acct/container?format=json&delimiter=/', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -755,7 +766,7 @@ class TestGetContainer(MockHttpTest): end_marker='end_marker') self.assertRequests([ ('GET', '/v1/acct/container?format=json&end_marker=end_marker', - '', {'x-auth-token': 'token'}), + '', {'x-auth-token': 'token', 'accept-encoding': 'gzip'}), ]) def test_param_path(self): @@ -766,6 +777,7 @@ class TestGetContainer(MockHttpTest): path='asdf') self.assertRequests([ ('GET', '/v1/acct/container?format=json&path=asdf', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'token'}), ]) @@ -780,6 +792,7 @@ class TestGetContainer(MockHttpTest): ('GET', '/container?format=json', '', { 'x-auth-token': 'TOKEN', 'x-client-key': 'client key', + 'accept-encoding': 'gzip', }), ]) @@ -790,6 +803,7 @@ class TestGetContainer(MockHttpTest): query_string="hello=20") self.assertRequests([ ('GET', '/asdf?format=json&hello=20', '', { + 'accept-encoding': 'gzip', 'x-auth-token': 'asdf'}), ]) @@ -1583,7 +1597,7 @@ class TestGetCapabilities(MockHttpTest): http_conn = conn('http://www.test.com/info') info = c.get_capabilities(http_conn) self.assertRequests([ - ('GET', '/info', '', {}), + ('GET', '/info', '', {'Accept-Encoding': 'gzip'}), ]) self.assertEqual(info, {}) self.assertTrue(http_conn[1].resp.has_been_read) @@ -1619,7 +1633,8 @@ class TestGetCapabilities(MockHttpTest): ('GET', '/auth/v1.0', '', { 'x-auth-user': 'user', 'x-auth-key': 'key'}), - ('GET', 'http://storage.example.com/info', '', {}), + ('GET', 'http://storage.example.com/info', '', { + 'accept-encoding': 'gzip'}), ]) def test_conn_get_capabilities_with_os_auth(self): @@ -2341,6 +2356,7 @@ class TestConnection(MockHttpTest): ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', + 'accept-encoding': 'gzip', }), ]) self.assertEqual(conn.attempts, 1) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0f210a3..f0de79c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import gzip import unittest import mock import six @@ -394,3 +395,39 @@ class TestGroupers(unittest.TestCase): result = list(u.n_groups(range(100), 12)) self.assertEqual([9] * 11 + [1], list(map(len, result))) + + +class TestApiResponeParser(unittest.TestCase): + + def test_utf8_default(self): + result = u.parse_api_response( + {}, u'{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_bad_json(self): + self.assertRaises(ValueError, u.parse_api_response, + {}, b'{"foo": "bar}') + + def test_bad_utf8(self): + self.assertRaises(UnicodeDecodeError, u.parse_api_response, + {}, b'{"foo": "b\xffr"}') + + def test_latin_1(self): + result = u.parse_api_response( + {'content-type': 'application/json; charset=iso8859-1'}, + b'{"t\xe9st": "\xff"}') + self.assertEqual({u't\xe9st': u'\xff'}, result) + + def test_gzipped_utf8(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) |