diff options
| -rw-r--r-- | .mailmap | 3 | ||||
| -rw-r--r-- | AUTHORS | 228 | ||||
| -rw-r--r-- | ChangeLog | 28 | ||||
| -rw-r--r-- | README.rst | 9 | ||||
| -rw-r--r-- | bindep.txt | 1 | ||||
| -rw-r--r-- | doc/Makefile | 4 | ||||
| -rw-r--r-- | doc/manpages/swift.1 | 6 | ||||
| -rw-r--r-- | doc/source/cli.rst | 39 | ||||
| -rw-r--r-- | releasenotes/notes/320_notes-bb367dba1053d34c.yaml | 12 | ||||
| -rw-r--r-- | swiftclient/client.py | 7 | ||||
| -rw-r--r-- | swiftclient/service.py | 8 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 35 | ||||
| -rw-r--r-- | swiftclient/utils.py | 53 | ||||
| -rw-r--r-- | tests/functional/test_swiftclient.py | 35 | ||||
| -rw-r--r-- | tests/unit/test_authv1.py | 2 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 19 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 40 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 44 | ||||
| -rw-r--r-- | tests/unit/test_utils.py | 53 | ||||
| -rwxr-xr-x | tools/tox_install.sh | 31 | ||||
| -rw-r--r-- | tox.ini | 13 |
21 files changed, 497 insertions, 173 deletions
@@ -91,3 +91,6 @@ James Nzomo <james@tdt.rocks> <kazikubwa@gmail.com> Alessandro Pilotti <ap@pilotti.it> <apilotti@cloudbasesolutions.com> Marek Kaleta <marek.kaleta@firma.seznam.cz> <Marek.Kaleta@firma.seznam.cz> Andreas Jaeger <aj@suse.de> <aj@suse.com> +Shashi Kant <shashi.kant@nectechnologies.in> +Nandini Tata <nandini.tata@intel.com> <nandini.tata.15@gmail.com> +Flavio Percoco <flaper87@gmail.com> @@ -1,129 +1,143 @@ -Paul Belanger (pabelanger@redhat.com) -Christian Berendt (berendt@b1-systems.de) -Luis de Bethencourt (luis@debethencourt.com) -Hu Bing (hubingsh@cn.ibm.com) -Darrell Bishop (darrell@swiftstack.com) -Fabien Boucher (fabien.boucher@enovance.com) -Chmouel Boudjnah (chmouel@enovance.com) -Clark Boylan (clark.boylan@gmail.com) -Cedric Brandily (zzelle@gmail.com) -Chris Buccella (chris.buccella@antallagon.com) -Tim Burke (tim.burke@gmail.com) -Clint Byrum (clint@fewbar.com) -Tristan Cacqueray (tristan.cacqueray@enovance.com) -Sergio Cazzolato (sergio.j.cazzolato@intel.com) -Mahati Chamarthy (mahati.chamarthy@gmail.com) -Chaozhe.Chen (chaozhe.chen@easystack.cn) -Ray Chen (oldsharp@163.com) -Taurus Cheung (Taurus.Cheung@harmonicinc.com) -Alistair Coles (alistair.coles@hpe.com) -Ian Cordasco (ian.cordasco@rackspace.com) -Nick Craig-Wood (nick@craig-wood.com) -Thiago da Silva (thiago@redhat.com) -Sean Dague (sean@dague.net) -Julien Danjou (julien@danjou.info) -Zack M. Davis (zdavis@swiftstack.com) -John Dickinson (me@not.mn) -EdLeafe (ed@leafe.com) -Sahid Orentino Ferdjaoui (sahid.ferdjaoui@cloudwatt.com) -Flaper Fesp (flaper87@gmail.com) -Florent Flament (florent.flament-ext@cloudwatt.com) -Josh Gachnang (josh@pcsforeducation.com) +Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) -Martin Geisler (martin@geisler.net) +Alexandra Settle (alexandra.settle@rackspace.com) +Alexis Lee (lxsli@hpe.com) +Alistair Coles (alistair.coles@hpe.com) +Andreas Jaeger (aj@suse.de) +Andrew Welleck (awellec@us.ibm.com) +Andy McCrae (andy.mccrae@gmail.com) +Anh Tran (anhtt@vn.fujitsu.com) Anne Gentle (anne@openstack.org) +Ben McCann (ben@benmccann.com) +Cedric Brandily (zzelle@gmail.com) +Chaozhe.Chen (chaozhe.chen@easystack.cn) +Charles Hsu (charles0126@gmail.com) +Cheng Li (shcli@cn.ibm.com) +Chmouel Boudjnah (chmouel@enovance.com) +Chris Buccella (chris.buccella@antallagon.com) +Christian Berendt (berendt@b1-systems.de) +Christian Schwede (cschwede@redhat.com) +Christopher Bartz (bartz@dkrz.de) +Chuck Short (chuck.short@canonical.com) +Clark Boylan (clark.boylan@gmail.com) +Claudiu Belu (cbelu@cloudbasesolutions.com) Clay Gerrard (clay.gerrard@gmail.com) +Clint Byrum (clint@fewbar.com) +Dan Prince (dprince@redhat.com) +Daniel Wakefield (daniel.wakefield@hp.com) +Darrell Bishop (darrell@swiftstack.com) David Goetz (david.goetz@rackspace.com) -Thomas Goirand (thomas@goirand.fr) -Sergey Gotliv (sgotliv@redhat.com) +David Kranz (david.kranz@qrclab.com) +David Shrewsbury (shrewsbury.dave@gmail.com) Davide Guerri (davide.guerri@hp.com) -Shashirekha Gundur (shashirekha.j.gundur@intel.com) -Romain Hardouin (romain_hardouin@yahoo.fr) -Steven Hardy (shardy@redhat.com) +Dean Troyer (dtroyer@gmail.com) +Dirk Mueller (dirk@dmllr.de) +Donagh McCabe (donagh.mccabe@hpe.com) Doug Hellmann (doug@doughellmann.com) +EdLeafe (ed@leafe.com) +Fabien Boucher (fabien.boucher@enovance.com) +Feng Liu (mefengliu23@gmail.com) +Flavio Percoco (flaper87@gmail.com) +Florent Flament (florent.flament-ext@cloudwatt.com) Greg Holt (gholt@rackspace.com) -Charles Hsu (charles0126@gmail.com) -Kun Huang (gareth@unitedstack.com) -Matthieu Huin (mhu@enovance.com) -Andreas Jaeger (aj@suse.de) -Jude Job (judeopenstack@gmail.com) -Vasyl Khomenko (vasiliyk@yahoo-inc.com) -Leah Klearman (lklrmn@gmail.com) -Marek Kaleta (marek.kaleta@firma.seznam.cz) +Greg Lange (greglange@gmail.com) +groqez (groqez@yopmail.net) +Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +hgangwx (hgangwx@cn.ibm.com) +Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) +Hiroshi Miura (miurahr@nttdata.co.jp) +howardlee (lihongweibj@inspur.com) +Hu Bing (hubingsh@cn.ibm.com) +Ian Cordasco (ian.cordasco@rackspace.com) Jaivish Kothari (jaivish.kothari@nectechnologies.in) Jakub Krajcovic (jakub.krajcovic@gmail.com) -David Kranz (david.kranz@qrclab.com) -Sushil Kumar (sushil.kumar2@globallogic.com) -Greg Lange (greglange@gmail.com) -Alexis Lee (lxsli@hpe.com) +James Nzomo (james@tdt.rocks) Jamie Lennox (jamielennox@gmail.com) -Cheng Li (shcli@cn.ibm.com) -Tong Li (litong01@us.ibm.com) -Peter Lisak (peter.lisak@firma.seznam.cz) -Feng Liu (mefengliu23@gmail.com) +Jeremy Stanley (fungi@yuggoth.org) +Ji-Wei (ji.wei3@zte.com.cn) +Jian Zhang (jian.zhang@intel.com) Jing Liuqing (jing.liuqing@99cloud.net) -Hemanth Makkapati (hemanth.makkapati@mailtrust.com) -Pratik Mallya (pratik.mallya@gmail.com) -Steve Martinelli (stevemar@ca.ibm.com) -Juan J. Martinez (juan@memset.com) -Donagh McCabe (donagh.mccabe@hpe.com) -Ben McCann (ben@benmccann.com) -Andy McCrae (andy.mccrae@gmail.com) -Stuart McLaren (stuart.mclaren@hpe.com) -Samuel Merritt (sam@swiftstack.com) -Min Min Ren (rminmin@cn.ibm.com) +Jiří Suchomel (jsuchome@suse.cz) +Joel Wright (joel.wright@sohonet.com) +John Dickinson (me@not.mn) Jola Mirecka (jola.mirecka@hp.com) -Hiroshi Miura (miurahr@nttdata.co.jp) -Sam Morrison (sorrison@gmail.com) -Dirk Mueller (dirk@dmllr.de) -Zhenguo Niu (zhenguo@unitedstack.com) -Ondrej Novy (ondrej.novy@firma.seznam.cz) -James Nzomo (james@tdt.rocks) -Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) -Alessandro Pilotti (ap@pilotti.it) -Stanislaw Pitucha (stanislaw.pitucha@hpe.com) -Dan Prince (dprince@redhat.com) -ricolin (rico.l@inwinstack.com) +Josh Gachnang (josh@pcsforeducation.com) +Juan J. Martinez (juan@memset.com) +Jude Job (judeopenstack@gmail.com) +Julien Danjou (julien@danjou.info) +Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) +Kun Huang (gareth@unitedstack.com) +Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) -Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) -Christian Schwede (cschwede@redhat.com) +Luis de Bethencourt (luis@debethencourt.com) +Mahati Chamarthy (mahati.chamarthy@gmail.com) +Marek Kaleta (marek.kaleta@firma.seznam.cz) Mark Seger (mark.seger@hpe.com) -Chuck Short (chuck.short@canonical.com) -David Shrewsbury (shrewsbury.dave@gmail.com) -Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) -Alexandra Settle (alexandra.settle@rackspace.com) -Jeremy Stanley (fungi@yuggoth.org) -Victor Stinner (victor.stinner@enovance.com) -Jiří Suchomel (jsuchome@suse.cz) -YUZAWA Takahiko (yuzawataka@intellilink.co.jp) -Nandini Tata (nandini.tata.15@gmail.com) +Mark Washenberger (mark.washenberger@rackspace.com) +Martin Geisler (martin@geisler.net) +Matthew Oliver (matt@oliver.net.au) +Matthieu Huin (mhu@enovance.com) +Mike Widman (mwidman@endurancewindpower.com) +Min Min Ren (rminmin@cn.ibm.com) +Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) +Nandini Tata (nandini.tata@intel.com) +Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) +Nick Craig-Wood (nick@craig-wood.com) +Ondrej Novy (ondrej.novy@firma.seznam.cz) +Pallavi (pallavi.s@nectechnologies.in) +Paul Belanger (pabelanger@redhat.com) +Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br) +Pete Zaitcev (zaitcev@kotori.zaitcev.us) +Peter Lisak (peter.lisak@firma.seznam.cz) +Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Pratik Mallya (pratik.mallya@gmail.com) +Qiu Yu (qiuyu@ebaysf.com) +Ray Chen (oldsharp@163.com) +ricolin (rico.l@inwinstack.com) +Romain Hardouin (romain_hardouin@yahoo.fr) +Sahid Orentino Ferdjaoui (sahid.ferdjaoui@cloudwatt.com) +SaiKiran (saikiranveeravarapu@gmail.com) +Sam Morrison (sorrison@gmail.com) +Samuel Merritt (sam@swiftstack.com) +Sean Dague (sean@dague.net) +Sergey Gotliv (sgotliv@redhat.com) +Sergio Cazzolato (sergio.j.cazzolato@intel.com) +Shane Wang (shane.wang@intel.com) +Shashi Kant (shashi.kant@nectechnologies.in) +Shashirekha Gundur (shashirekha.j.gundur@intel.com) +shu-mutou (shu-mutou@rf.jp.nec.com) +Stanislav Vitkovskiy (stas.vitkovsky@gmail.com) +Stanislaw Pitucha (stanislaw.pitucha@hpe.com) +Steve Martinelli (stevemar@ca.ibm.com) +Steven Hardy (shardy@redhat.com) +Stuart McLaren (stuart.mclaren@hpe.com) +Sushil Kumar (sushil.kumar2@globallogic.com) +tanlin (lin.tan@intel.com) +Taurus Cheung (Taurus.Cheung@harmonicinc.com) TheSriram (sriram@klusterkloud.com) +Thiago da Silva (thiago@redhat.com) +Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) -Dean Troyer (dtroyer@gmail.com) -Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) -Stanislav Vitkovskiy (stas.vitkovsky@gmail.com) -Daniel Wakefield (daniel.wakefield@hp.com) -Shane Wang (shane.wang@intel.com) -Mark Washenberger (mark.washenberger@rackspace.com) -Andrew Welleck (awellec@us.ibm.com) +Tim Burke (tim.burke@gmail.com) +Tong Li (litong01@us.ibm.com) +Tony Breeds (tony@bakeyournoodle.com) +Tristan Cacqueray (tristan.cacqueray@enovance.com) +Vasyl Khomenko (vasiliyk@yahoo-inc.com) +venkatamahesh (venkatamaheshkotha@gmail.com) +Victor Stinner (victor.stinner@enovance.com) +wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) -Mike Widman (mwidman@endurancewindpower.com) -Joel Wright (joel.wright@sohonet.com) -You Yamagata (bi.yamagata@gmail.com) -zheng yin (yin.zheng@easystack.cn) -Qiu Yu (qiuyu@ebaysf.com) YangLei (yanglyy@cn.ibm.com) -Pete Zaitcev (zaitcev@kotori.zaitcev.us) -Jian Zhang (jian.zhang@intel.com) -Yuan Zhou (yuan.zhou@intel.com) -groqez (groqez@yopmail.net) -tanlin (lin.tan@intel.com) yangxurong (yangxurong@huawei.com) +You Yamagata (bi.yamagata@gmail.com) +Yuan Zhou (yuan.zhou@intel.com) +Yushiro FURUKAWA (y.furukawa_2@jp.fujitsu.com) yuxcer (yuxcer@126.com) -zhang-jinnan (ben.os@99cloud.net) -hgangwx (hgangwx@cn.ibm.com) -shu-mutou (shu-mutou@rf.jp.nec.com) -SaiKiran (saikiranveeravarapu@gmail.com) -venkatamahesh (venkatamaheshkotha@gmail.com) yuyafei (yu.yafei@zte.com.cn) +YUZAWA Takahiko (yuzawataka@intellilink.co.jp) +Zack M. Davis (zdavis@swiftstack.com) +zhang-jinnan (ben.os@99cloud.net) +zhangyanxian (zhangyanxianmail@163.com) +zheng yin (yin.zheng@easystack.cn) +Zhenguo Niu (zhenguo@unitedstack.com) @@ -1,3 +1,31 @@ +3.3.0 +----- + +* Added support for prefix-based tempurls. This allows you to create a + tempurl that is valid for all objects which share a given prefix and + matches the feature in Swift 2.12.0 and later. + +* In the SDK, we previously only accepted iterables of strings like + 'Header: Value'. Now, we'll also accept lists of tuples like + ('Header', 'Value') as well as dictionaries like {'Header': 'Value'}. + +* Improved help message strings + +* Various other minor bug fixes and improvements. + +3.2.0 +----- + +* Added Keystone session support and a "v1password" plugin for Keystone. + This plugin provides a way for Keystone sessions (and clients that + use them, like python-openstackclient) to communicate with old auth + endpoints that still use this mechanism. + +* HEAD, GET, and DELETE now support sending additional headers to match + existing functionality on PUT requests. + +* Various other minor bug fixes and improvements. + 3.1.0 ----- @@ -1,3 +1,12 @@ +======================== +Team and repository tags +======================== + +.. image:: http://governance.openstack.org/badges/python-swiftclient.svg + :target: http://governance.openstack.org/reference/tags/index.html + +.. Change things from this point on + Python bindings to the OpenStack Object Storage API =================================================== @@ -1,5 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; # see http://docs.openstack.org/infra/bindep/ for additional information. +curl pypy [test] pypy-dev [test] diff --git a/doc/Makefile b/doc/Makefile index 73aeb6e..3943da4 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -62,9 +62,9 @@ qthelp: @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/python-novaclient.qhcp" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/python-swiftclient.qhcp" @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/python-novaclient.qhc" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/python-swiftclient.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 5d21c18..bbf5280 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -134,10 +134,12 @@ programs, such as jq. .RS 4 Generates a temporary URL allowing unauthenticated access to the Swift object at the given path, using the given HTTP method, for the given number of -seconds, using the given TempURL key. If optional \-\-absolute argument is +seconds, using the given TempURL key. With the optional \-\-prefix\-based option a +prefix-based URL is generated. If optional \-\-absolute argument is provided, seconds is instead interpreted as a Unix timestamp at which the URL should expire. \fBExample\fR: tempurl GET $(date \-d "Jan 1 2016" +%s) /v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key \-\-absolute + .RE \fBauth\fR @@ -184,4 +186,4 @@ swift \-A https://127.0.0.1:443/auth/v1.0 \-U swiftops:swiftops \-K swiftops sta .SH DOCUMENTATION .LP More in depth documentation about OpenStack Swift as a whole can be found at -.BI http://swift.openstack.org +.BI https://docs.openstack.org/developer/swift diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 87020c9..1f76b05 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -228,7 +228,7 @@ Capabilities Tempurl ------- - ``tempurl [method] [seconds] [path] [key]`` + ``tempurl [command-options] [method] [seconds] [path] [key]`` Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to allow for this temporary URL that is usually 'GET' or 'PUT'. ``seconds`` option sets @@ -236,7 +236,10 @@ Tempurl is passed, the Unix timestamp when the temporary URL will expire. ``path`` option sets the full path to the Swift object. Example: ``/v1/AUTH_account/c/o``. ``key`` option is the secret temporary URL key set on the Swift cluster. To set a key, run - ``swift post -m "Temp-URL-Key: <your secret key>"``. + ``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary + URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not + forget to append the desired objectname at the end of the path portion (and before the + query portion) before sharing the URL. Auth ---- @@ -244,7 +247,7 @@ Auth ``auth`` Display authentication variables in shell friendly format. Command to run to export storage - url and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. + URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic authentication: ``swift auth -v -U test:tester -K testing``. @@ -294,6 +297,30 @@ List the contents of a container: testSwift.txt +Copy an object to new destination: + +.. code-block:: bash + + > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt + + SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt + +Delete an object from a container: + +.. code-block:: bash + + > swift delete TestContainer testSwift.txt + + testSwift.txt + +Delete a container: + +.. code-block:: bash + + > swift delete TestContainer + + TestContainer + Display auth related authentication variables in shell friendly format: .. code-block:: bash @@ -318,8 +345,10 @@ Download an object from a container: To upload an object to a container, your current working directory must be where the file is located or you must provide the complete path to the file. - In the case that you provide the complete path of the file, that complete - path will be the name of the uploaded object. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. For example: diff --git a/releasenotes/notes/320_notes-bb367dba1053d34c.yaml b/releasenotes/notes/320_notes-bb367dba1053d34c.yaml new file mode 100644 index 0000000..a603555 --- /dev/null +++ b/releasenotes/notes/320_notes-bb367dba1053d34c.yaml @@ -0,0 +1,12 @@ +features: + - > + Added Keystone session support and a "v1password" plugin for Keystone. + This plugin provides a way for Keystone sessions (and clients that + use them, like python-openstackclient) to communicate with old auth + endpoints that still use this mechanism. + + - > + HEAD, GET, and DELETE now support sending additional headers to match + existing functionality on PUT requests. + + - Various other minor bug fixes and improvements. diff --git a/swiftclient/client.py b/swiftclient/client.py index 21cbe27..80b6eda 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -32,7 +32,8 @@ import six from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException from swiftclient.utils import ( - iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response) + iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response, + get_body) # Default is 100, increase to 256 http_client._MAXHEADERS = 256 @@ -165,7 +166,9 @@ def http_log(args, kwargs, resp, body): log_method("RESP STATUS: %s %s", resp.status, resp.reason) log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders())) if body: - log_method("RESP BODY: %s", body) + resp_headers = resp_header_dict(resp) + nbody = get_body(resp_headers, body) + log_method("RESP BODY: %s", nbody) def parse_header_string(data): diff --git a/swiftclient/service.py b/swiftclient/service.py index de30d50..8c6880a 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -29,11 +29,10 @@ from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread -from six import StringIO, text_type +from six import Iterator, StringIO, string_types, text_type from six.moves.queue import Queue from six.moves.queue import Empty as QueueEmpty from six.moves.urllib.parse import quote -from six import Iterator, string_types import json @@ -274,7 +273,10 @@ def split_headers(options, prefix=''): """ Splits 'Key: Value' strings and returns them as a dictionary. - :param options: An array of 'Key: Value' strings + :param options: Must be one of: + * an iterable of 'Key: Value' strings + * an iterable of ('Key', 'Value') pairs + * a dict of {'Key': 'Value'} pairs :param prefix: String to prepend to all of the keys in the dictionary. reporting. """ diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 9fd7aac..ceca592 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -59,6 +59,7 @@ st_delete_options = '''[--all] [--leave-segments] [--object-threads <threads>] [--container-threads <threads>] [--header <header:value>] + [--prefix <prefix>] [<container> [<object>] [...]] ''' @@ -82,6 +83,7 @@ Optional arguments: --container-threads <threads> Number of threads to use for deleting containers. Default is 10. + --prefix <prefix> Only delete objects beginning with <prefix>. '''.strip("\n") @@ -91,7 +93,7 @@ def st_delete(parser, args, output_manager): default=False, help='Delete all containers and objects.') parser.add_argument( '-p', '--prefix', dest='prefix', - help='Only delete items beginning with the <prefix>.') + help='Only delete items beginning with <prefix>.') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -793,7 +795,7 @@ Positional arguments: Optional arguments: -d, --destination </container[/object]> The container and name of the destination object. Name - of destination object can be ommited, then will be + of destination object can be omitted, then will be same as name of source object. Supplying multiple objects and destination with object name is invalid. -M, --fresh-metadata Copy the object without any existing metadata, @@ -884,7 +886,8 @@ st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] <container> <file_or_directory> [<file_or_directory>] [...] ''' -st_upload_help = ''' Uploads specified files and directories to the given container. +st_upload_help = ''' +Uploads specified files and directories to the given container. Positional arguments: <container> Name of container to upload to. @@ -1121,7 +1124,8 @@ def st_upload(parser, args, output_manager): output_manager.error(e.value) -st_capabilities_options = "[--json] [<proxy_url>]" +st_capabilities_options = '''[--json] [<proxy_url>] +''' st_info_options = st_capabilities_options st_capabilities_help = ''' Retrieve capability of the proxy. @@ -1220,7 +1224,7 @@ def st_auth(parser, args, thread_manager): print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) -st_tempurl_options = '''[--absolute] +st_tempurl_options = '''[--absolute] [--prefix-based] <method> <seconds> <path> <key>''' @@ -1244,6 +1248,7 @@ Optional arguments: --absolute Interpret the <seconds> positional argument as a Unix timestamp rather than a number of seconds in the future. + --prefix-based If present, a prefix-based tempURL will be generated. '''.strip('\n') @@ -1253,8 +1258,14 @@ def st_tempurl(parser, args, thread_manager): dest='absolute_expiry', default=False, help=("If present, seconds argument will be interpreted as a Unix " "timestamp representing when the tempURL should expire, rather " - "than an offset from the current time") + "than an offset from the current time"), + ) + parser.add_argument( + '--prefix-based', action='store_true', + default=False, + help=("If present, a prefix-based tempURL will be generated."), ) + (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 4: @@ -1271,7 +1282,8 @@ def st_tempurl(parser, args, thread_manager): method.upper()) try: path = generate_temp_url(parsed.path, seconds, key, method, - absolute=options['absolute_expiry']) + absolute=options['absolute_expiry'], + prefix=options['prefix_based'],) except ValueError as err: thread_manager.error(err) return @@ -1329,9 +1341,12 @@ def parse_args(parser, args, enforce_requires=True): logging.basicConfig(level=logging.INFO) if args and options.get('help'): - _help = globals().get('st_%s_help' % args[0], - "no help for %s" % args[0]) - print(_help) + _help = globals().get('st_%s_help' % args[0]) + _options = globals().get('st_%s_options' % args[0], "\n") + if _help: + print("Usage: %s %s %s\n%s" % (BASENAME, args[0], _options, _help)) + else: + print("no such command: %s" % args[0]) exit() # Short circuit for tempurl, which doesn't need auth diff --git a/swiftclient/utils.py b/swiftclient/utils.py index e14602d..47856c2 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" +import collections import gzip import hashlib import hmac @@ -62,12 +63,14 @@ def prt_bytes(num_bytes, human_flag): return '%.1f%s' % (num, suffix) -def generate_temp_url(path, seconds, key, method, absolute=False): +def generate_temp_url(path, seconds, key, method, absolute=False, + prefix=False): """Generates a temporary URL that gives unauthenticated access to the Swift object. - :param path: The full path to the Swift object. Example: - /v1/AUTH_account/c/o. + :param path: The full path to the Swift object or prefix if + a prefix-based temporary URL should be generated. Example: + /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. :param seconds: If absolute is False then this specifies the amount of time in seconds for which the temporary URL will be valid. If absolute is True then this specifies an absolute time at which the temporary URL @@ -80,6 +83,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False): :param absolute: if True then the seconds parameter is interpreted as an absolute Unix time, otherwise seconds is interpreted as a relative time offset from current time. + :param prefix: if True then a prefix-based temporary URL will be generated. :raises: ValueError if seconds is not a whole number or path is not to an object. :return: the path portion of a temporary URL @@ -103,8 +107,12 @@ def generate_temp_url(path, seconds, key, method, absolute=False): path_for_body = path parts = path_for_body.split('/', 4) - if len(parts) != 5 or parts[0] or not all(parts[1:]): - raise ValueError('path must be full path to an object e.g. /v1/a/c/o') + if len(parts) != 5 or parts[0] or not all(parts[1:(4 if prefix else 5)]): + if prefix: + raise ValueError('path must at least contain /v1/a/c/') + else: + raise ValueError('path must be full path to an object' + ' e.g. /v1/a/c/o') standard_methods = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'] if method.upper() not in standard_methods: @@ -116,7 +124,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False): expiration = int(time.time() + seconds) else: expiration = seconds - hmac_body = u'\n'.join([method.upper(), str(expiration), path_for_body]) + hmac_body = u'\n'.join([method.upper(), str(expiration), + ('prefix:' if prefix else '') + path_for_body]) # Encode to UTF-8 for py3 compatibility if not isinstance(key, six.binary_type): @@ -125,6 +134,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False): temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) + if prefix: + temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller if isinstance(path, six.binary_type): return temp_url.encode('utf-8') @@ -132,11 +143,16 @@ def generate_temp_url(path, seconds, key, method, absolute=False): return temp_url -def parse_api_response(headers, body): +def get_body(headers, body): if headers.get('content-encoding') == 'gzip': with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz: - body = gz.read() + nbody = gz.read() + return nbody + return body + +def parse_api_response(headers, body): + body = get_body(headers, body) charset = 'utf-8' # Swift *should* be speaking UTF-8, but check content-type just in case content_type = headers.get('content-type', '') @@ -148,15 +164,24 @@ def parse_api_response(headers, body): def split_request_headers(options, prefix=''): headers = {} + if isinstance(options, collections.Mapping): + options = options.items() for item in options: - split_item = item.split(':', 1) - if len(split_item) == 2: - headers[(prefix + split_item[0]).title()] = split_item[1].strip() - else: + if isinstance(item, six.string_types): + if ':' not in item: + raise ValueError( + "Metadata parameter %s must contain a ':'.\n" + "Example: 'Color:Blue' or 'Size:Large'" + % item + ) + item = item.split(':', 1) + if len(item) != 2: raise ValueError( - "Metadata parameter %s must contain a ':'.\n%s" - % (item, "Example: 'Color:Blue' or 'Size:Large'") + "Metadata parameter %r must have exactly two items.\n" + "Example: ('Color', 'Blue') or ['Size', 'Large']" + % (item, ) ) + headers[(prefix + item[0]).title()] = item[1].strip() return headers diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 7fe9a54..d60ae06 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -110,15 +110,20 @@ class TestFunctional(unittest.TestCase): pass def _check_account_headers(self, headers): - self.assertTrue(headers.get('content-length')) - self.assertTrue(headers.get('x-account-object-count')) - self.assertTrue(headers.get('x-timestamp')) - self.assertTrue(headers.get('x-trans-id')) - self.assertTrue(headers.get('date')) - self.assertTrue(headers.get('x-account-bytes-used')) - self.assertTrue(headers.get('x-account-container-count')) - self.assertTrue(headers.get('content-type')) - self.assertTrue(headers.get('accept-ranges')) + headers_to_check = [ + 'content-length', + 'x-account-object-count', + 'x-timestamp', + 'x-trans-id', + 'date', + 'x-account-bytes-used', + 'x-account-container-count', + 'content-type', + 'accept-ranges', + ] + for h in headers_to_check: + self.assertIn(h, headers) + self.assertTrue(headers[h]) def test_stat_account(self): headers = self.conn.head_account() @@ -322,16 +327,16 @@ class TestFunctional(unittest.TestCase): # verify that put using a generator yielding empty strings does not # cause connection to be closed def data(): - yield "should" - yield "" - yield " tolerate" - yield "" - yield " empty chunks" + yield b"should" + yield b"" + yield b" tolerate" + yield b"" + yield b" empty chunks" self.conn.put_object( self.containername, self.objectname, data()) hdrs, body = self.conn.get_object(self.containername, self.objectname) - self.assertEqual("should tolerate empty chunks", body) + self.assertEqual(b"should tolerate empty chunks", body) def test_download_object_retry_chunked(self): resp_chunk_size = 2 diff --git a/tests/unit/test_authv1.py b/tests/unit/test_authv1.py index 968109a..2ddf24b 100644 --- a/tests/unit/test_authv1.py +++ b/tests/unit/test_authv1.py @@ -174,7 +174,7 @@ class TestPlugin(TestDataNoAccount, unittest.TestCase): auth_plugin = authv1.PasswordPlugin(**self.options) self.mock_response.headers['X-Auth-Token-Expires'] = 'foo' access = auth_plugin.get_access(self.mock_session) - self.assertEqual(None, access.expires) + self.assertIsNone(access.expires) self.assertIs(False, access.will_expire_soon(60)) self.assertIs(False, access.will_expire_soon(1e20)) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index e8385b7..2fc827c 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -592,11 +592,24 @@ class TestServiceUtils(unittest.TestCase): actual = swiftclient.service.split_headers(mock_headers, 'prefix-') self.assertEqual(expected, actual) - def test_split_headers_error(self): - mock_headers = ['notvalid'] + def test_split_headers_list_of_tuples(self): + mock_headers = [('color', 'blue'), ('size', 'large')] + expected = {'Prefix-Color': 'blue', 'Prefix-Size': 'large'} + + actual = swiftclient.service.split_headers(mock_headers, 'prefix-') + self.assertEqual(expected, actual) + def test_split_headers_dict(self): + expected = {'Color': 'blue', 'Size': 'large'} + + actual = swiftclient.service.split_headers(expected) + self.assertEqual(expected, actual) + + def test_split_headers_error(self): + self.assertRaises(SwiftError, swiftclient.service.split_headers, + ['notvalid']) self.assertRaises(SwiftError, swiftclient.service.split_headers, - mock_headers) + [('also', 'not', 'valid')]) class TestSwiftUploadObject(unittest.TestCase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 9408e73..f7e9c1a 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1549,7 +1549,17 @@ class TestShell(unittest.TestCase): "secret_key"] swiftclient.shell.main(argv) temp_url.assert_called_with( - '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False) + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + prefix=False) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_prefix_based(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/", + "secret_key", "--prefix-based"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, + prefix=True) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1557,7 +1567,8 @@ class TestShell(unittest.TestCase): "secret_key", "--absolute"] swiftclient.shell.main(argv) temp_url.assert_called_with( - '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True) + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, + prefix=False) def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1575,6 +1586,15 @@ class TestShell(unittest.TestCase): expected = "http://saio:8080%s" % expected self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/", + "secret_key", "--absolute", "--prefix"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' + expected = ("/v1/a/c/?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_prefix=\n" % sig) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -1587,6 +1607,15 @@ class TestShell(unittest.TestCase): 'Expected %r but got %r for path %r' % (expected, output.err, bad_path)) + expected = 'path must at least contain /v1/a/c/\n' + argv = ["", "tempurl", "GET", "60", '/v1/a/c', + "secret_key", "--absolute", '--prefix-based'] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + self.assertEqual(expected, output.err, + 'Expected %r but got %r for path %r' % + (expected, output.err, bad_path)) + @mock.patch('swiftclient.service.Connection') def test_capabilities(self, connection): argv = ["", "capabilities"] @@ -1688,18 +1717,21 @@ class TestSubcommandHelp(unittest.TestCase): def test_subcommand_help(self): for command in swiftclient.shell.commands: help_var = 'st_%s_help' % command + options_var = 'st_%s_options' % command self.assertTrue(hasattr(swiftclient.shell, help_var)) with CaptureOutput() as out: argv = ['', command, '--help'] self.assertRaises(SystemExit, swiftclient.shell.main, argv) - expected = vars(swiftclient.shell)[help_var] + expected = 'Usage: swift %s %s\n%s' % ( + command, vars(swiftclient.shell).get(options_var, "\n"), + vars(swiftclient.shell)[help_var]) self.assertEqual(out.strip('\n'), expected) def test_no_help(self): with CaptureOutput() as out: argv = ['', 'bad_command', '--help'] self.assertRaises(SystemExit, swiftclient.shell.main, argv) - expected = 'no help for bad_command' + expected = 'no such command: bad_command' self.assertEqual(out.strip('\n'), expected) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 3a24b00..d4a704e 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import gzip import logging import mock import six @@ -82,7 +83,7 @@ class TestClientException(unittest.TestCase): class MockHttpResponse(object): - def __init__(self, status=0, headers=None, verify=False): + def __init__(self, status=0, headers=None, verify=False, need_items=None): self.status = status self.status_code = status self.reason = "OK" @@ -91,6 +92,7 @@ class MockHttpResponse(object): self.verify = verify self.md5sum = md5() self.headers = {'etag': '"%s"' % EMPTY_ETAG} + self.need_items = need_items if headers: self.headers.update(headers) self.closed = False @@ -117,6 +119,8 @@ class MockHttpResponse(object): return self.headers.get(name, default) def getheaders(self): + if self.need_items: + return dict(self.headers).items() return dict(self.headers) def fake_response(self): @@ -2607,6 +2611,44 @@ class TestLogging(MockHttpTest): self.assertNotIn(unicode_token_value, output) self.assertNotIn(set_cookie_value, output) + def test_logging_body(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') + set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, + 'content-encoding': 'gzip', + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_encoded + }, + need_items=True, + ), + buf.getvalue(), + ) + self.assertEqual( + mock.call( + 'RESP BODY: %s', u'{"test": "\u2603"}'.encode('utf8')), + mock_log.mock_calls[3]) + def test_show_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 787f645..c5961e8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -150,6 +150,35 @@ class TestTempURL(unittest.TestCase): ]) self.assertIsInstance(url, type(self.url)) + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_prefix(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + prefixes = ['', 'o', 'p0/p1/'] + for p in prefixes: + hmac_mock.reset_mock() + path = '/v1/AUTH_account/c/' + p + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_prefix=' + p) + expected_body = '\n'.join([ + self.method, + '1400003600', + 'prefix:' + path, + ]).encode('utf-8') + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, prefix=True) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + self.assertEqual(url, expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + + self.assertIsInstance(url, type(path)) + def test_generate_temp_url_invalid_path(self): with self.assertRaises(ValueError) as exc_manager: u.generate_temp_url(b'/v1/a/c/\xff', self.seconds, self.key, @@ -221,6 +250,12 @@ class TestTempURL(unittest.TestCase): self.assertEqual(exc_manager.exception.args[0], 'path must be full path to an object e.g. /v1/a/c/o') + with self.assertRaises(ValueError) as exc_manager: + u.generate_temp_url('/v1/a/c', 60, self.key, self.method, + prefix=True) + self.assertEqual(exc_manager.exception.args[0], + 'path must at least contain /v1/a/c/') + class TestTempURLUnicodePathAndKey(TestTempURL): url = u'/v1/\u00e4/c/\u00f3' @@ -472,3 +507,21 @@ class TestApiResponeParser(unittest.TestCase): {'content-encoding': 'gzip'}, buf.getvalue()) self.assertEqual({'test': u'\u2603'}, result) + + +class TestGetBody(unittest.TestCase): + + def test_not_gzipped(self): + result = u.parse_api_response( + {}, u'{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': u'\u2603'}, result) + + def test_gzipped_body(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.close() + result = u.parse_api_response( + {'content-encoding': 'gzip'}, + buf.getvalue()) + self.assertEqual({'test': u'\u2603'}, result) diff --git a/tools/tox_install.sh b/tools/tox_install.sh new file mode 100755 index 0000000..15aa9de --- /dev/null +++ b/tools/tox_install.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Client constraint file contains this client version pin that is in conflict +# with installing the client from source. We should remove the version pin in +# the constraints file before applying it for from-source installation. + +set -e + +if [[ -z "$CONSTRAINTS_FILE" ]]; then + echo 'WARNING: expected $CONSTRAINTS_FILE to be set' >&2 + PIP_FLAGS=(-U) +else + # NOTE(tonyb): Place this in the tox enviroment's log dir so it will get + # published to logs.openstack.org for easy debugging. + localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + + if [[ "$CONSTRAINTS_FILE" != http* ]]; then + CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" + fi + curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" + + pip install -c"$localfile" openstack-requirements + + # This is the main purpose of the script: Allow local installation of + # the current repo. It is listed in constraints file and thus any + # install will be constrained and we need to unconstrain it. + edit-constraints "$localfile" -- "$CLIENT_NAME" + PIP_FLAGS=(-c"$localfile" -U) +fi + +pip install "${PIP_FLAGS[@]}" "$@" @@ -1,14 +1,17 @@ [tox] envlist = py27,py34,py35,pypy,pep8 -minversion = 1.6 +minversion = 2.0 skipsdist = True [testenv] usedevelop = True -install_command = pip install -U {opts} {packages} +install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} + BRANCH_NAME=master + CLIENT_NAME=python-swiftclient + CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -28,11 +31,13 @@ commands = commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage +commands = python setup.py testr --coverage coverage report [testenv:func] -setenv = OS_TEST_PATH=tests.functional +setenv = + {[testenv]setenv} + OS_TEST_PATH=tests.functional whitelist_externals = coverage rm |