diff options
52 files changed, 3732 insertions, 1062 deletions
@@ -2,9 +2,13 @@ set -e export OS_TEST_PATH='tests.functional' -python setup.py testr --coverage --testr-args="--concurrency=1" +export PYTHON='coverage run --source swiftclient --parallel-mode' +stestr run --concurrency=1 RET=$? +coverage combine +coverage html -d cover +coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage exit $RET @@ -7,6 +7,7 @@ dist/ .DS_Store *.log .testrepository +.stestr/ subunit.log build swiftclient/versioninfo @@ -16,3 +17,4 @@ cover/ coverage.xml doc/build doc/source/api/ +.idea @@ -1,4 +1,4 @@ [gerrit] -host=review.openstack.org +host=review.opendev.org port=29418 project=openstack/python-swiftclient.git @@ -58,7 +58,8 @@ Madhuri Kumari <madhuri.rai07@gmail.com> madhuri <madhuri@madhuri-VirtualBox.(no Morgan Fainberg <morgan.fainberg@gmail.com> <m@metacloud.com> Hua Zhang <zhuadl@cn.ibm.com> <zhuadl@cn.ibm.com> Yummy Bian <yummy.bian@gmail.com> <yummy.bian@gmail.com> -Alistair Coles <alistair.coles@hpe.com> <alistair.coles@hp.com> +Alistair Coles <alistairncoles@gmail.com> <alistair.coles@hp.com> +Alistair Coles <alistairncoles@gmail.com> <alistair.coles@hpe.com> Tong Li <litong01@us.ibm.com> <litong01@us.ibm.com> Paul Luse <paul.e.luse@intel.com> <paul.e.luse@intel.com> Yuan Zhou <yuan.zhou@intel.com> <yuan.zhou@intel.com> @@ -94,3 +95,5 @@ Andreas Jaeger <aj@suse.de> <aj@suse.com> Shashi Kant <shashi.kant@nectechnologies.in> Nandini Tata <nandini.tata@intel.com> <nandini.tata.15@gmail.com> Flavio Percoco <flaper87@gmail.com> +Timur Alperovich <timuralp@swiftstack.com> <timur@timuralp.com> +Thiago da Silva <thiagodasilva@gmail.com> <thiago@redhat.com> diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..5228f20 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_path=${OS_TEST_PATH:-./tests/unit} +top_dir=./ + diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index f3fca90..0000000 --- a/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..ac1d966 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,59 @@ +- job: + name: swiftclient-swift-functional + parent: swift-dsvm-functional + description: | + Run swift's functional tests with python-swiftclient + installed from source instead as package from PyPI. + # Ensure that we install python-swiftclient from git and + # do not install from pypi. This is needed since the parent + # job sets zuul_work_dir to the swift directory and uses tox + # for installation. + required-projects: + - opendev.org/openstack/python-swiftclient + +- job: + name: swiftclient-functional + parent: swift-dsvm-functional + description: | + Run functional tests of python-swiftclient with + python-swiftclient installed from source instead as package from + PyPI. + required-projects: + - opendev.org/openstack/python-swiftclient + vars: + # Override value from parent job to use swiftclient tests + zuul_work_dir: "{{ zuul.projects['opendev.org/openstack/python-swiftclient'].src_dir }}" + +- job: + name: swiftclient-functional-py2 + parent: swiftclient-functional + description: | + Run functional tests of python-swiftclient under Python 2 + vars: + tox_envlist: py2func + +- project: + templates: + - check-requirements + - lib-forward-testing + - openstack-lower-constraints-jobs + - openstack-pypy-jobs-nonvoting + - openstack-python-jobs + - openstack-python35-jobs + - openstack-python36-jobs + - openstack-python37-jobs + - publish-openstack-docs-pti + - release-notes-jobs-python3 + check: + jobs: + - swiftclient-swift-functional + - swiftclient-functional + - swiftclient-functional-py2 + gate: + jobs: + - swiftclient-swift-functional + - swiftclient-functional + - swiftclient-functional-py2 + post: + jobs: + - openstack-tox-cover @@ -2,7 +2,7 @@ Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) Alexandra Settle (alexandra.settle@rackspace.com) Alexis Lee (lxsli@hpe.com) -Alistair Coles (alistair.coles@hpe.com) +Alistair Coles (alistairncoles@gmail.com) Andreas Jaeger (aj@suse.de) Andrew Welleck (awellec@us.ibm.com) Andy McCrae (andy.mccrae@gmail.com) @@ -12,6 +12,7 @@ Ben McCann (ben@benmccann.com) Cedric Brandily (zzelle@gmail.com) Chaozhe.Chen (chaozhe.chen@easystack.cn) Charles Hsu (charles0126@gmail.com) +Chen (dstbtgagt@foxmail.com) Cheng Li (shcli@cn.ibm.com) Chmouel Boudjnah (chmouel@enovance.com) Chris Buccella (chris.buccella@antallagon.com) @@ -35,6 +36,7 @@ Dirk Mueller (dirk@dmllr.de) Donagh McCabe (donagh.mccabe@hpe.com) Doug Hellmann (doug@doughellmann.com) EdLeafe (ed@leafe.com) +Erik Olof Gunnar Andersson (eandersson@blizzard.com) Fabien Boucher (fabien.boucher@enovance.com) Feng Liu (mefengliu23@gmail.com) Flavio Percoco (flaper87@gmail.com) @@ -42,6 +44,7 @@ Florent Flament (florent.flament-ext@cloudwatt.com) Greg Holt (gholt@rackspace.com) Greg Lange (greglange@gmail.com) groqez (groqez@yopmail.net) +Hangdong Zhang (hdzhang@fiberhome.com) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) hgangwx (hgangwx@cn.ibm.com) Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) @@ -65,11 +68,15 @@ Josh Gachnang (josh@pcsforeducation.com) Juan J. Martinez (juan@memset.com) Jude Job (judeopenstack@gmail.com) Julien Danjou (julien@danjou.info) +Kazufumi Noto (noto.kazufumi@gmail.com) Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) +lingyongxu (lyxu@fiberhome.com) +liuyamin (liuyamin@fiberhome.com) Luis de Bethencourt (luis@debethencourt.com) +M V P Nitesh (m.nitesh@nectechnologies.in) Mahati Chamarthy (mahati.chamarthy@gmail.com) Marek Kaleta (marek.kaleta@firma.seznam.cz) Mark Seger (mark.seger@hpe.com) @@ -79,9 +86,13 @@ Matthew Oliver (matt@oliver.net.au) Matthieu Huin (mhu@enovance.com) Mike Widman (mwidman@endurancewindpower.com) Min Min Ren (rminmin@cn.ibm.com) +mmcardle (mark.mcardle@sohonet.com) Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) +Nelson Marcos (nelsonmarcos@gmail.com) +Nguyen Hai (nguyentrihai93@gmail.com) +Nguyen Hai Truong (truongnh@vn.fujitsu.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -90,8 +101,10 @@ Paul Belanger (pabelanger@redhat.com) Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br) Pete Zaitcev (zaitcev@kotori.zaitcev.us) Peter Lisak (peter.lisak@firma.seznam.cz) +Petr Kovar (pkovar@redhat.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) Pratik Mallya (pratik.mallya@gmail.com) +qingszhao (zhao.daqing@99cloud.net) Qiu Yu (qiuyu@ebaysf.com) Ray Chen (oldsharp@163.com) ricolin (rico.l@inwinstack.com) @@ -104,6 +117,7 @@ Sean Dague (sean@dague.net) Sergey Gotliv (sgotliv@redhat.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Shane Wang (shane.wang@intel.com) +shangxiaobj (shangxiaobj@inspur.com) Shashi Kant (shashi.kant@nectechnologies.in) Shashirekha Gundur (shashirekha.j.gundur@intel.com) shu-mutou (shu-mutou@rf.jp.nec.com) @@ -112,22 +126,29 @@ Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Steve Martinelli (stevemar@ca.ibm.com) Steven Hardy (shardy@redhat.com) Stuart McLaren (stuart.mclaren@hpe.com) +sunjia (sunjia@inspur.com) Sushil Kumar (sushil.kumar2@globallogic.com) tanlin (lin.tan@intel.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) TheSriram (sriram@klusterkloud.com) -Thiago da Silva (thiago@redhat.com) +Thiago da Silva (thiagodasilva@gmail.com) Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) Tim Burke (tim.burke@gmail.com) +Timur Alperovich (timuralp@swiftstack.com) Tong Li (litong01@us.ibm.com) Tony Breeds (tony@bakeyournoodle.com) +Tovin Seven (vinhnt@vn.fujitsu.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) +Vitaly Gridnev (vgridnev@mirantis.com) +Vu Cong Tuan (tuanvc@vn.fujitsu.com) +wangqi (wang.qi@99cloud.net) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) +wu.chunyang (wu.chunyang@99cloud.net) YangLei (yanglyy@cn.ibm.com) yangxurong (yangxurong@huawei.com) You Yamagata (bi.yamagata@gmail.com) @@ -141,3 +162,5 @@ zhang-jinnan (ben.os@99cloud.net) zhangyanxian (zhangyanxianmail@163.com) zheng yin (yin.zheng@easystack.cn) Zhenguo Niu (zhenguo@unitedstack.com) +ZhijunWei (wzj334965317@outlook.com) +zhubx007 (zhu.boxiang@99cloud.net) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ec32d85..0bde968 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,13 +1,13 @@ If you would like to contribute to the development of OpenStack, you must follow the steps in this page: - http://docs.openstack.org/infra/manual/developers.html + https://docs.openstack.org/infra/manual/developers.html Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at: - http://docs.openstack.org/infra/manual/developers.html#development-workflow. + https://docs.openstack.org/infra/manual/developers.html#development-workflow Gerrit is the review system used in the OpenStack projects. We're sorry, but we won't be able to respond to pull requests submitted through @@ -1,3 +1,95 @@ +3.7.0 +----- + +* Added the delimiter keyword parameter to `get_account()` to match the + functionality of `get_container()`. + +* Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + +* Various other minor bug fixes and improvements. + + +3.6.0 +----- + +* Add the `--prompt` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + `--key`, `--os-password` or an environment variable will be ignored. + +* Added bash completion support to the `swift` CLI. Enable this by sourcing + the included `tools/swift.bash_completion` file. Make it permanent by + including this file in the system's `/etc/bash_completion.d` directory. + +* Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported in Swift 2.19.0 or later. + +* The client.py SDK now supports a `query_string` option on the + `head_object()` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + +* The client.py SDK now respects `region_name` when using sessions. + +* Added a `.close()` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + +* Fixed a bug where using `--debug` in the CLI with unicode account names + would cause a client crash. + +* Make OS_AUTH_URL work in DevStack (for testing) by default. + +* Dropped Python 3.4 testing. + +* Various other minor bug fixes and improvements. + + +3.5.0 +----- + +* Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + +* The ``--meta`` option can now be set on the upload command. + +* Updated PyPy test dependency references to be more accurate + on different distros. + +* Various other minor bug fixes and improvements. + +3.4.0 +----- + +* The `swift` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + `--object-name` must be given when content is loaded from stdin. + +* Tolerate RFC-compliant ETags returned from the server. + +* Skip checksum validation on partial downloads. + +* Buffer reads from disk, resulting in much faster upload throughput. + +* Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + +* Added an option to ignore mtime metadata entry (`--ignore-mtime`). + +* When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + +* Expose `--prefix` as an option for st_delete. + +* Imported docs content from openstack-manuals project. + +* Various other minor bug fixes and improvements. + 3.3.0 ----- @@ -2,8 +2,8 @@ Team and repository tags ======================== -.. image:: https://governance.openstack.org/badges/python-swiftclient.svg - :target: https://governance.openstack.org/reference/tags/index.html +.. image:: https://governance.openstack.org/tc/badges/python-swiftclient.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html .. Change things from this point on @@ -11,26 +11,22 @@ Python bindings to the OpenStack Object Storage API =================================================== .. image:: https://img.shields.io/pypi/v/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ + :target: https://pypi.org/project/python-swiftclient/ :alt: Latest Version -.. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ - :alt: Downloads - This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). Development takes place via the usual OpenStack processes as outlined in the `OpenStack wiki`__. -__ http://docs.openstack.org/infra/manual/developers.html +__ https://docs.openstack.org/infra/manual/developers.html This code is based on the original client previously included with `OpenStack's Swift`__ The python-swiftclient is licensed under the Apache License like the rest of OpenStack. -__ http://github.com/openstack/swift +__ https://github.com/openstack/swift * Free software: Apache license * `PyPI`_ - package installation @@ -41,16 +37,17 @@ __ http://github.com/openstack/swift * `Source`_ * `Specs`_ * `How to Contribute`_ +* `Release Notes`_ -.. _PyPI: https://pypi.python.org/pypi/python-swiftclient -.. _Online Documentation: http://docs.openstack.org/developer/python-swiftclient +.. _PyPI: https://pypi.org/project/python-swiftclient +.. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient -.. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient -.. _How to Contribute: http://docs.openstack.org/infra/manual/developers.html -.. _Specs: http://specs.openstack.org/openstack/swift-specs/ - +.. _Source: https://opendev.org/openstack/python-swiftclient +.. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html +.. _Specs: https://specs.openstack.org/openstack/swift-specs/ +.. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient .. contents:: Contents: :local: @@ -1,6 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; -# see http://docs.openstack.org/infra/bindep/ for additional information. +# see https://docs.openstack.org/infra/bindep/ for additional information. -curl -pypy [test] -pypy-dev [test] +pypy [test !platform:fedora] +pypy-dev [test platform:dpkg] +pypy-devel [test platform:rpm !platform:fedora] diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 1f288d6..00e1440 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the remaining args. The \-c or \-\-changed is an option that will only upload files that have changed since the last upload. The \-\-object\-name <object\-name> is an option that will upload file and name object to <object\-name> or upload dir -and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size> -and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more). +and use <object\-name> as object prefix. If the file name is "-", reads the +content from standard input. In this case, \-\-object\-name is required and no +other files may be given. The \-S <size> or \-\-segment\-size <size> and +\-\-leave\-segments and others are options as well (see swift upload \-\-help +for more). .RE \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] @@ -102,6 +105,9 @@ with \-\-no-download actually not to write anything to disk. The \-\-ignore-checksum is an option that turns off checksum validation. You can specify optional headers with the repeatable cURL-like option \-H [\-\-header]. For more details and options see swift download \-\-help. +The \-\-ignore\-mtime option ignores the x\-object\-meta\-mtime metadata entry +on the object (if present) and instead creates the downloaded files with +fresh atime and mtime values. .RE \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] @@ -205,4 +211,4 @@ swift \-A https://127.0.0.1:443/auth/v1.0 \-U swiftops:swiftops \-K swiftops sta .SH DOCUMENTATION .LP More in depth documentation about OpenStack Swift as a whole can be found at -.BI https://docs.openstack.org/developer/swift +.BI https://docs.openstack.org/swift/latest/ diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..6cdad2a --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,5 @@ +keystoneauth1>=3.4.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.20.0 # Apache-2.0 diff --git a/doc/source/_static/.gitignore b/doc/source/_static/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/doc/source/_static/.gitignore diff --git a/doc/source/cli.rst b/doc/source/cli.rst deleted file mode 100644 index 8d80d1b..0000000 --- a/doc/source/cli.rst +++ /dev/null @@ -1,442 +0,0 @@ -==== -CLI -==== - -The ``swift`` tool is a command line utility for communicating with an OpenStack -Object Storage (swift) environment. It allows one to perform several types of -operations. - -Authentication -~~~~~~~~~~~~~~ - -This section covers the options for authenticating with a swift -object store. The combinations of options required for each authentication -version are detailed below, but are just a subset of those that can be used -to successfully authenticate. These are the most common and recommended -combinations. - -You should obtain the details of your authentication version and credentials -from your storage provider. These details should make it clearer which of the -authentication sections below are most likely to allow you to connect to your -storage account. - -Keystone v3 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-name project1 --os-project-domain-name domain1 \ - --os-username user --os-user-domain-name domain1 \ - --os-password password list - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-id 0123456789abcdef0123456789abcdef \ - --os-user-id abcdef0123456789abcdef0123456789 \ - --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following combinations of environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=3 - OS_USERNAME=user - OS_USER_DOMAIN_NAME=domain1 - OS_PASSWORD=password - OS_PROJECT_NAME=project1 - OS_PROJECT_DOMAIN_NAME=domain1 - OS_AUTH_URL=https://api.example.com:5000/v3 - - ST_AUTH_VERSION=3 - OS_USER_ID=abcdef0123456789abcdef0123456789 - OS_PASSWORD=password - OS_PROJECT_ID=0123456789abcdef0123456789abcdef - OS_AUTH_URL=https://api.example.com:5000/v3 - -Keystone v2 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v2.0 \ - --os-tenant-name tenant \ - --os-username user --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=2.0 - OS_USERNAME=user - OS_PASSWORD=password - OS_TENANT_NAME=tenant - OS_AUTH_URL=https://api.example.com:5000/v2.0 - -Legacy auth systems -------------------- - -You can configure swift to work with any number of other authentication systems -that we will not cover in this document. If your storage provider is not using -Keystone to provide access tokens, please contact them for instructions on the -required options. It is likely that the options will need to be specified as -below: - -.. code-block:: bash - - swift -A https://api.example.com/v1.0 -U user -K api_key list - -Specifying the options above manually on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=1.0 - ST_AUTH=https://api.example.com/v1.0 - ST_USER=user - ST_KEY=key - -It is also possible that you need to use a completely separate auth system, in which -case ``swiftclient`` cannot request a token for you. In this case you should make the -authentication request separately and access your storage using the token and -storage URL options shown below: - -.. code-block:: bash - - swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ - list - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. - -CLI commands -~~~~~~~~~~~~ - -Stat ----- - - ``stat [container [object]]`` - - Displays information for the account, container, or object depending on - the arguments given (if any). In verbose mode, the storage URL and the - authentication token are displayed as well. - -List ----- - - ``list [command-options] [container]`` - - Lists the containers for the account or the objects for a container. - The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only - list items beginning with that prefix. The ``-d <delimiter>`` or - ``--delimiter <delimiter>`` is an option (for container listings only) - that will roll up items with the given delimiter (see `OpenStack Swift - general documentation <http://docs.openstack.org/developer/swift/>` for - what this means). - - The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` - and ``ls -lh``, the latter providing sizes in human readable format - (For example: ``3K``, ``12M``, etc). The latter two switches use more - overhead to retrieve the displayed details, which is directly proportional - to the number of container or objects listed. - -Upload ------- - - ``upload [command-options] container file_or_directory [file_or_directory] [...]`` - - Uploads the files and directories specified by the remaining arguments to the - given container. The ``-c`` or ``--changed`` is an option that will only - upload files that have changed since the last upload. The - ``--object-name <object-name>`` is an option that will upload a file and - name object to ``<object-name>`` or upload a directory and use ``<object-name>`` - as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and - ``--leave-segments`` are options as well (see ``--help`` for more). - -Post ----- - - ``post [command-options] [container] [object]`` - - Updates meta information for the account, container, or object depending - on the arguments given. If the container is not found, the ``swiftclient`` - will create it automatically, but this is not true for accounts and - objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl - <read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. - The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, - and is used to define the user metadata items to set in the form ``Name:Value``. - You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` - - For more information about ACL formats see the documentation: - `ACLs <http://docs.openstack.org/developer/swift/misc.html#acls/>`_. - -Download --------- - - ``download [command-options] [container] [object] [object] [...]`` - - Downloads everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. For a - single object download, you may use the ``-o <filename>`` or ``--output <filename>`` - option to redirect the output to a specific file or ``-`` to - redirect to stdout. The ``--ignore-checksum`` is an option that turn off - checksum validation. You can specify optional headers with the repeatable - cURL-like option ``-H [--header <name:value>]``. - -Delete ------- - - ``delete [command-options] [container] [object] [object] [...]`` - - Deletes everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. Segments - of manifest objects will be deleted as well, unless you specify the - ``--leave-segments`` option. - -Copy ----- - - ``copy [command-options] container object`` - - Copies an object to a new destination or adds user metadata to an object. Depending - on the options supplied, you can preserve existing metadata in contrast to the post - command. The ``--destination`` option sets the copy target destination in the form - ``/container/object``. If not set, the object will be copied onto itself which is useful - for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy - an object without existing user meta data, and the ``-m`` or ``--meta`` option - to define user meta data items to set in the form ``Name:Value``. You can repeat - this option. For example: ``copy -m Color:Blue -m Size:Large``. - -Capabilities ------------- - - ``capabilities [proxy-url]`` - - Displays cluster capabilities. The output includes the list of the - activated Swift middlewares as well as relevant options for each ones. - Additionally the command displays relevant options for the Swift core. If - the ``proxy-url`` option is not provided, the storage URL retrieved after - authentication is used as ``proxy-url``. - -Tempurl -------- - - ``tempurl [command-options] [method] [time] [path] [key]`` - - Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to - allow for this temporary URL that is usually ``GET` or ``PUT``. ``time`` option sets - the amount of time the temporary URL will be valid for. - ``time`` can be specified as an integer, denoting the number of seconds - from now on until the URL shall be valid; or, if ``--absolute`` - is passed, the Unix timestamp when the temporary URL will expire. - But beyond that, ``time`` can also be specified as an ISO 8601 timestamp - in one of following formats: - - i) Complete date: YYYY-MM-DD (eg 1997-07-16) - - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) - - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) - - Please be aware that if you don't provide the UTC designator (i.e., Z) - the timestamp is generated using your local timezone. If only a date is - specified, the time part used will equal to ``00:00:00``. - - ``path`` option sets the full path to the Swift object. - Example: ``/v1/AUTH_account/c/o``. ``key`` option is - the secret temporary URL key set on the Swift cluster. To set a key, run - ``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary - URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not - forget to append the desired objectname at the end of the path portion (and before the - query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the - URL by using the ``--iso8601`` option. - -Auth ----- - - ``auth`` - - Display authentication variables in shell friendly format. Command to run to export storage - URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. - Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic - authentication: ``swift auth -v -U test:tester -K testing``. - -Examples -~~~~~~~~ - -In this section we present some example usage of the ``swift`` CLI. To keep the -examples as short as possible, these examples assume that the relevant authentication -options have been set using environment variables. You can obtain the full list of -commands and options available in the ``swift`` CLI by executing the following: - -.. code-block:: bash - - > swift --help - > swift <command> --help - -Simple examples ---------------- - -List the existing swift containers: - -.. code-block:: bash - - > swift list - - container_1 - -Create a new container: - -.. code-block:: bash - - > swift post TestContainer - -Upload an object into a container: - -.. code-block:: bash - - > swift upload TestContainer testSwift.txt - - testSwift.txt - -List the contents of a container: - -.. code-block:: bash - - > swift list TestContainer - - testSwift.txt - -Copy an object to new destination: - -.. code-block:: bash - - > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt - - SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt - -Delete an object from a container: - -.. code-block:: bash - - > swift delete TestContainer testSwift.txt - - testSwift.txt - -Delete a container: - -.. code-block:: bash - - > swift delete TestContainer - - TestContainer - -Display auth related authentication variables in shell friendly format: - -.. code-block:: bash - - > swift auth - - export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 - export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 - -Download an object from a container: - -.. code-block:: bash - - > swift download TestContainer testSwift.txt - - testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name <object-name> is an option that will upload - file and name object to <object-name> or upload directory and use <object-name> as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. - -For example: - -.. code-block:: bash - - > swift upload TestContainer /home/swift/testSwift/testSwift.txt - - home/swift/testSwift/testSwift.txt - - > swift list TestContainer - - home/swift/testSwift/testSwift.txt - -More complex examples ---------------------- - -Swift has a single object size limit of 5GiB. In order to upload files larger -than this, we must create a large object that consists of smaller segments. -The example below shows how to upload a large video file as a static large -object in 1GiB segments: - -.. code-block:: bash - - > swift upload videos --use-slo --segment-size 1G myvideo.mp4 - - myvideo.mp4 segment 8 - myvideo.mp4 segment 4 - myvideo.mp4 segment 2 - myvideo.mp4 segment 7 - myvideo.mp4 segment 0 - myvideo.mp4 segment 1 - myvideo.mp4 segment 3 - myvideo.mp4 segment 6 - myvideo.mp4 segment 5 - myvideo.mp4 - -This command will upload segments to a container named ``videos_segments``, and -create a manifest file describing the entire object in the ``videos`` container. -For more information on large objects, see the documentation `here -<http://docs.openstack.org/developer/swift/overview_large_objects.html>`_. - -.. code-block:: bash - - > swift list videos - - myvideo.mp4 - - > swift list videos_segments - - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 - -Firstly, the key should be set, then generate a temporary URL for a Swift object: - -.. code-block:: bash - - > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" - - > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ - /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 - - /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ - 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..1762989 --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,958 @@ +==== +CLI +==== + +The ``swift`` tool is a command line utility for communicating with an OpenStack +Object Storage (swift) environment. It allows one to perform several types of +operations. + + +For help on a specific :command:`swift` command, enter: + +.. code-block:: console + + $ swift COMMAND --help + +.. _swift_command_usage: + +swift usage +~~~~~~~~~~~ + +.. code-block:: console + + Usage: swift [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + <subcommand> [--help] [<subcommand options>] + +**Subcommands:** + +``delete`` + Delete a container or objects within a container. + +``download`` + Download objects from containers. + +``list`` + Lists the containers for the account or the objects + for a container. + +``post`` + Updates meta information for the account, container, + or object; creates containers if not present. + +``copy`` + Copies object, optionally adds meta + +``stat`` + Displays information for the account, container, + or object. + +``upload`` + Uploads files or directories to the given container. + +``capabilities`` + List cluster capabilities. + +``tempurl`` + Create a temporary URL. + +``auth`` + Display auth related environment variables. + +.. _swift_command_options: + +swift optional arguments +~~~~~~~~~~~~~~~~~~~~~~~~ + +``--version`` + show program's version number and exit + +``-h, --help`` + show this help message and exit + +``--os-help`` + Show OpenStack authentication options. + +``-s, --snet`` + Use SERVICENET internal network. + +``-v, --verbose`` + Print more info. + +``--debug`` + Show the curl commands and results of all http queries + regardless of result status. + +``--info`` + Show the curl commands and results of all http queries + which return an error. + +``-q, --quiet`` + Suppress status output. + +``-A AUTH, --auth=AUTH`` + URL for obtaining an auth token. + +``-V AUTH_VERSION, --auth-version=AUTH_VERSION, --os-identity-api-version=AUTH_VERSION`` + Specify a version for authentication. Defaults to + ``env[ST_AUTH_VERSION]``, ``env[OS_AUTH_VERSION]``, + ``env[OS_IDENTITY_API_VERSION]`` or 1.0. + +``-U USER, --user=USER`` + User name for obtaining an auth token. + +``-K KEY, --key=KEY`` + Key for obtaining an auth token. + +``-R RETRIES, --retries=RETRIES`` + The number of times to retry a failed connection. + +``--insecure`` + Allow swiftclient to access servers without having to + verify the SSL certificate. Defaults to + ``env[SWIFTCLIENT_INSECURE]`` (set to 'true' to enable). + +``--no-ssl-compression`` + This option is deprecated and not used anymore. SSL + compression should be disabled by default by the + system SSL library. + +``--prompt`` + Prompt user to enter a password which overrides any password supplied via + ``--key``, ``--os-password`` or environment variables. + +Authentication +~~~~~~~~~~~~~~ + +This section covers the options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are just a subset of those that can be used +to successfully authenticate. These are the most common and recommended +combinations. + +You should obtain the details of your authentication version and credentials +from your storage provider. These details should make it clearer which of the +authentication sections below are most likely to allow you to connect to your +storage account. + +Keystone v3 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-name project1 --os-project-domain-name domain1 \ + --os-username user --os-user-domain-name domain1 \ + --os-password password list + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-id 0123456789abcdef0123456789abcdef \ + --os-user-id abcdef0123456789abcdef0123456789 \ + --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following combinations of environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=3 + OS_USERNAME=user + OS_USER_DOMAIN_NAME=domain1 + OS_PASSWORD=password + OS_PROJECT_NAME=project1 + OS_PROJECT_DOMAIN_NAME=domain1 + OS_AUTH_URL=https://api.example.com:5000/v3 + + ST_AUTH_VERSION=3 + OS_USER_ID=abcdef0123456789abcdef0123456789 + OS_PASSWORD=password + OS_PROJECT_ID=0123456789abcdef0123456789abcdef + OS_AUTH_URL=https://api.example.com:5000/v3 + +Keystone v2 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v2.0 \ + --os-tenant-name tenant \ + --os-username user --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=2.0 + OS_USERNAME=user + OS_PASSWORD=password + OS_TENANT_NAME=tenant + OS_AUTH_URL=https://api.example.com:5000/v2.0 + +Legacy auth systems +------------------- + +You can configure swift to work with any number of other authentication systems +that we will not cover in this document. If your storage provider is not using +Keystone to provide access tokens, please contact them for instructions on the +required options. It is likely that the options will need to be specified as +below: + +.. code-block:: bash + + swift -A https://api.example.com/v1.0 -U user -K api_key list + +Specifying the options above manually on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=1.0 + ST_AUTH=https://api.example.com/v1.0 + ST_USER=user + ST_KEY=key + +It is also possible that you need to use a completely separate auth system, in which +case ``swiftclient`` cannot request a token for you. In this case you should make the +authentication request separately and access your storage using the token and +storage URL options shown below: + +.. code-block:: bash + + swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ + list + +.. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +CLI commands +~~~~~~~~~~~~ + +.. _swift_auth: + +Auth +---- + +.. code-block:: console + + Usage: swift auth + +Display authentication variables in shell friendly format. Command to run to export storage +URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. +Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic +authentication: ``swift auth -v -U test:tester -K testing``. + +.. _swift_stat: + +swift stat +---------- + +.. code-block:: console + + Usage: swift stat [--lh] [--header <header:value>] + [<container> [<object>]] + +Displays information for the account, container, or object depending on +the arguments given (if any). In verbose mode, the storage URL and the +authentication token are displayed as well. + +**Positional arguments:** + +``[container]`` + Name of container to stat from. + +``[object]`` + Name of object to stat. + +**Optional arguments:** + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-H, --header <header:value>`` + Adds a custom request header to use for stat. + +.. _swift_list: + +swift list +---------- + +.. code-block:: console + + Usage: swift list [--long] [--lh] [--totals] [--prefix <prefix>] + [--delimiter <delimiter>] [--header <header:value>] + [<container>] + +Lists the containers for the account or the objects for a container. +The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only +list items beginning with that prefix. The ``-d <delimiter>`` or +``--delimiter <delimiter>`` is an option (for container listings only) +that will roll up items with the given delimiter (see `OpenStack Swift +general documentation <https://docs.openstack.org/swift/latest/>` for +what this means). + +The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` +and ``ls -lh``, the latter providing sizes in human readable format +(For example: ``3K``, ``12M``, etc). The latter two switches use more +overhead to retrieve the displayed details, which is directly proportional +to the number of container or objects listed. + +**Positional arguments:** + +``[container]`` + Name of container to list object in. + +**Optional arguments:** + +``-l, --long`` + Long listing format, similar to ls -l. + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-t, --totals`` + Used with -l or --lh, only report totals. + +``-p <prefix>, --prefix <prefix>`` + Only list items beginning with the prefix. + +``-d <delim>, --delimiter <delim>`` + Roll up items with the given delimiter. For containers + only. See OpenStack Swift API documentation for what + this means. + +``-H, --header <header:value>`` + Adds a custom request header to use for listing. + +.. _swift_upload: + +swift upload +------------ + +.. code-block:: console + + Usage: swift upload [--changed] [--skip-identical] [--segment-size <size>] + [--segment-container <container>] [--leave-segments] + [--object-threads <thread>] [--segment-threads <threads>] + [--header <header>] [--use-slo] [--ignore-checksum] + [--object-name <object-name>] + <container> <file_or_directory> [<file_or_directory>] [...] + +Uploads the files and directories specified by the remaining arguments to the +given container. The ``-c`` or ``--changed`` is an option that will only +upload files that have changed since the last upload. The +``--object-name <object-name>`` is an option that will upload a file and +name object to ``<object-name>`` or upload a directory and use ``<object-name>`` +as object prefix. If the file name is "-", client reads content from standard +input. In this case ``--object-name`` is required to set the name of the object +and no other files may be given. The ``-S <size>`` or ``--segment-size <size>`` +and ``--leave-segments`` are options as well (see ``--help`` for more). + +**Positional arguments:** + +``<container>`` + Name of container to upload to. + +``<file_or_directory>`` + Name of file or directory to upload. Specify multiple + times for multiple uploads. + +**Optional arguments:** + +``-c, --changed`` + Only upload files that have changed since the last + upload. + +``--skip-identical`` + Skip uploading files that are identical on both sides. + +``-S, --segment-size <size>`` + Upload files in segments no larger than <size> (in + Bytes) and then create a "manifest" file that will + download all the segments as if it were the original + file. + +``--segment-container <container>`` + Upload the segments into the specified container. If + not specified, the segments will be uploaded to a + <container>_segments container to not pollute the + main <container> listings. + +``--leave-segments`` + Indicates that you want the older segments of manifest + objects left alone (in the case of overwrites). + +``--object-threads <threads>`` + Number of threads to use for uploading full objects. + Default is 10. + +``--segment-threads <threads>`` + Number of threads to use for uploading object segments. + Default is 10. + +``-H, --header <header:value>`` + Adds a customized request header. This option may be + repeated. Example: -H "content-type:text/plain" + -H "Content-Length: 4000". + +``--use-slo`` + When used in conjunction with --segment-size it will + create a Static Large Object instead of the default + Dynamic Large Object. + +``--object-name <object-name>`` + Upload file and name object to <object-name> or upload + dir and use <object-name> as object prefix instead of + folder name. + +``--ignore-checksum`` + Turn off checksum validation for uploads. + + +.. _swift_post: + +swift post +---------- + +.. code-block:: console + + Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>] + [--sync-key <sync-key>] [--meta <name:value>] + [--header <header>] + [<container> [<object>]] + +Updates meta information for the account, container, or object depending +on the arguments given. If the container is not found, the ``swiftclient`` +will create it automatically, but this is not true for accounts and +objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl +<read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. +The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, +and is used to define the user metadata items to set in the form ``Name:Value``. +You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` + +For more information about ACL formats see the documentation: +`ACLs <https://docs.openstack.org/swift/latest/misc.html#acls>`_. + +**Positional arguments:** + +``[container]`` + Name of container to post to. + +``[object]`` + Name of object to post. + +**Optional arguments:** + +``-r, --read-acl <acl>`` + Read ACL for containers. Quick summary of ACL syntax: + ``.r:*``, ``.r:-.example.com``, ``.r:www.example.com``, + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-w, --write-acl <acl>`` + Write ACL for containers. Quick summary of ACL syntax: + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-t, --sync-to <sync-to>`` + Sync To for containers, for multi-cluster replication. + +``-k, --sync-key <sync-key>`` + Sync Key for containers, for multi-cluster replication. + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. + This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_download: + +swift download +-------------- + +.. code-block:: console + + Usage: swift download [--all] [--marker <marker>] [--prefix <prefix>] + [--output <out_file>] [--output-dir <out_directory>] + [--object-threads <threads>] [--ignore-checksum] + [--container-threads <threads>] [--no-download] + [--skip-identical] [--remove-prefix] + [--header <header:value>] [--no-shuffle] + [<container> [<object>] [...]] + +Downloads everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. For a +single object download, you may use the ``-o <filename>`` or ``--output <filename>`` +option to redirect the output to a specific file or ``-`` to +redirect to stdout. The ``--ignore-checksum`` is an option that turn off +checksum validation. You can specify optional headers with the repeatable +cURL-like option ``-H [--header <name:value>]``. ``--ignore-mtime`` ignores the +``x-object-meta-mtime`` metadata entry on the object (if present) and instead +creates the downloaded files with fresh atime and mtime values. + +**Positional arguments:** + +``<container>`` + Name of container to download from. To download a + whole account, omit this and specify --all. + +``<object>`` + Name of object to download. Specify multiple times + for multiple objects. Omit this to download all + objects from the container. + +**Optional arguments:** + +``-a, --all`` + Indicates that you really want to download + everything in the account. + +``-m, --marker <marker>`` + Marker to use when starting a container or account + download. + +``-p, --prefix <prefix>`` + Only download items beginning with <prefix> + +``-r, --remove-prefix`` + An optional flag for --prefix <prefix>, use this + option to download items without <prefix> + +``-o, --output <out_file>`` + For a single file download, stream the output to + <out_file>. Specifying "-" as <out_file> will + redirect to stdout. + +``-D, --output-dir <out_directory>`` + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. + +``--object-threads <threads>`` + Number of threads to use for downloading objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for downloading containers. + Default is 10. + +``--no-download`` + Perform download(s), but don't actually write anything + to disk. + +``-H, --header <header:value>`` + Adds a customized request header to the query, like + "Range" or "If-Match". This option may be repeated. + + Example: --header "content-type:text/plain" + +``--skip-identical`` + Skip downloading files that are identical on both + sides. + +``--ignore-checksum`` + Turn off checksum validation for downloads. + +``--no-shuffle`` + By default, when downloading a complete account or + container, download order is randomised in order to + reduce the load on individual drives when multiple + clients are executed simultaneously to download the + same set of objects (e.g. a nightly automated download + script to multiple servers). Enable this option to + submit download jobs to the thread pool in the order + they are listed in the object store. + +.. _swift_delete: + +swift delete +------------ + +.. code-block:: console + + Usage: swift delete [--all] [--leave-segments] + [--object-threads <threads>] + [--container-threads <threads>] + [--header <header:value>] + [<container> [<object>] [...]] + +Deletes everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. Segments +of manifest objects will be deleted as well, unless you specify the +``--leave-segments`` option. + +**Positional arguments:** + +``[<container>]`` + Name of container to delete from. + +``[<object>]`` + Name of object to delete. Specify multiple times + for multiple objects. + +**Optional arguments:** + +``-a, --all`` + Delete all containers and objects. + +``--leave-segments`` + Do not delete segments of manifest objects. + +``-H, --header <header:value>`` + Adds a custom request header to use for deleting + objects or an entire container. + + +``--object-threads <threads>`` + Number of threads to use for deleting objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for deleting containers. + Default is 10. + +.. _swift_copy: + +swift copy +---------- + +.. code-block:: console + + Usage: swift copy [--destination </container/object>] [--fresh-metadata] + [--meta <name:value>] [--header <header>] <container> + <object> [<object>] [...] + +Copies an object to a new destination or adds user metadata to an object. Depending +on the options supplied, you can preserve existing metadata in contrast to the post +command. The ``--destination`` option sets the copy target destination in the form +``/container/object``. If not set, the object will be copied onto itself which is useful +for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy +an object without existing user meta data, and the ``-m`` or ``--meta`` option +to define user meta data items to set in the form ``Name:Value``. You can repeat +this option. For example: ``copy -m Color:Blue -m Size:Large``. + +**Positional arguments:** + +``<container>`` + Name of container to copy from. + +``<object>`` + Name of object to copy. Specify multiple times for multiple objects + +**Optional arguments:** + +``-d, --destination </container[/object]>`` + The container and name of the destination object. Name + of destination object can be omitted, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + +``-M, --fresh-metadata`` + Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_capabilities: + +swift capabilities +------------------ + +.. code-block:: console + + Usage: swift capabilities [--json] [<proxy_url>] + +Displays cluster capabilities. The output includes the list of the +activated Swift middlewares as well as relevant options for each ones. +Additionally the command displays relevant options for the Swift core. If +the ``proxy-url`` option is not provided, the storage URL retrieved after +authentication is used as ``proxy-url``. + +**Optional positional arguments:** + +``<proxy_url>`` + Proxy URL of the cluster to retrieve capabilities. + +``--json`` + Print the cluster capabilities in JSON format. + +.. _swift_tempurl: + +swift tempurl +------------- + +.. code-block:: console + + Usage: swift tempurl [--absolute] [--prefix-based] + <method> <seconds> <path> <key> + +Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to +allow for this temporary URL that is usually ``GET`` or ``PUT``. ``time`` option sets +the amount of time the temporary URL will be valid for. +``time`` can be specified as an integer, denoting the number of seconds +from now on until the URL shall be valid; or, if ``--absolute`` +is passed, the Unix timestamp when the temporary URL will expire. +But beyond that, ``time`` can also be specified as an ISO 8601 timestamp +in one of following formats: + +i) Complete date: YYYY-MM-DD (e.g. 1997-07-16) + +ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (e.g. 1997-07-16T19:20:30) + +iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (e.g. 1997-07-16T19:20:30Z) + +Please be aware that if you don't provide the UTC designator (i.e., Z) +the timestamp is generated using your local timezone. If only a date is +specified, the time part used will equal to ``00:00:00``. + +``path`` option sets the full path to the Swift object. +Example: ``/v1/AUTH_account/c/o``. ``key`` option is +the secret temporary URL key set on the Swift cluster. To set a key, run +``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary +URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not +forget to append the desired objectname at the end of the path portion (and before the +query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the +URL by using the ``--iso8601`` option. + +**Positional arguments:** + +``<method>`` + An HTTP method to allow for this temporary URL. + Usually 'GET' or 'PUT'. + +``<seconds>`` + The amount of time in seconds the temporary URL will be + valid for; or, if --absolute is passed, the Unix + timestamp when the temporary URL will expire. + +``<path>`` + The full path to the Swift object. + + Example: /v1/AUTH_account/c/o + or: http://saio:8080/v1/AUTH_account/c/o + +``<key>`` + The secret temporary URL key set on the Swift cluster. + To set a key, run 'swift post -m + "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"' + +**Optional arguments:** + +``--absolute`` + Interpret the <seconds> positional argument as a Unix + timestamp rather than a number of seconds in the + future. + +``--prefix-based`` + If present, a prefix-based tempURL will be generated. + +Examples +~~~~~~~~ + +In this section we present some example usage of the ``swift`` CLI. To keep the +examples as short as possible, these examples assume that the relevant authentication +options have been set using environment variables. You can obtain the full list of +commands and options available in the ``swift`` CLI by executing the following: + +.. code-block:: bash + + > swift --help + > swift <command> --help + +Simple examples +--------------- + +List the existing swift containers: + +.. code-block:: bash + + > swift list + + container_1 + +Create a new container: + +.. code-block:: bash + + > swift post TestContainer + +Upload an object into a container: + +.. code-block:: bash + + > swift upload TestContainer testSwift.txt + + testSwift.txt + +List the contents of a container: + +.. code-block:: bash + + > swift list TestContainer + + testSwift.txt + +Copy an object to new destination: + +.. code-block:: bash + + > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt + + SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt + +Delete an object from a container: + +.. code-block:: bash + + > swift delete TestContainer testSwift.txt + + testSwift.txt + +Delete a container: + +.. code-block:: bash + + > swift delete TestContainer + + TestContainer + +Display auth related authentication variables in shell friendly format: + +.. code-block:: bash + + > swift auth + + export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 + export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 + +Download an object from a container: + +.. code-block:: bash + + > swift download TestContainer testSwift.txt + + testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] + +.. note:: + + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. + +For example: + +.. code-block:: bash + + > swift upload TestContainer /home/swift/testSwift/testSwift.txt + + home/swift/testSwift/testSwift.txt + + > swift list TestContainer + + home/swift/testSwift/testSwift.txt + +More complex examples +--------------------- + +Swift has a single object size limit of 5GiB. In order to upload files larger +than this, we must create a large object that consists of smaller segments. +The example below shows how to upload a large video file as a static large +object in 1GiB segments: + +.. code-block:: bash + + > swift upload videos --use-slo --segment-size 1G myvideo.mp4 + + myvideo.mp4 segment 8 + myvideo.mp4 segment 4 + myvideo.mp4 segment 2 + myvideo.mp4 segment 7 + myvideo.mp4 segment 0 + myvideo.mp4 segment 1 + myvideo.mp4 segment 3 + myvideo.mp4 segment 6 + myvideo.mp4 segment 5 + myvideo.mp4 + +This command will upload segments to a container named ``videos_segments``, and +create a manifest file describing the entire object in the ``videos`` container. +For more information on large objects, see the documentation `here +<https://docs.openstack.org/swift/latest/overview_large_objects.html>`_. + +.. code-block:: bash + + > swift list videos + + myvideo.mp4 + + > swift list videos_segments + + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 + +Firstly, the key should be set, then generate a temporary URL for a Swift object: + +.. code-block:: bash + + > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" + + > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ + /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 + + /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ + 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/conf.py b/doc/source/conf.py index 3505f13..85dd81e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -31,8 +31,11 @@ sys.path.insert(0, ROOT) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'oslosphinx'] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'openstackdocstheme'] autoclass_content = 'both' autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] @@ -50,17 +53,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Swiftclient' copyright = u'2013-2016 OpenStack, LLC.' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -import swiftclient.version -release = swiftclient.version.version_string -version = swiftclient.version.version_string - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -104,7 +98,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -#html_theme = 'nature' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/source/index.rst b/doc/source/index.rst index f123b7b..ab05c6b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,7 +16,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 - cli + cli/index service-api client-api @@ -39,17 +39,17 @@ Indices and tables License ~~~~~~~ - Copyright 2013 OpenStack, LLC. +Copyright 2013 OpenStack, LLC. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +* http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst index 926b1b9..d6f9819 100644 --- a/doc/source/introduction.rst +++ b/doc/source/introduction.rst @@ -16,41 +16,41 @@ Use Cases Alongside the command line tool, the ``python-swiftclient`` includes two levels of API: - * A low level client API that provides simple Python wrappers around the - various authentication mechanisms and the individual HTTP requests. - * A high level service API that provides methods for performing common - operations in parallel on a thread pool. +* A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. +* A high level service API that provides methods for performing common + operations in parallel on a thread pool. Example use cases: - * Uploading and retrieving data - Use the command line tool if you are simply uploading and downloading - files and directories to and from your filesystem. The command line tool - can be integrated into a shell script to automate tasks. - - * Integrating into an automated Python workflow - Use the ``SwiftService`` API to perform operations offered by the CLI - if your use case requires integration with a Python-based workflow. - This method offers greater control and flexibility over individual object - operations, such as the metadata set on each object. The ``SwiftService`` - class provides methods to perform multiple sets of operations against a - swift object store using a configurable shared thread pool. A single - instance of the ``SwiftService`` class can be shared between multiple - threads in your own code. - - * Developing an application in Python to access a swift object store - Use the ``SwiftService`` API to develop Python applications that use - swift to store and retrieve objects. A ``SwiftService`` instance provides - a configurable thread pool for performing all operations supported by the - CLI. - - * Fine-grained control over threading or the requests being performed - Use the ``Connection`` API if your use case requires fine grained control - over advanced features or you wish to use your own existing threading - model. Examples of advanced features requiring the use of the - ``Connection`` API include creating an SLO manifest that references - already existing objects, or fine grained control over the query strings - supplied with each HTTP request. +* Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + +* Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + +* Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + +* Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. Important considerations ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,19 +66,19 @@ the proper use case, and not treat the storage like a traditional filesystem. There are two main restrictions to bear in mind when designing an application that uses an object store: - * You cannot rename objects. Due to fact that the name of an object is one - of the factors that determines where the object and its replicas are stored, - renaming would require multiple copies of the data to be moved between - physical storage devices. If you want to rename an object you must upload - to the new location, or make a server side copy request to the new location, - and then delete the original. - - * You cannot modify objects. Objects are stored in multiple locations and - are checked for integrity based on the MD5 sum calculated during - upload. In order to modify the contents of an object, the entire desired - contents must be re-uploaded. In certain special cases it is possible to - work around this restriction using large objects, but no general - file-like access is available to modify a stored object. +* You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + +* You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. Objects cannot be locked ------------------------ diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index 8efd43a..cc4dcc2 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -26,10 +26,10 @@ the auth version based on the combination of options specified, but supplying options from multiple different auth versions can cause unexpected behaviour. - .. note:: +.. note:: - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. Keystone V3 ~~~~~~~~~~~ @@ -109,17 +109,17 @@ in this dictionary are described below, along with their defaults: Options ~~~~~~~ - ``retries``: ``5`` +``retries``: ``5`` The number of times that the library should attempt to retry HTTP actions before giving up and reporting a failure. - ``container_threads``: ``10`` +``container_threads``: ``10`` - ``object_dd_threads``: ``10`` +``object_dd_threads``: ``10`` - ``object_uu_threads``: ``10`` +``object_uu_threads``: ``10`` - ``segment_threads``: ``10`` +``segment_threads``: ``10`` The above options determine the size of the available thread pools for performing swift operations. Container operations (such as listing a container) operate in the container threads, and a similar pattern @@ -131,86 +131,86 @@ Options ``uu`` and ``dd``. This stands for "upload/update" and "download/delete", and the corresponding actions will be run on separate threads pools. - ``segment_size``: ``None`` +``segment_size``: ``None`` If specified, this option enables uploading of large objects. Should the object being uploaded be larger than 5G in size, this option is mandatory otherwise the upload will fail. This option should be specified as a size in bytes. - ``use_slo``: ``False`` +``use_slo``: ``False`` Used in combination with the above option, ``use_slo`` will upload large objects as static rather than dynamic. Only static large objects provide error checking for the downloaded object, so we recommend this option. - ``segment_container``: ``None`` +``segment_container``: ``None`` Allows the user to select the container into which large object segments will be uploaded. We do not recommend changing this value as it could make locating orphaned segments more difficult in the case of errors. - ``leave_segments``: ``False`` +``leave_segments``: ``False`` Setting this option to true means that when deleting or overwriting a large object, its segments will be left in the object store and must be cleaned up manually. This option can be useful when sharing large object segments between multiple objects in more advanced scenarios, but must be treated with care, as it could lead to ever increasing storage usage. - ``changed``: ``None`` +``changed``: ``None`` This option affects uploads and simply means that those objects which already exist in the object store will not be overwritten if the ``mtime`` and size of the source is the same as the existing object. - ``skip_identical``: ``False`` +``skip_identical``: ``False`` A slightly more thorough case of the above, but rather than ``mtime`` and size uses an object's ``MD5 sum``. - ``yes_all``: ``False`` +``yes_all``: ``False`` This options affects only download and delete, and in each case must be specified in order to download/delete the entire contents of an account. This option has no effect on any other calls. - ``no_download``: ``False`` +``no_download``: ``False`` This option only affects download and means that all operations proceed as normal with the exception that no data is written to disk. - ``header``: ``[]`` +``header``: ``[]`` Used with upload and post operations to set headers on objects. Headers are specified as colon separated strings, e.g. "content-type:text/plain". - ``meta``: ``[]`` +``meta``: ``[]`` Used to set metadata on an object similarly to headers. .. note:: Setting metadata is a destructive operation, so when updating one of many metadata values all desired metadata for an object must be re-applied. - ``long``: ``False`` +``long``: ``False`` Affects only list operations, and results in more metrics being made available in the results at the expense of lower performance. - ``fail_fast``: ``False`` +``fail_fast``: ``False`` Applies to delete and upload operations, and attempts to abort queued tasks in the event of errors. - ``prefix``: ``None`` +``prefix``: ``None`` Affects list operations; only objects with the given prefix will be returned/affected. It is not advisable to set at the service level, as those operations that call list to discover objects on which they should operate will also be affected. - ``delimiter``: ``None`` +``delimiter``: ``None`` Affects list operations, and means that listings only contain results up to the first instance of the delimiter in the object name. This is useful for working with objects containing '/' in their names to simulate folder structures. - ``dir_marker``: ``False`` +``dir_marker``: ``False`` Affects uploads, and allows empty 'pseudofolder' objects to be created when the source of an upload is ``None``. - ``checksum``: ``True`` +``checksum``: ``True`` Affects uploads and downloads. If set check md5 sum for the transfer. - ``shuffle``: ``False`` +``shuffle``: ``False`` When downloading objects, the default behaviour of the CLI is to shuffle lists of objects in order to spread the load on storage drives when multiple clients are downloading the same files to multiple locations (e.g. in the @@ -220,12 +220,12 @@ Options are downloaded in lexically-sorted order. Setting this option to ``True`` gives the same shuffling behaviour as the CLI. - ``destination``: ``None`` +``destination``: ``None`` When copying objects, this specifies the destination where the object will be copied to. The default of None means copy will be the same as source. - ``fresh_metadata``: ``None`` +``fresh_metadata``: ``None`` When copying objects, this specifies that the object metadata on the source will *not* be applied to the destination object - the destination object will have a new fresh set of metadata that includes diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..ae61948 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,45 @@ +alabaster==0.7.10 +Babel==2.3.4 +certifi==2018.1.18 +chardet==3.0.4 +coverage==4.0 +docutils==0.11 +dulwich==0.15.0 +extras==1.0.0 +fixtures==3.0.0 +flake8==2.2.4 +futures==3.0.0 +hacking==0.10.0 +idna==2.6 +imagesize==0.7.1 +iso8601==0.1.8 +Jinja2==2.10 +keystoneauth1==3.4.0 +linecache2==1.0.0 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==1.2.0 +netaddr==0.7.10 +openstackdocstheme==1.20.0 +oslo.config==1.2.0 +pbr==2.0.0 +pep8==1.5.7 +PrettyTable==0.7 +pyflakes==0.8.1 +Pygments==2.2.0 +python-keystoneclient==0.7.0 +python-mimeparse==1.6.0 +python-subunit==1.0.0 +pytz==2013.6 +PyYAML==3.12 +reno==2.5.0 +requests==1.1.0 +six==1.9.0 +snowballstemmer==1.2.1 +sphinx==1.6.2 +sphinxcontrib-websupport==1.0.1 +stestr==2.0.0 +testtools==2.2.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 diff --git a/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml new file mode 100644 index 0000000..0aae5cf --- /dev/null +++ b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml @@ -0,0 +1,20 @@ +--- +features: + + - The ``swift`` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + ``--object-name`` must be given when content is loaded from stdin. + - Tolerate RFC-compliant ETags returned from the server. + - Skip checksum validation on partial downloads. + - Buffer reads from disk, resulting in much faster upload throughput. + - > + Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + - Added an option to ignore mtime metadata entry (``--ignore-mtime``). + - > + When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + - Expose `--prefix` as an option for st_delete. + - Imported docs content from openstack-manuals project. + - Various other minor bug fixes and improvements. diff --git a/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml new file mode 100644 index 0000000..2e6b4ea --- /dev/null +++ b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + + - | + The ``--meta`` option can now be set on the upload command. + + - | + Updated PyPy test dependency references to be more accurate + on different distros. diff --git a/releasenotes/notes/360_notes-1ec385df13a3a735.yaml b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml new file mode 100644 index 0000000..8d82b06 --- /dev/null +++ b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml @@ -0,0 +1,40 @@ +--- +features: + - | + Add the ``--prompt`` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + ``--key`` , ``--os-password`` or an environment variable will be ignored. + + - | + Added bash completion support to the ``swift`` CLI. Enable this by sourcing + the included ``tools/swift.bash_completion`` file. Make it permanent by + including this file in the system's ``/etc/bash_completion.d`` directory. + + - | + Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported are Swift 2.19.0 or later. + + - | + The client.py SDK now supports a ``query_string`` option on the + ``head_object()`` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + + - | + The client.py SDK now respects ``region_name`` when using sessions. + + - | + Added a ``.close()`` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + + - | + Fixed a bug where using ``--debug`` in the CLI with unicode account names + would cause a client crash. + + - | + Make OS_AUTH_URL work in DevStack (for testing) by default. + + - | + Dropped Python 3.4 testing. + + - | + Various other minor bug fixes and improvements. diff --git a/releasenotes/notes/361_notes-59e020e68bcdd709.yaml b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml new file mode 100644 index 0000000..f6b4892 --- /dev/null +++ b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Added the delimiter keyword parameter to ``get_account()`` to match the + functionality of ``get_container()``. + + - | + Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + + - | + Various other minor bug fixes and improvements. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..c71f41d --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# swift documentation build configuration file, created by +# sphinx-quickstart on Mon Oct 3 17:01:55 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import datetime + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'reno.sphinxext', + 'openstackdocstheme', +] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +# todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# "<project> v<release> documentation" by default. +# +# html_title = u'swift v2.10.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'SwiftClientReleaseNotesdoc' + +# -- Options for LaTeX output --------------------------------------------- + +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', + +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', + +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', + +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# (master_doc, 'swift.tex', u'swift Documentation', +# u'swift', 'manual'), +# ] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +# man_pages = [ +# (master_doc, 'swift', u'swift Documentation', +# [author], 1) +# ] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +# texinfo_documents = [ +# (master_doc, 'swift', u'swift Documentation', +# author, 'swift', 'One line description of project.', +# 'Miscellaneous'), +# ] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + +locale_dirs = ['locale/'] + +# -- Options for openstackdocstheme ------------------------------------------- +repository_name = 'openstack/python-swiftclient' +bug_project = 'python-swiftclient' +bug_tag = '' diff --git a/releasenotes/source/current.rst b/releasenotes/source/current.rst new file mode 100644 index 0000000..87a748f --- /dev/null +++ b/releasenotes/source/current.rst @@ -0,0 +1,5 @@ +==================================== + Current (Unreleased) Release Notes +==================================== + +.. release-notes:: diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..27f675e --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,14 @@ +============================ + Swift Client Release Notes +============================ + +.. toctree:: + :maxdepth: 1 + + current + stein + rocky + queens + pike + ocata + newton diff --git a/releasenotes/source/newton.rst b/releasenotes/source/newton.rst new file mode 100644 index 0000000..59418a3 --- /dev/null +++ b/releasenotes/source/newton.rst @@ -0,0 +1,6 @@ +============================= + Newton Series Release Notes +============================= + +.. release-notes:: + :branch: stable/newton diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst new file mode 100644 index 0000000..726307b --- /dev/null +++ b/releasenotes/source/ocata.rst @@ -0,0 +1,6 @@ +============================ + Ocata Series Release Notes +============================ + +.. release-notes:: + :branch: stable/ocata diff --git a/releasenotes/source/pike.rst b/releasenotes/source/pike.rst new file mode 100644 index 0000000..e2c4806 --- /dev/null +++ b/releasenotes/source/pike.rst @@ -0,0 +1,6 @@ +=========================== + Pike Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/pike diff --git a/releasenotes/source/queens.rst b/releasenotes/source/queens.rst new file mode 100644 index 0000000..36ac616 --- /dev/null +++ b/releasenotes/source/queens.rst @@ -0,0 +1,6 @@ +=================================== + Queens Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/queens diff --git a/releasenotes/source/rocky.rst b/releasenotes/source/rocky.rst new file mode 100644 index 0000000..40dd517 --- /dev/null +++ b/releasenotes/source/rocky.rst @@ -0,0 +1,6 @@ +=================================== + Rocky Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/rocky diff --git a/releasenotes/source/stein.rst b/releasenotes/source/stein.rst new file mode 100644 index 0000000..efaceb6 --- /dev/null +++ b/releasenotes/source/stein.rst @@ -0,0 +1,6 @@ +=================================== + Stein Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/stein diff --git a/requirements.txt b/requirements.txt index 6d31e09..1c2ce33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD -requests>=1.1 -six>=1.5.2 +futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +requests>=1.1.0 +six>=1.9.0 @@ -4,8 +4,8 @@ summary = OpenStack Object Storage API Client Library description-file = README.rst author = OpenStack -author-email = openstack-dev@lists.openstack.org -home-page = http://docs.openstack.org/developer/python-swiftclient +author-email = openstack-discuss@lists.openstack.org +home-page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -17,8 +17,9 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 [global] setup-hooks = diff --git a/swiftclient/client.py b/swiftclient/client.py index 80b6eda..f071182 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -17,6 +17,7 @@ OpenStack Swift client library used internally """ import socket +import re import requests import logging import warnings @@ -25,7 +26,7 @@ from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError from six.moves import http_client from six.moves.urllib.parse import quote as _quote, unquote -from six.moves.urllib.parse import urlparse, urlunparse +from six.moves.urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time import six @@ -38,6 +39,7 @@ from swiftclient.utils import ( # Default is 100, increase to 256 http_client._MAXHEADERS = 256 +VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -58,6 +60,19 @@ except ImportError: def createLock(self): self.lock = None +ksexceptions = ksclient_v2 = ksclient_v3 = None +try: + from keystoneclient import exceptions as ksexceptions + # prevent keystoneclient warning us that it has no log handlers + logging.getLogger('keystoneclient').addHandler(NullHandler()) + from keystoneclient.v2_0 import client as ksclient_v2 +except ImportError: + pass +try: + from keystoneclient.v3 import client as ksclient_v3 +except ImportError: + pass + # requests version 1.2.3 try to encode headers in ascii, preventing # utf-8 encoded header to be 'prepared' if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): @@ -149,7 +164,7 @@ def http_log(args, kwargs, resp, body): elif element in ('GET', 'POST', 'PUT'): string_parts.append(' -X %s' % element) else: - string_parts.append(' %s' % element) + string_parts.append(' %s' % parse_header_string(element)) if 'headers' in kwargs: headers = scrub_headers(kwargs['headers']) for element in headers: @@ -233,8 +248,8 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) - and header.startswith(USER_METADATA_TYPE)): + if (isinstance(header, six.string_types) and + header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) ret[header] = value @@ -271,6 +286,9 @@ class _ObjectBody(object): def __next__(self): return self.next() + def close(self): + self.resp.close() + class _RetryBody(_ObjectBody): """ @@ -302,7 +320,7 @@ class _RetryBody(_ObjectBody): self.obj = obj self.query_string = query_string self.response_dict = response_dict - self.headers = headers if headers is not None else {} + self.headers = dict(headers) if headers is not None else {} self.bytes_read = 0 def read(self, length=None): @@ -384,6 +402,7 @@ class HTTPConnection(object): self.request_session = requests.Session() # Don't use requests's default headers self.request_session.headers = None + self.resp = None if self.parsed_url.scheme not in ('http', 'https'): raise ClientException('Unsupported scheme "%s" in url "%s"' % (self.parsed_url.scheme, url)) @@ -453,11 +472,23 @@ class HTTPConnection(object): self.resp.status = self.resp.status_code old_getheader = self.resp.raw.getheader + def _decode_header(string): + if string is None or six.PY2: + return string + return string.encode('iso-8859-1').decode('utf-8') + + def _encode_header(string): + if string is None or six.PY2: + return string + return string.encode('utf-8').decode('iso-8859-1') + def getheaders(): - return self.resp.headers.items() + return [(_decode_header(k), _decode_header(v)) + for k, v in self.resp.headers.items()] def getheader(k, v=None): - return old_getheader(k.lower(), v) + return _decode_header(old_getheader( + _encode_header(k.lower()), _encode_header(v))) def releasing_read(*args, **kwargs): chunk = self.resp.raw.read(*args, **kwargs) @@ -476,6 +507,11 @@ class HTTPConnection(object): return self.resp + def close(self): + if self.resp: + self.resp.close() + self.request_session.close() + def http_connection(*arg, **kwarg): """:returns: tuple of (parsed url, connection object)""" @@ -497,6 +533,8 @@ def get_auth_1_0(url, user, key, snet, **kwargs): conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() + resp.close() + conn.close() http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') @@ -511,8 +549,9 @@ def get_auth_1_0(url, user, key, snet, **kwargs): netloc = parsed[1] parsed[1] = 'snet-' + netloc url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) + + token = resp.getheader('x-storage-token', resp.getheader('x-auth-token')) + return url, token def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): @@ -522,25 +561,6 @@ def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): return get_auth_keystone(auth_url, user, key, os_options, **kwargs) -def _import_keystone_client(auth_version): - # the attempted imports are encapsulated in this function to allow - # mocking for tests - try: - if auth_version in AUTH_VERSIONS_V3: - from keystoneclient.v3 import client as ksclient - else: - from keystoneclient.v2_0 import client as ksclient - from keystoneclient import exceptions - # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) - return ksclient, exceptions - except ImportError: - raise ClientException(''' -Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth -version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment -variables to be set or overridden with -A, -U, or -K.''') - - def get_auth_keystone(auth_url, user, key, os_options, **kwargs): """ Authenticate against a keystone server. @@ -550,10 +570,39 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): insecure = kwargs.get('insecure', False) timeout = kwargs.get('timeout', None) - auth_version = kwargs.get('auth_version', '2.0') + auth_version = kwargs.get('auth_version', None) debug = logger.isEnabledFor(logging.DEBUG) - ksclient, exceptions = _import_keystone_client(auth_version) + # Add the version suffix in case of versionless Keystone endpoints. If + # auth_version is also unset it is likely that it is v3 + if not VERSIONFUL_AUTH_PATH.match( + urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]): + # Normalize auth_url to end in a slash because urljoin + auth_url = auth_url.rstrip('/') + '/' + if auth_version and auth_version in AUTH_VERSIONS_V2: + auth_url = urljoin(auth_url, "v2.0") + else: + auth_url = urljoin(auth_url, "v3") + auth_version = '3' + logger.debug("Versionless auth_url - using %s as endpoint" % auth_url) + + # Legacy default if not set + if auth_version is None: + auth_version = '2' + + ksclient = None + if auth_version in AUTH_VERSIONS_V3: + if ksclient_v3 is not None: + ksclient = ksclient_v3 + else: + if ksclient_v2 is not None: + ksclient = ksclient_v2 + + if ksclient is None: + raise ClientException(''' +Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth +version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment +variables to be set or overridden with -A, -U, or -K.''') try: _ksclient = ksclient.Client( @@ -574,13 +623,13 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): cert=kwargs.get('cert'), key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) - except exceptions.Unauthorized: + except ksexceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' if auth_version in AUTH_VERSIONS_V3: msg = ('Unauthorized. Check username/id, password, ' 'tenant name/id and user/tenant domain name/id.') raise ClientException(msg) - except exceptions.AuthorizationFailure as err: + except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) service_type = os_options.get('service_type') or 'object-store' endpoint_type = os_options.get('endpoint_type') or 'publicURL' @@ -593,7 +642,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): service_type=service_type, endpoint_type=endpoint_type, **filter_kwargs) - except exceptions.EndpointNotFound: + except ksexceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) return endpoint, _ksclient.auth_token @@ -631,8 +680,10 @@ def get_auth(auth_url, user, key, **kwargs): if session: service_type = os_options.get('service_type', 'object-store') interface = os_options.get('endpoint_type', 'public') + region_name = os_options.get('region_name') storage_url = session.get_endpoint(service_type=service_type, - interface=interface) + interface=interface, + region_name=region_name) token = session.get_token() elif auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -655,9 +706,9 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') - or os_options.get('project_name') - or os_options.get('project_id')): + if not (os_options.get('tenant_name') or os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') @@ -706,7 +757,7 @@ def store_response(resp, response_dict): def get_account(url, token, marker=None, limit=None, prefix=None, end_marker=None, http_conn=None, full_listing=False, - service_token=None, headers=None): + service_token=None, headers=None, delimiter=None): """ Get a listing of containers for the account. @@ -722,6 +773,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param delimiter: delimiter query :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -735,14 +787,14 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if not http_conn: http_conn = http_connection(url) if full_listing: - rv = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, headers=req_headers) + rv = get_account(url, token, marker, limit, prefix, end_marker, + http_conn, headers=req_headers, delimiter=delimiter) listing = rv[1] while listing: marker = listing[-1]['name'] listing = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, - headers=req_headers)[1] + end_marker, http_conn, headers=req_headers, + delimiter=delimiter)[1] if listing: rv[1].extend(listing) return rv @@ -754,6 +806,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) @@ -834,13 +888,15 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, path = parsed.path if query_string: path += '?' + query_string - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request(method, path, data, headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request(method, path, data, req_headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) + http_log((url, method,), {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -882,12 +938,6 @@ def get_container(url, token, container, marker=None, limit=None, """ if not http_conn: http_conn = http_connection(url) - if headers: - headers = dict(headers) - else: - headers = {} - headers['X-Auth-Token'] = token - headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -922,17 +972,20 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&path=%s' % quote(path) if query_string: qs += '&%s' % query_string.lstrip('?') + req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) method = 'GET' - conn.request(method, '%s?%s' % (cont_path, qs), '', headers) + conn.request(method, '%s?%s' % (cont_path, qs), '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%(url)s%(cont_path)s?%(qs)s' % {'url': url.replace(parsed.path, ''), 'cont_path': cont_path, 'qs': qs}, method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container GET failed', body) @@ -1005,23 +1058,23 @@ def put_container(url, token, container, headers=None, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'PUT' - if not headers: - headers = {} - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + if 'content-length' not in (k.lower() for k in req_headers): + req_headers['Content-Length'] = '0' if query_string: path += '?' + query_string.lstrip('?') - conn.request(method, path, '', headers) + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() store_response(resp, response_dict) http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container PUT failed', body) @@ -1048,16 +1101,18 @@ def post_container(url, token, container, headers, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' - conn.request(method, path, '', headers) + req_headers['Content-Length'] = '0' + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1175,7 +1230,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None, headers=None): + service_token=None, headers=None, query_string=None): """ Get object info @@ -1196,6 +1251,8 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if query_string: + path += '?' + query_string if headers: headers = dict(headers) else: @@ -1286,8 +1343,10 @@ def put_object(url, token=None, container=None, name=None, contents=None, if content_type is not None: headers['Content-Type'] = content_type elif 'Content-Type' not in headers: - # python-requests sets application/x-www-form-urlencoded otherwise - headers['Content-Type'] = '' + if StrictVersion(requests.__version__) < StrictVersion('2.4.0'): + # python-requests sets application/x-www-form-urlencoded otherwise + # if using python3. + headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' @@ -1350,14 +1409,16 @@ def post_object(url, token, container, name, headers, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request('POST', path, '', headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request('POST', path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1525,7 +1586,7 @@ class Connection(object): os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, retry_on_ratelimit=False, - timeout=None, session=None): + timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1561,6 +1622,8 @@ class Connection(object): after a backoff. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. + :param force_auth_retry: reset auth info even if client got unexpected + error except 401 Unauthorized. """ self.session = session self.authurl = authurl @@ -1593,16 +1656,14 @@ class Connection(object): self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit self.timeout = timeout + self.force_auth_retry = force_auth_retry def close(self): - if (self.http_conn and isinstance(self.http_conn, tuple) - and len(self.http_conn) > 1): + if (self.http_conn and isinstance(self.http_conn, tuple) and + len(self.http_conn) > 1): conn = self.http_conn[1] - if hasattr(conn, 'close') and callable(conn.close): - # XXX: Our HTTPConnection object has no close, should be - # trying to close the requests.Session here? - conn.close() - self.http_conn = None + conn.close() + self.http_conn = None def get_auth(self): self.url, self.token = get_auth(self.authurl, self.user, self.key, @@ -1662,10 +1723,10 @@ class Connection(object): try: if not self.url or not self.token: self.url, self.token = self.get_auth() - self.http_conn = None + self.close() if self.service_auth and not self.service_token: self.url, self.service_token = self.get_service_auth() - self.http_conn = None + self.close() self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() @@ -1707,6 +1768,10 @@ class Connection(object): pass else: raise + + if self.force_auth_retry: + self.url = self.token = self.service_token = None + sleep(backoff) backoff = min(backoff * 2, self.max_backoff) if reset_func: @@ -1717,14 +1782,16 @@ class Connection(object): return self._retry(None, head_account, headers=headers) def get_account(self, marker=None, limit=None, prefix=None, - end_marker=None, full_listing=False, headers=None): + end_marker=None, full_listing=False, headers=None, + delimiter=None): """Wrapper for :func:`get_account`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_account, marker=marker, limit=limit, prefix=prefix, end_marker=end_marker, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + delimiter=delimiter) def post_account(self, headers, response_dict=None, query_string=None, data=None): @@ -1770,9 +1837,10 @@ class Connection(object): query_string=query_string, headers=headers) - def head_object(self, container, obj, headers=None): + def head_object(self, container, obj, headers=None, query_string=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj, headers=headers) + return self._retry(None, head_object, container, obj, headers=headers, + query_string=query_string) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): @@ -1817,7 +1885,9 @@ class Connection(object): reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) + + def reset_func(*a, **kw): + seek(orig_pos) elif reset: reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, @@ -1850,8 +1920,7 @@ class Connection(object): url = url or self.url if not url: url, _ = self.get_auth() - scheme = urlparse(url).scheme - netloc = urlparse(url).netloc - url = scheme + '://' + netloc + '/info' - http_conn = self.http_connection(url) - return get_capabilities(http_conn) + parsed = urlparse(urljoin(url, '/info')) + if not self.http_conn: + self.http_conn = self.http_connection(url) + return get_capabilities((parsed, self.http_conn[1])) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 5e03ed7..fcf0ed9 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -175,6 +175,14 @@ class ConnectionThreadPoolExecutor(ThreadPoolExecutor): super(ConnectionThreadPoolExecutor, self).__init__(max_workers) def submit(self, fn, *args, **kwargs): + """ + Schedules the callable, `fn`, to be executed + + :param fn: the callable to be invoked + :param args: the positional arguments for the callable + :param kwargs: the keyword arguments for the callable + :returns: a Future object representing the execution of the callable + """ def conn_fn(): priority = None conn = None diff --git a/swiftclient/service.py b/swiftclient/service.py index 223641b..06de091 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -17,6 +17,7 @@ import logging import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,12 +45,13 @@ from swiftclient.command_helpers import ( from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") @@ -143,6 +145,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), "os_user_domain_name": environ.get('OS_USER_DOMAIN_NAME'), @@ -171,6 +174,7 @@ def _build_default_global_options(): 'container_threads': 10 } + _default_global_options = _build_default_global_options() _default_local_options = { @@ -203,6 +207,7 @@ _default_local_options = { 'shuffle': False, 'destination': None, 'fresh_metadata': False, + 'ignore_mtime': False, } POLICY = 'X-Storage-Policy' @@ -259,7 +264,8 @@ def get_conn(options): insecure=options['insecure'], cert=options['os_cert'], cert_key=options['os_key'], - ssl_compression=options['ssl_compression']) + ssl_compression=options['ssl_compression'], + force_auth_retry=options['force_auth_retry']) def mkdirs(path): @@ -378,10 +384,24 @@ class _SwiftReader(object): self._actual_read = 0 self._content_length = None self._actual_md5 = None - self._expected_etag = headers.get('etag') - - if ('x-object-manifest' not in headers - and 'x-static-large-object' not in headers and checksum): + self._expected_md5 = headers.get('etag', '') + + if len(self._expected_md5) > 1 and self._expected_md5[0] == '"' \ + and self._expected_md5[-1] == '"': + self._expected_md5 = self._expected_md5[1:-1] + + # Some headers indicate the MD5 of the response + # definitely *won't* match the ETag + bad_md5_headers = set([ + 'content-range', + 'x-object-manifest', + 'x-static-large-object', + ]) + if bad_md5_headers.intersection(headers): + # This isn't a useful checksum + self._expected_md5 = '' + + if self._expected_md5 and checksum: self._actual_md5 = md5() if 'content-length' in headers: @@ -399,15 +419,15 @@ class _SwiftReader(object): self._check_contents() def _check_contents(self): - if self._actual_md5 and self._expected_etag: + if self._actual_md5 and self._expected_md5: etag = self._actual_md5.hexdigest() - if etag != self._expected_etag: + if etag != self._expected_md5: raise SwiftError('Error downloading {0}: md5sum != etag, ' '{1} != {2}'.format( - self._path, etag, self._expected_etag)) + self._path, etag, self._expected_md5)) - if (self._content_length is not None - and self._actual_read != self._content_length): + if (self._content_length is not None and + self._actual_read != self._content_length): raise SwiftError('Error downloading {0}: read_length != ' 'content_length, {1:d} != {2:d}'.format( self._path, self._actual_read, @@ -433,7 +453,9 @@ class SwiftService(object): **_default_local_options ) process_options(self._options) - create_connection = lambda: get_conn(self._options) + + def create_connection(): + return get_conn(self._options) self.thread_manager = MultiThreadingManager( create_connection, segment_threads=self._options['segment_threads'], @@ -1111,14 +1133,14 @@ class SwiftService(object): if options['skip_identical']: filename = out_file if out_file else path try: - fp = open(filename, 'rb') + fp = open(filename, 'rb', DISK_BUFFER) except IOError: pass else: with fp: md5sum = md5() while True: - data = fp.read(65536) + data = fp.read(DISK_BUFFER) if not data: break md5sum.update(data) @@ -1126,7 +1148,7 @@ class SwiftService(object): try: start_time = time() - get_args = {'resp_chunk_size': 65536, + get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} if options['skip_identical']: @@ -1209,10 +1231,10 @@ class SwiftService(object): if not no_file: if out_file: - fp = open(out_file, 'wb') + fp = open(out_file, 'wb', DISK_BUFFER) else: if basename(path): - fp = open(path, 'wb') + fp = open(path, 'wb', DISK_BUFFER) else: pseudodir = True @@ -1226,7 +1248,8 @@ class SwiftService(object): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if 'x-object-meta-mtime' in headers and not no_file: + if ('x-object-meta-mtime' in headers and not no_file and + not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: @@ -1485,7 +1508,8 @@ class SwiftService(object): if hasattr(s, 'read'): # We've got a file like object to upload to o file_future = self.thread_manager.object_uu_pool.submit( - self._upload_object_job, container, s, o, object_options + self._upload_object_job, container, s, o, object_options, + results_queue=rq ) details['file'] = s details['object'] = o @@ -1717,7 +1741,7 @@ class SwiftService(object): } fp = None try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) @@ -1767,6 +1791,132 @@ class SwiftService(object): if fp is not None: fp.close() + @staticmethod + def _put_object(conn, container, name, content, headers=None, md5=None): + """ + Upload object into a given container and verify the resulting ETag, if + the md5 optional parameter is passed. + + :param conn: The Swift connection to use for uploads. + :param container: The container to put the object into. + :param name: The name of the object. + :param content: Object content. + :param headers: Headers (optional) to associate with the object. + :param md5: MD5 sum of the content. If passed in, will be used to + verify the returned ETag. + + :returns: A dictionary as the response from calling put_object. + The keys are: + - status + - reason + - headers + On error, the dictionary contains the following keys: + - success (with value False) + - error - the encountered exception (object) + - error_timestamp + - response_dict - results from the put_object call, as + documented above + - attempts - number of attempts made + """ + if headers is None: + headers = {} + else: + headers = dict(headers) + if md5 is not None: + headers['etag'] = md5 + results = {} + try: + etag = conn.put_object( + container, name, content, content_length=len(content), + headers=headers, response_dict=results) + if md5 is not None and etag != md5: + raise SwiftError('Upload verification failed for {0}: md5 ' + 'mismatch {1} != {2}'.format(name, md5, etag)) + results['success'] = True + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + return { + 'success': False, + 'error': err, + 'error_timestamp': err_time, + 'response_dict': results, + 'attempts': conn.attempts, + 'traceback': traceback + } + return results + + @staticmethod + def _upload_stream_segment(conn, container, object_name, + segment_container, segment_name, + segment_size, segment_index, + headers, fd): + """ + Upload a segment from a stream, buffering it in memory first. The + resulting object is placed either as a segment in the segment + container, or if it is smaller than a single segment, as the given + object name. + + :param conn: Swift Connection to use. + :param container: Container in which the object would be placed. + :param object_name: Name of the final object (used in case the stream + is smaller than the segment_size) + :param segment_container: Container to hold the object segments. + :param segment_name: The name of the segment. + :param segment_size: Minimum segment size. + :param segment_index: The segment index. + :param headers: Headers to attach to the segment/object. + :param fd: File-like handle for the content. Must implement read(). + + :returns: Dictionary, containing the following keys: + - complete -- whether the stream is exhausted + - segment_size - the actual size of the segment (may be + smaller than the passed in segment_size) + - segment_location - path to the segment + - segment_index - index of the segment + - segment_etag - the ETag for the segment + """ + buf = [] + dgst = md5() + bytes_read = 0 + while bytes_read < segment_size: + data = fd.read(segment_size - bytes_read) + if not data: + break + bytes_read += len(data) + dgst.update(data) + buf.append(data) + buf = b''.join(buf) + segment_hash = dgst.hexdigest() + + if not buf and segment_index > 0: + # Happens if the segment size aligns with the object size + return {'complete': True, + 'segment_size': 0, + 'segment_index': None, + 'segment_etag': None, + 'segment_location': None, + 'success': True} + + if segment_index == 0 and len(buf) < segment_size: + ret = SwiftService._put_object( + conn, container, object_name, buf, headers, segment_hash) + ret['segment_location'] = '/%s/%s' % (container, object_name) + else: + ret = SwiftService._put_object( + conn, segment_container, segment_name, buf, headers, + segment_hash) + ret['segment_location'] = '/%s/%s' % ( + segment_container, segment_name) + + ret.update( + dict(complete=len(buf) < segment_size, + segment_size=len(buf), + segment_index=segment_index, + segment_etag=segment_hash, + for_object=object_name)) + return ret + def _get_chunk_data(self, conn, container, obj, headers, manifest=None): chunks = [] if 'x-object-manifest' in headers: @@ -1794,8 +1944,10 @@ class SwiftService(object): return chunks def _is_identical(self, chunk_data, path): + if path is None: + return False try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) except IOError: return False @@ -1804,7 +1956,7 @@ class SwiftService(object): to_read = chunk['bytes'] md5sum = md5() while to_read: - data = fp.read(min(65536, to_read)) + data = fp.read(min(DISK_BUFFER, to_read)) if not data: return False md5sum.update(data) @@ -1814,6 +1966,47 @@ class SwiftService(object): # Each chunk is verified; check that we're at the end of the file return not fp.read(1) + @staticmethod + def _upload_slo_manifest(conn, segment_results, container, obj, headers): + """ + Upload an SLO manifest, given the results of uploading each segment, to + the specified container. + + :param segment_results: List of response_dict structures, as populated + by _upload_segment_job. Specifically, each + entry must container the following keys: + - segment_location + - segment_etag + - segment_size + - segment_index + :param container: The container to put the manifest into. + :param obj: The name of the manifest object to use. + :param headers: Optional set of headers to attach to the manifest. + """ + if headers is None: + headers = {} + segment_results.sort(key=lambda di: di['segment_index']) + for seg in segment_results: + seg_loc = seg['segment_location'].lstrip('/') + if isinstance(seg_loc, text_type): + seg_loc = seg_loc.encode('utf-8') + + manifest_data = json.dumps([ + { + 'path': d['segment_location'], + 'etag': d['segment_etag'], + 'size_bytes': d['segment_size'] + } for d in segment_results + ]) + + response = {} + conn.put_object( + container, obj, manifest_data, + headers=headers, + query_string='multipart-manifest=put', + response_dict=response) + return response + def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): if obj.startswith('./') or obj.startswith('.\\'): @@ -1847,8 +2040,8 @@ class SwiftService(object): new_slo_manifest_paths = set() segment_size = int(0 if options['segment_size'] is None else options['segment_size']) - if (options['changed'] or options['skip_identical'] - or not options['leave_segments']): + if (options['changed'] or options['skip_identical'] or + not options['leave_segments']): try: headers = conn.head_object(container, obj) is_slo = config_true_value( @@ -1869,9 +2062,9 @@ class SwiftService(object): cl = int(headers.get('content-length')) mt = headers.get('x-object-meta-mtime') - if (path is not None and options['changed'] - and cl == getsize(path) - and mt == put_headers['x-object-meta-mtime']): + if (path is not None and options['changed'] and + cl == getsize(path) and + mt == put_headers['x-object-meta-mtime']): res.update({ 'success': True, 'status': 'skipped-changed' @@ -1880,11 +2073,9 @@ class SwiftService(object): if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -1898,14 +2089,16 @@ class SwiftService(object): return res # Merge the command line header options to the put_headers + put_headers.update(split_headers( + options['meta'], 'X-Object-Meta-')) put_headers.update(split_headers(options['header'], '')) # Don't do segment job if object is not big enough, and never do # a segment job if we're reading from a stream - we may fail if we # go over the single object limit, but this gives us a nice way # to create objects from memory - if (path is not None and segment_size - and (getsize(path) > segment_size)): + if (path is not None and segment_size and + (getsize(path) > segment_size)): res['large_object'] = True seg_container = container + '_segments' if options['segment_container']: @@ -1969,30 +2162,12 @@ class SwiftService(object): res['segment_results'] = segment_results if options['use_slo']: - segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): - seg_loc = seg_loc.encode('utf-8') - new_slo_manifest_paths.add(seg_loc) - - manifest_data = json.dumps([ - { - 'path': d['segment_location'], - 'etag': d['segment_etag'], - 'size_bytes': d['segment_size'] - } for d in segment_results - ]) - - put_headers['x-static-large-object'] = 'true' - mr = {} - conn.put_object( - container, obj, manifest_data, - headers=put_headers, - query_string='multipart-manifest=put', - response_dict=mr - ) - res['manifest_response_dict'] = mr + response = self._upload_slo_manifest( + conn, segment_results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2010,6 +2185,52 @@ class SwiftService(object): response_dict=mr ) res['manifest_response_dict'] = mr + elif options['use_slo'] and segment_size and not path: + segment = 0 + results = [] + while True: + segment_name = '%s/slo/%s/%s/%08d' % ( + obj, put_headers['x-object-meta-mtime'], + segment_size, segment + ) + seg_container = container + '_segments' + if options['segment_container']: + seg_container = options['segment_container'] + ret = self._upload_stream_segment( + conn, container, obj, + seg_container, + segment_name, + segment_size, + segment, + put_headers, + stream + ) + if not ret['success']: + return ret + if (ret['complete'] and segment == 0) or\ + ret['segment_size'] > 0: + results.append(ret) + if results_queue is not None: + # Don't insert the 0-sized segments or objects + # themselves + if ret['segment_location'] != '/%s/%s' % ( + container, obj) and ret['segment_size'] > 0: + results_queue.put(ret) + if ret['complete']: + break + segment += 1 + if results[0]['segment_location'] != '/%s/%s' % ( + container, obj): + response = self._upload_slo_manifest( + conn, results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) + res['large_object'] = True + else: + res['response_dict'] = ret + res['large_object'] = False else: res['large_object'] = False obr = {} @@ -2017,7 +2238,7 @@ class SwiftService(object): try: if path is not None: content_length = getsize(path) - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) contents = LengthWrapper(fp, content_length, md5=options['checksum']) @@ -2043,14 +2264,12 @@ class SwiftService(object): finally: if fp is not None: fp.close() - if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2062,10 +2281,8 @@ class SwiftService(object): if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): @@ -2209,8 +2426,8 @@ class SwiftService(object): # Cancel the remaining container deletes, but yield # any pending results - if (not cancelled and options['fail_fast'] - and not res['success']): + if (not cancelled and options['fail_fast'] and + not res['success']): cancelled = True def _bulk_delete_page_size(self, objects): @@ -2257,17 +2474,18 @@ class SwiftService(object): def _delete_segment(conn, container, obj, results_queue=None): results_dict = {} try: - conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} + conn.delete_object(container, obj, response_dict=results_dict) except Exception as err: - traceback, err_time = report_traceback() - logger.exception(err) - res = { - 'success': False, - 'error': err, - 'traceback': traceback, - 'error_timestamp': err_time - } + if not isinstance(err, ClientException) or err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 841ed6e..cc4f325 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,20 +17,24 @@ from __future__ import print_function, unicode_literals import argparse +import getpass +import io import json import logging import signal import socket +import warnings from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 from six.moves.urllib.parse import unquote, urlparse -from sys import argv as sys_argv, exit, stderr +from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime from swiftclient import RequestException -from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes +from swiftclient.utils import config_true_value, generate_temp_url, \ + prt_bytes, JSONableIterable from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version @@ -48,13 +52,14 @@ except ImportError: BASENAME = 'swift' commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', - 'capabilities', 'info', 'tempurl', 'auth') + 'capabilities', 'info', 'tempurl', 'auth', 'bash_completion') def immediate_exit(signum, frame): stderr.write(" Aborted\n") os_exit(2) + st_delete_options = '''[--all] [--leave-segments] [--object-threads <threads>] [--container-threads <threads>] @@ -87,7 +92,7 @@ Optional arguments: '''.strip("\n") -def st_delete(parser, args, output_manager): +def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') @@ -111,6 +116,11 @@ def st_delete(parser, args, output_manager): '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' 'Its value must be a positive integer. Default is 10.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options['yes_all']) or (args and options['yes_all']): @@ -272,10 +282,13 @@ Optional arguments: script to multiple servers). Enable this option to submit download jobs to the thread pool in the order they are listed in the object store. + --ignore-mtime Ignore the 'X-Object-Meta-Mtime' header when + downloading an object. Instead, create atime and mtime + with fresh timestamps. '''.strip("\n") -def st_download(parser, args, output_manager): +def st_download(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' @@ -332,6 +345,17 @@ def st_download(parser, args, output_manager): 'nightly automated download script to multiple servers). Enable this ' 'option to submit download jobs to the thread pool in the order they ' 'are listed in the object store.') + parser.add_argument( + '--ignore-mtime', action='store_true', dest='ignore_mtime', + default=False, help='By default, the object-meta-mtime header is used ' + 'to store the access and modified timestamp for the downloaded file. ' + 'With this option, the header is ignored and the timestamps are ' + 'created freshly.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': @@ -482,7 +506,7 @@ Optional arguments: '''.strip('\n') -def st_list(parser, args, output_manager): +def st_list(parser, args, output_manager, return_parser=False): def _print_stats(options, stats, human): total_count = total_bytes = 0 @@ -555,10 +579,17 @@ def st_list(parser, args, output_manager): help='Roll up items with the given delimiter. For containers ' 'only. See OpenStack Swift API documentation for ' 'what this means.') + parser.add_argument('-j', '--json', action='store_true', + help='print listing information in json') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for listing.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if options['delimiter'] and not args: @@ -588,6 +619,20 @@ def st_list(parser, args, output_manager): else: stats_parts_gen = swift.list(container=container) + if options.get('json', False): + def listing(stats_parts_gen=stats_parts_gen): + for stats in stats_parts_gen: + if stats["success"]: + for item in stats['listing']: + yield item + else: + raise stats["error"] + + json.dump( + JSONableIterable(listing()), output_manager.print_stream, + sort_keys=True, indent=2) + output_manager.print_msg('') + return for stats in stats_parts_gen: if stats["success"]: _print_stats(options, stats, human) @@ -617,7 +662,7 @@ Optional arguments: '''.strip('\n') -def st_stat(parser, args, output_manager): +def st_stat(parser, args, output_manager, return_parser=False): parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') @@ -626,6 +671,10 @@ def st_stat(parser, args, output_manager): default=[], help='Adds a custom request header to use for stat.') + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] @@ -677,7 +726,7 @@ def st_stat(parser, args, output_manager): output_manager.error(e.value) -st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to] +st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>] [--sync-key <sync-key>] [--meta <name:value>] [--header <header>] [<container> [<object>]] @@ -713,7 +762,7 @@ Optional arguments: '''.strip('\n') -def st_post(parser, args, output_manager): +def st_post(parser, args, output_manager, return_parser=False): parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' @@ -738,6 +787,11 @@ def st_post(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (options['read_acl'] or options['write_acl'] or options['sync_to'] or @@ -810,7 +864,7 @@ Optional arguments: '''.strip('\n') -def st_copy(parser, args, output_manager): +def st_copy(parser, args, output_manager, return_parser=False): parser.add_argument( '-d', '--destination', help='The container and name of the ' 'destination object') @@ -827,6 +881,11 @@ def st_copy(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] @@ -881,8 +940,8 @@ def st_copy(parser, args, output_manager): st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] [--segment-container <container>] [--leave-segments] [--object-threads <thread>] [--segment-threads <threads>] - [--header <header>] [--use-slo] [--ignore-checksum] - [--object-name <object-name>] + [--meta <name:value>] [--header <header>] [--use-slo] + [--ignore-checksum] [--object-name <object-name>] <container> <file_or_directory> [<file_or_directory>] [...] ''' @@ -892,7 +951,9 @@ Uploads specified files and directories to the given container. Positional arguments: <container> Name of container to upload to. <file_or_directory> Name of file or directory to upload. Specify multiple - times for multiple uploads. + times for multiple uploads. If "-" is specified, reads + content from standard input (--object-name is required + in this case). Optional arguments: -c, --changed Only upload files that have changed since the last @@ -916,6 +977,9 @@ Optional arguments: --segment-threads <threads> Number of threads to use for uploading object segments. Default is 10. + -m, --meta <name:value> + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large -H, --header <header:value> Adds a customized request header. This option may be repeated. Example: -H "content-type:text/plain" @@ -931,7 +995,9 @@ Optional arguments: '''.strip('\n') -def st_upload(parser, args, output_manager): +def st_upload(parser, args, output_manager, return_parser=False): + DEFAULT_STDIN_SEGMENT = 10 * 1024 * 1024 + parser.add_argument( '-c', '--changed', action='store_true', dest='changed', default=False, help='Only upload files that have changed since ' @@ -967,6 +1033,10 @@ def st_upload(parser, args, output_manager): help='Number of threads to use for uploading object segments. ' 'Its value must be a positive integer. Default is 10.') parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' ' This option may be repeated. Example: -H "content-type:text/plain" ' @@ -983,6 +1053,11 @@ def st_upload(parser, args, output_manager): parser.add_argument( '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -993,6 +1068,11 @@ def st_upload(parser, args, output_manager): else: container = args[0] files = args[1:] + from_stdin = '-' in files + if from_stdin and len(files) > 1: + output_manager.error( + 'upload from stdin cannot be used along with other files') + return if options['object_name'] is not None: if len(files) > 1: @@ -1000,6 +1080,10 @@ def st_upload(parser, args, output_manager): return else: orig_path = files[0] + elif from_stdin: + output_manager.error( + 'object-name must be specified with uploads from stdin') + return if options['segment_size']: try: @@ -1032,12 +1116,26 @@ def st_upload(parser, args, output_manager): st_upload_help) return + if from_stdin: + if not options['use_slo']: + options['use_slo'] = True + if not options['segment_size']: + options['segment_size'] = DEFAULT_STDIN_SEGMENT + options['object_uu_threads'] = options['object_threads'] with SwiftService(options=options) as swift: try: objs = [] dir_markers = [] for f in files: + if f == '-': + fd = io.open(stdin.fileno(), mode='rb') + objs.append(SwiftUploadObject( + fd, object_name=options['object_name'])) + # We ensure that there is exactly one "file" to upload in + # this case -- stdin + break + if isfile(f): objs.append(f) elif isdir(f): @@ -1051,7 +1149,7 @@ def st_upload(parser, args, output_manager): # Now that we've collected all the required files and dir markers # build the tuples for the call to upload - if options['object_name'] is not None: + if options['object_name'] is not None and not from_stdin: objs = [ SwiftUploadObject( o, object_name=o.replace( @@ -1139,7 +1237,7 @@ Optional arguments: st_info_help = st_capabilities_help -def st_capabilities(parser, args, output_manager): +def st_capabilities(parser, args, output_manager, return_parser=False): def _print_compo_cap(name, capabilities): for feature, options in sorted(capabilities.items(), key=lambda x: x[0]): @@ -1152,6 +1250,11 @@ def st_capabilities(parser, args, output_manager): parser.add_argument('--json', action='store_true', help='print capability information in json') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1200,7 +1303,12 @@ Display auth related authentication variables in shell friendly format. '''.strip('\n') -def st_auth(parser, args, thread_manager): +def st_auth(parser, args, thread_manager, return_parser=False): + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if options['verbose'] > 1: if options['auth_version'] in ('1', '1.0'): @@ -1279,10 +1387,12 @@ Optional arguments: generated. --iso8601 If present, the generated temporary URL will contain an ISO 8601 UTC timestamp instead of a Unix timestamp. + --ip-range If present, the temporary URL will be restricted to the + given ip or ip range. '''.strip('\n') -def st_tempurl(parser, args, thread_manager): +def st_tempurl(parser, args, thread_manager, return_parser=False): parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, @@ -1302,6 +1412,16 @@ def st_tempurl(parser, args, thread_manager): help=("If present, the temporary URL will contain an ISO 8601 UTC " "timestamp instead of a Unix timestamp."), ) + parser.add_argument( + '--ip-range', action='store', + default=None, + help=("If present, the temporary URL will be restricted to the " + "given ip or ip range."), + ) + + # We return the parser to build up the bash_completion + if return_parser: + return parser (options, args) = parse_args(parser, args) args = args[1:] @@ -1321,7 +1441,8 @@ def st_tempurl(parser, args, thread_manager): path = generate_temp_url(parsed.path, timestamp, key, method, absolute=options['absolute_expiry'], iso8601=options['iso8601'], - prefix=options['prefix_based']) + prefix=options['prefix_based'], + ip_range=options['ip_range']) except ValueError as err: thread_manager.error(err) return @@ -1333,6 +1454,66 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg(url) +st_bash_completion_help = '''Retrieve command specific flags used by bash_completion. + +Optional positional arguments: + <command> Swift client command to filter the flags by. +'''.strip('\n') + + +st_bash_completion_options = '''[command] +''' + + +def st_bash_completion(parser, args, thread_manager, return_parser=False): + if return_parser: + return parser + + global commands + com = args[1] if len(args) > 1 else None + + if com: + if com in commands: + fn_commands = ["st_%s" % com] + else: + print("") + return + else: + fn_commands = [fn for fn in globals().keys() + if fn.startswith('st_') and + not fn.endswith('_options') and + not fn.endswith('_help')] + + subparsers = parser.add_subparsers() + subcommands = {} + if not com: + subcommands['base'] = parser + for command in fn_commands: + cmd = command[3:] + if com: + subparser = subparsers.add_parser( + cmd, help=globals()['%s_help' % command]) + add_default_args(subparser) + subparser = globals()[command]( + subparser, args, thread_manager, True) + subcommands[cmd] = subparser + else: + subcommands[cmd] = None + + cmds = set() + opts = set() + for sc_str, sc in list(subcommands.items()): + cmds.add(sc_str) + if sc: + for option in sc._optionals._option_string_actions: + opts.add(option) + + for cmd_to_remove in (com, 'bash_completion', 'base'): + if cmd_to_remove in cmds: + cmds.remove(cmd_to_remove) + print(' '.join(cmds | opts)) + + class HelpFormatter(argparse.HelpFormatter): def _format_action_invocation(self, action): if not action.option_strings: @@ -1366,6 +1547,30 @@ class HelpFormatter(argparse.HelpFormatter): return action.dest +def prompt_for_password(): + """ + Prompt the user for a password. + + :raise SystemExit: if a password cannot be entered without it being echoed + to the terminal. + :return: the entered password. + """ + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=getpass.GetPassWarning, + append=True) + try: + # temporarily set signal handling back to default to avoid user + # Ctrl-c leaving terminal in weird state + signal.signal(signal.SIGINT, signal.SIG_DFL) + return getpass.getpass() + except EOFError: + return None + except getpass.GetPassWarning: + exit('Input stream incompatible with --prompt option') + finally: + signal.signal(signal.SIGINT, immediate_exit) + + def parse_args(parser, args, enforce_requires=True): options, args = parser.parse_known_args(args or ['-h']) options = vars(options) @@ -1391,6 +1596,10 @@ def parse_args(parser, args, enforce_requires=True): if args and args[0] == 'tempurl': return options, args + # do this before process_options sets default auth version + if enforce_requires and options['prompt']: + options['key'] = options['os_password'] = prompt_for_password() + # Massage auth version; build out os_options subdict process_options(options) @@ -1425,88 +1634,7 @@ adding "-V 2" is necessary for this.'''.strip('\n')) return options, args -def main(arguments=None): - argv = sys_argv if arguments is None else arguments - - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] - - version = client_version - parser = argparse.ArgumentParser( - add_help=False, formatter_class=HelpFormatter, usage=''' -%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] - [--debug] [--info] [--quiet] [--auth <auth_url>] - [--auth-version <auth_version> | - --os-identity-api-version <auth_version> ] - [--user <username>] - [--key <api_key>] [--retries <num_retries>] - [--os-username <auth-user-name>] [--os-password <auth-password>] - [--os-user-id <auth-user-id>] - [--os-user-domain-id <auth-user-domain-id>] - [--os-user-domain-name <auth-user-domain-name>] - [--os-tenant-id <auth-tenant-id>] - [--os-tenant-name <auth-tenant-name>] - [--os-project-id <auth-project-id>] - [--os-project-name <auth-project-name>] - [--os-project-domain-id <auth-project-domain-id>] - [--os-project-domain-name <auth-project-domain-name>] - [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] - [--os-storage-url <storage-url>] [--os-region-name <region-name>] - [--os-service-type <service-type>] - [--os-endpoint-type <endpoint-type>] - [--os-cacert <ca-certificate>] [--insecure] - [--os-cert <client-certificate-file>] - [--os-key <client-certificate-key-file>] - [--no-ssl-compression] - <subcommand> [--help] [<subcommand options>] - -Command-line interface to the OpenStack Swift API. - -Positional arguments: - <subcommand> - delete Delete a container or objects within a container. - download Download objects from containers. - list Lists the containers for the account or the objects - for a container. - post Updates meta information for the account, container, - or object; creates containers if not present. - copy Copies object, optionally adds meta - stat Displays information for the account, container, - or object. - upload Uploads files or directories to the given container. - capabilities List cluster capabilities. - tempurl Create a temporary URL. - auth Display auth related environment variables. - -Examples: - %(prog)s download --help - - %(prog)s -A https://api.example.com/v1.0 \\ - -U user -K api_key stat -v - - %(prog)s --os-auth-url https://api.example.com/v2.0 \\ - --os-tenant-name tenant \\ - --os-username user --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-name project1 --os-project-domain-name domain1 \\ - --os-username user --os-user-domain-name domain1 \\ - --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-id 0123456789abcdef0123456789abcdef \\ - --os-user-id abcdef0123456789abcdef0123456789 \\ - --os-password password list - - %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ - list - - %(prog)s list --lh -'''.strip('\n')) - parser.add_argument('--version', action='version', - version='python-swiftclient %s' % version) - parser.add_argument('-h', '--help', action='store_true') - +def add_default_args(parser): default_auth_version = '1.0' for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): try: @@ -1566,6 +1694,17 @@ Examples: help='This option is deprecated and not used anymore. ' 'SSL compression should be disabled by default ' 'by the system SSL library.') + parser.add_argument('--force-auth-retry', + action='store_true', dest='force_auth_retry', + default=False, + help='Force a re-auth attempt on ' + 'any error other than 401 unauthorized') + parser.add_argument('--prompt', + action='store_true', dest='prompt', + default=False, + help='Prompt user to enter a password which overrides ' + 'any password supplied via --key, --os-password ' + 'or environment variables.') os_grp = parser.add_argument_group("OpenStack authentication options") os_grp.add_argument('--os-username', @@ -1708,6 +1847,100 @@ Examples: default=environ.get('OS_KEY'), help='Specify a client certificate key file (for ' 'client auth). Defaults to env[OS_KEY].') + + +def main(arguments=None): + argv = sys_argv if arguments is None else arguments + + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] + [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] + [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] + [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] + [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + [--force-auth-retry] + <subcommand> [--help] [<subcommand options>] + +Command-line interface to the OpenStack Swift API. + +Positional arguments: + <subcommand> + delete Delete a container or objects within a container. + download Download objects from containers. + list Lists the containers for the account or the objects + for a container. + post Updates meta information for the account, container, + or object; creates containers if not present. + copy Copies object, optionally adds meta + stat Displays information for the account, container, + or object. + upload Uploads files or directories to the given container. + capabilities List cluster capabilities. + tempurl Create a temporary URL. + auth Display auth related environment variables. + bash_completion Outputs option and flag cli data ready for + bash_completion. + +Examples: + %(prog)s download --help + + %(prog)s -A https://api.example.com/v1.0 \\ + -U user -K api_key stat -v + + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ + --os-username user --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-name project1 --os-project-domain-name domain1 \\ + --os-username user --os-user-domain-name domain1 \\ + --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-id 0123456789abcdef0123456789abcdef \\ + --os-user-id abcdef0123456789abcdef0123456789 \\ + --os-password password list + + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ + list + + %(prog)s list --lh +'''.strip('\n')) + + version = client_version + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') + + add_default_args(parser) + options, args = parse_args(parser, argv[1:], enforce_requires=False) if options['help'] or options['os_help']: @@ -1728,9 +1961,14 @@ Examples: parser.usage = globals()['st_%s_help' % args[0]] if options['insecure']: import requests - from requests.packages.urllib3.exceptions import \ - InsecureRequestWarning - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + try: + from requests.packages.urllib3.exceptions import \ + InsecureRequestWarning + except ImportError: + pass + else: + requests.packages.urllib3.disable_warnings( + InsecureRequestWarning) try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8afcde9..9e43237 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -69,12 +69,12 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False): + prefix=False, iso8601=False, ip_range=None): """Generates a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object or prefix if - a prefix-based temporary URL should be generated. Example: + a prefix-based temporary URL should be generated. Example: /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. :param seconds: time in seconds or ISO 8601 timestamp. If absolute is False and this is the string representation of an @@ -92,6 +92,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False, :param prefix: if True then a prefix-based temporary URL will be generated. :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp instead of a UNIX timestamp will be created. + :param ip_range: if a valid ip range, restricts the temporary URL to the + range of ips. :raises ValueError: if timestamp or path is not in valid format. :return: the path portion of a temporary URL """ @@ -155,8 +157,21 @@ def generate_temp_url(path, seconds, key, method, absolute=False, expiration = int(time.time() + timestamp) else: expiration = timestamp - hmac_body = u'\n'.join([method.upper(), str(expiration), - ('prefix:' if prefix else '') + path_for_body]) + + hmac_parts = [method.upper(), str(expiration), + ('prefix:' if prefix else '') + path_for_body] + + if ip_range: + if isinstance(ip_range, six.binary_type): + try: + ip_range = ip_range.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + 'ip_range must be representable as UTF-8' + ) + hmac_parts.insert(0, "ip=%s" % ip_range) + + hmac_body = u'\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility if not isinstance(key, six.binary_type): @@ -169,6 +184,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False, temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) + + if ip_range: + temp_url += u'&temp_url_ip_range={}'.format(ip_range) + if prefix: temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller @@ -376,3 +395,33 @@ def n_at_a_time(seq, n): def n_groups(seq, n): items_per_group = ((len(seq) - 1) // n) + 1 return n_at_a_time(seq, items_per_group) + + +def normalize_manifest_path(path): + if six.PY2 and isinstance(path, six.text_type): + path = path.encode('utf-8') + if path.startswith('/'): + return path[1:] + return path + + +class JSONableIterable(list): + def __init__(self, iterable): + self._iterable = iter(iterable) + try: + self._peeked = next(self._iterable) + self._has_items = True + except StopIteration: + self._peeked = None + self._has_items = False + + def __bool__(self): + return self._has_items + + __nonzero__ = __bool__ + + def __iter__(self): + if self._has_items: + yield self._peeked + for item in self._iterable: + yield item diff --git a/test-requirements.txt b/test-requirements.txt index 0ce9cec..b3ca5f8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,6 @@ -hacking>=0.10.0,<0.11 +hacking>=1.1.0,<1.2.0 # Apache-2.0 -coverage>=3.6 -mock>=1.2 -oslosphinx>=4.7.0 # Apache-2.0 -sphinx>=1.1.2,<1.2 -testrepository>=0.0.18 +coverage!=4.4,>=4.0 # Apache-2.0 +keystoneauth1>=3.4.0 # Apache-2.0 +mock>=1.2.0 # BSD +stestr>=2.0.0 # Apache-2.0 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index d60ae06..bae3044 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -46,11 +46,34 @@ class TestFunctional(unittest.TestCase): config.read(config_file) self.config = config if config.has_section('func_test'): - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') + if config.has_option('func_test', 'auth_uri'): + self.auth_url = config.get('func_test', 'auth_uri') + try: + self.auth_version = config.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + self.auth_version = last_piece + else: + raise + else: + auth_host = config.get('func_test', 'auth_host') + auth_port = config.getint('func_test', 'auth_port') + auth_ssl = config.getboolean('func_test', 'auth_ssl') + auth_prefix = config.get('func_test', 'auth_prefix') + self.auth_version = config.get('func_test', 'auth_version') + self.auth_url = "" + if auth_ssl: + self.auth_url += "https://" + else: + self.auth_url += "http://" + self.auth_url += "%s:%s%s" % ( + auth_host, auth_port, auth_prefix) + if self.auth_version == "1": + self.auth_url += 'v1.0' + try: self.account_username = config.get('func_test', 'account_username') @@ -59,15 +82,6 @@ class TestFunctional(unittest.TestCase): username = config.get('func_test', 'username') self.account_username = "%s:%s" % (account, username) self.password = config.get('func_test', 'password') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://" - else: - self.auth_url += "http://" - self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - else: self.skip_tests = True @@ -108,6 +122,7 @@ class TestFunctional(unittest.TestCase): self.conn.delete_container(container) except swiftclient.ClientException: pass + self.conn.close() def _check_account_headers(self, headers): headers_to_check = [ @@ -153,6 +168,18 @@ class TestFunctional(unittest.TestCase): self.assertTrue(len(containers) >= 1) self.assertEqual(self.containername_2, containers[0].get('name')) + # Test prefix + _, containers = self.conn.get_account(prefix='dne') + self.assertEqual(0, len(containers)) + + # Test delimiter + _, containers = self.conn.get_account( + prefix=self.containername, delimiter='_') + self.assertEqual(2, len(containers)) + self.assertEqual(self.containername, containers[0].get('name')) + self.assertTrue( + self.containername_2.startswith(containers[1].get('subdir'))) + def _check_container_headers(self, headers): self.assertTrue(headers.get('content-length')) self.assertTrue(headers.get('x-container-object-count')) @@ -485,9 +512,6 @@ class TestUsingKeystone(TestFunctional): self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - def setUp(self): - super(TestUsingKeystone, self).setUp() - class TestUsingKeystoneV3(TestFunctional): """ @@ -514,6 +538,3 @@ class TestUsingKeystoneV3(TestFunctional): return swiftclient.Connection(self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - - def setUp(self): - super(TestUsingKeystoneV3, self).setUp() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 260f1cb..12fbaa0 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -36,6 +36,8 @@ from swiftclient.service import ( SwiftService, SwiftError, SwiftUploadObject ) +from tests.unit import utils as test_utils + clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -119,25 +121,36 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) def test_create_with_large_object_headers(self): # md5 should not be initialized if large object headers are present - sr = self.sr('path', 'body', {'x-object-manifest': 'test'}) + sr = self.sr('path', 'body', {'x-object-manifest': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) - sr = self.sr('path', 'body', {'x-static-large-object': 'test'}) + def test_create_with_content_range_header(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_ignore_checksum(self): @@ -146,7 +159,7 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_content_length(self): @@ -155,10 +168,9 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertEqual(sr._content_length, 5) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) # Check Contentlength raises error if it isn't an integer self.assertRaises(SwiftError, self.sr, 'path', 'body', @@ -175,11 +187,17 @@ class TestSwiftReader(unittest.TestCase): # Check error is raised if expected etag doesn't match calculated md5. # md5 for a SwiftReader that has done nothing is # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing - sr = self.sr('path', BytesIO(b'body'), {'etag': 'doesntmatch'}) + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'doesntmatch').hexdigest()}) self.assertRaises(SwiftError, _consume, sr) sr = self.sr('path', BytesIO(b'body'), - {'etag': '841a2d689ad86bd1611447453c22c6fc'}) + {'etag': md5(b'body').hexdigest()}) + _consume(sr) + + # Should still work if etag was quoted + sr = self.sr('path', BytesIO(b'body'), + {'etag': '"%s"' % md5(b'body').hexdigest()}) _consume(sr) # Check error is raised if SwiftReader doesn't read the same length @@ -191,11 +209,13 @@ class TestSwiftReader(unittest.TestCase): _consume(sr) # Check that the iterator generates expected length and etag values - sr = self.sr('path', ['abc'.encode()] * 3, {}) + sr = self.sr('path', ['abc'.encode()] * 3, + {'content-length': 9, + 'etag': md5('abc'.encode() * 3).hexdigest()}) _consume(sr) self.assertEqual(sr._actual_read, 9) self.assertEqual(sr._actual_md5.hexdigest(), - '97ac82a5b825239e782d0339e2d7b910') + md5('abc'.encode() * 3).hexdigest()) class _TestServiceBase(unittest.TestCase): @@ -1070,6 +1090,83 @@ class TestService(unittest.TestCase): self.assertEqual(upload_obj_resp['path'], obj['path']) self.assertTrue(mock_open.return_value.closed) + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + class TestServiceUpload(_TestServiceBase): @@ -1128,14 +1225,9 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='テスト/dummy.dat', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 10, - 'segment_container': None, - 'use_slo': False, - 'checksum': True}) + options=dict(s._options, + segment_size=10, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1213,6 +1305,141 @@ class TestServiceUpload(_TestServiceBase): self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + def test_etag_mismatch_with_ignore_checksum(self): def _consuming_conn(*a, **kw): contents = a[2] @@ -1332,12 +1559,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1387,12 +1610,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = float(r['headers']['x-object-meta-mtime']) self.assertEqual(mtime, expected_mtime) @@ -1434,12 +1653,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) self.assertIs(r['success'], False) self.assertIn('md5 mismatch', str(r.get('error'))) @@ -1933,7 +2148,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) written_content.write.assert_called_once_with(b'objcontent') mock_conn.get_object.assert_called_once_with( @@ -1977,7 +2192,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) mock_utime.assert_called_once_with( 'test_o', (1454113727.682512, 1454113727.682512)) written_content.write.assert_called_once_with(b'objcontent') @@ -2023,7 +2238,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual(0, len(mock_utime.mock_calls)) written_content.write.assert_called_once_with(b'objcontent') @@ -2033,6 +2248,52 @@ class TestServiceDownload(_TestServiceBase): ) self.assertEqual(expected_r, actual_r) + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index fba9be7..d9ddb3e 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals -from genericpath import getmtime +import contextlib +from genericpath import getmtime +import getpass import hashlib import json import logging @@ -25,8 +27,8 @@ import unittest import textwrap from time import localtime, mktime, strftime, strptime -from requests.packages.urllib3.exceptions import InsecureRequestWarning import six +import sys import swiftclient from swiftclient.service import SwiftError @@ -35,12 +37,16 @@ import swiftclient.utils from os.path import basename, dirname from .utils import ( - CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, + CaptureOutput, fake_get_auth_keystone, FakeKeystone, StubResponse, MockHttpTest) from swiftclient.utils import ( EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning +except ImportError: + InsecureRequestWarning = None if six.PY2: BUILTIN_OPEN = '__builtin__.open' @@ -111,6 +117,20 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): return args, env +@contextlib.contextmanager +def patch_disable_warnings(): + if InsecureRequestWarning is None: + # If InsecureRequestWarning isn't available, disbale_warnings won't + # be either; they both came in with + # https://github.com/requests/requests/commit/811ee4e and left again + # in https://github.com/requests/requests/commit/8e17600 + yield None + else: + with mock.patch('requests.packages.urllib3.disable_warnings') \ + as patched: + yield patched + + @mock.patch.dict(os.environ, mocked_os_environ) class TestShell(unittest.TestCase): def setUp(self): @@ -278,6 +298,26 @@ class TestShell(unittest.TestCase): headers={'Skip-Middleware': 'Test'})]) @mock.patch('swiftclient.service.Connection') + def test_list_json(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, []], + ] + + argv = ["", "list", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + listing = [{'name': 'container'}, + {'name': u'\u263A', 'some-custom-key': 'and value'}] + expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' + self.assertEqual(output.out, expected) + + @mock.patch('swiftclient.service.Connection') def test_list_account(self, connection): # Test account listing connection.return_value.get_account.side_effect = [ @@ -473,7 +513,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) makedirs.reset_mock() @@ -490,7 +530,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks @@ -507,7 +547,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) self.assertEqual([], makedirs.mock_calls) @@ -553,7 +593,7 @@ class TestShell(unittest.TestCase): mock_shuffle.assert_any_call(['container']) mock_shuffle.assert_any_call(['object']) mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -577,7 +617,7 @@ class TestShell(unittest.TestCase): argv = ["", "download", "--all", "--no-shuffle"] swiftclient.shell.main(argv) self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -610,7 +650,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([ mock.call('pseudo'), ], mock_mkdir.mock_calls) @@ -623,7 +663,8 @@ class TestShell(unittest.TestCase): connection.return_value.put_object.return_value = EMPTY_ETAG connection.return_value.attempts = 0 argv = ["", "upload", "container", self.tmpfile, - "-H", "X-Storage-Policy:one"] + "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] swiftclient.shell.main(argv) connection.return_value.put_container.assert_called_once_with( 'container', @@ -636,7 +677,8 @@ class TestShell(unittest.TestCase): mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, - 'X-Storage-Policy': 'one'}, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) # upload to pseudo-folder (via <container> param) @@ -717,10 +759,9 @@ class TestShell(unittest.TestCase): mock.ANY, headers={ 'x-object-meta-mtime': mock.ANY, - 'x-static-large-object': 'true' }, query_string='multipart-manifest=put', - response_dict={}) + response_dict=mock.ANY) @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): @@ -778,11 +819,11 @@ class TestShell(unittest.TestCase): response_dict={}) expected_delete_calls = [ mock.call( - b'container1', b'old_seg1', + 'container1', 'old_seg1', response_dict={} ), mock.call( - b'container2', b'old_seg2', + 'container2', 'old_seg2', response_dict={} ) ] @@ -814,6 +855,46 @@ class TestShell(unittest.TestCase): self.assertFalse(connection.return_value.delete_object.mock_calls) @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_upload_delete_dlo_segments(self, connection): # Upload delete existing segments connection.return_value.head_container.return_value = { @@ -906,6 +987,44 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 0) @mock.patch('swiftclient.service.Connection') @@ -1625,7 +1744,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -1634,7 +1753,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True) + iso8601=False, prefix=True, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -1646,7 +1765,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -1655,7 +1774,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False) + iso8601=True, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1664,7 +1783,16 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1727,6 +1855,17 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -1743,7 +1882,7 @@ class TestShell(unittest.TestCase): argv = ["", "tempurl", "GET", "60", '/v1/a/c', "secret_key", "--absolute", '--prefix-based'] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for path %r' % (expected, output.err, '/v1/a/c')) @@ -1753,7 +1892,7 @@ class TestShell(unittest.TestCase): argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', "secret_key", "--absolute"] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for time %r' % (expected, output.err, bad_time)) @@ -1917,8 +2056,8 @@ class TestBase(unittest.TestCase): self._environ_vars = {} keys = list(os.environ.keys()) for k in keys: - if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') - or k.startswith('OS_')): + if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or + k.startswith('OS_')): self._environ_vars[k] = os.environ.pop(k) def _replace_swift_env_vars(self): @@ -2243,17 +2382,66 @@ class TestParsing(TestBase): os_opts = {"password": "secret", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) os_opts = {"username": "user", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) os_opts = {"username": "user", "password": "secret"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') def test_no_tenant_name_or_id_v2(self): os_opts = {"password": "secret", @@ -2406,7 +2594,17 @@ class TestKeystoneOptions(MockHttpTest): cmd_args=cmd_args) ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' ks_token = 'fake_auth_token' + # check correct auth version gets used + key = 'auth-version' fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) + if no_auth: + fake_ks2 = fake_ks3 = None + elif opts.get(key, self.defaults.get(key)) == '2.0': + fake_ks2 = fake_ks + fake_ks3 = None + else: + fake_ks2 = None + fake_ks3 = fake_ks # fake_conn will check that storage_url and auth_token are as expected endpoint = os_opts.get('storage-url', ks_endpoint) token = os_opts.get('auth-token', ks_token) @@ -2414,12 +2612,11 @@ class TestKeystoneOptions(MockHttpTest): storage_url=endpoint, auth_token=token) - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)), \ + with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ + mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ mock.patch('swiftclient.client.http_connection', fake_conn), \ mock.patch.dict(os.environ, env, clear=True), \ - mock.patch('requests.packages.urllib3.disable_warnings') as \ - mock_disable_warnings: + patch_disable_warnings() as mock_disable_warnings: try: swiftclient.shell.main(args) except SystemExit as e: @@ -2427,23 +2624,19 @@ class TestKeystoneOptions(MockHttpTest): except SwiftError as err: self.fail('Unexpected SwiftError: %s' % err) - if 'insecure' in flags: - self.assertEqual([mock.call(InsecureRequestWarning)], - mock_disable_warnings.mock_calls) - else: - self.assertEqual([], mock_disable_warnings.mock_calls) + if InsecureRequestWarning is not None: + if 'insecure' in flags: + self.assertEqual([mock.call(InsecureRequestWarning)], + mock_disable_warnings.mock_calls) + else: + self.assertEqual([], mock_disable_warnings.mock_calls) if no_auth: - # check that keystone client was not used and terminate tests - self.assertIsNone(getattr(fake_ks, 'auth_version')) - self.assertEqual(len(fake_ks.calls), 0) + # We patched out both keystoneclient versions to be None; + # they *can't* have been used and if we tried to, we would + # have raised ClientExceptions return - # check correct auth version was passed to _import_keystone_client - key = 'auth-version' - expected = opts.get(key, self.defaults.get(key)) - self.assertEqual(expected, fake_ks.auth_version) - # check args passed to keystone Client __init__ self.assertEqual(len(fake_ks.calls), 1) actual_args = fake_ks.calls[0] @@ -2814,9 +3007,9 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.account = 'AUTH_alice' # keystone returns endpoint for another account - fake_ks = FakeKeystone(endpoint='http://example.com:8080/v1/AUTH_bob', - token='bob_token') - self.fake_ks_import = _make_fake_import_keystone_client(fake_ks) + self.fake_ks = FakeKeystone( + endpoint='http://example.com:8080/v1/AUTH_bob', + token='bob_token') self.cont = 'c1' self.cont_path = '/v1/%s/%s' % (self.account, self.cont) @@ -2846,12 +3039,12 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): Modify response code to 200 if cross account permissions match. """ status = 403 - if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) - and read_ok and method in ('GET', 'HEAD')): + if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and + read_ok and method in ('GET', 'HEAD')): status = 200 elif (path.startswith('/v1/%s/%s%s' - % (self.account, self.cont, self.obj)) - and write_ok and method in ('PUT', 'POST', 'DELETE')): + % (self.account, self.cont, self.obj)) and + write_ok and method in ('PUT', 'POST', 'DELETE')): status = 200 return status return on_request @@ -2895,8 +3088,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -2918,8 +3110,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): on_request=req_handler) args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -2945,8 +3136,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): '--segment-size=10', '--segment-container=%s' % self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -2984,8 +3174,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): cmd_args=[self.cont, self.obj, '--leave-segments', '--segment-size=10']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3021,8 +3210,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3079,8 +3267,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3101,8 +3288,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3120,8 +3306,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3145,8 +3330,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): fake_conn = self.fake_http_connection(resp, on_request=req_handler) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3163,8 +3347,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): fake_conn = self.fake_http_connection(403) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 2cd5cf1..2d45deb 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -26,11 +26,13 @@ import tempfile from hashlib import md5 from six import binary_type from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, - FakeKeystone, _make_fake_import_keystone_client) + FakeKeystone) from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException from swiftclient import client as c import swiftclient.utils import swiftclient @@ -320,8 +322,7 @@ class TestGetAuth(MockHttpTest): # TestConnection.test_timeout_passed_down but is required to check that # get_auth does the right thing when it is not passed a timeout arg fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): c.get_auth('http://www.test.com', 'asdf', 'asdf', os_options=dict(tenant_name='tenant'), auth_version="2.0", timeout=42.0) @@ -575,6 +576,67 @@ class TestGetAuth(MockHttpTest): self.assertTrue(url.startswith("http")) self.assertTrue(token) + def test_get_auth_keystone_versionless(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://authurl', 'user', 'key', {}) + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionless_auth_version_set(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): + c.get_auth_keystone('http://auth_url', 'user', 'key', + {}, auth_version='2.0') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v2.0', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + def test_auth_with_session(self): mock_session = mock.MagicMock() mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' @@ -642,6 +704,18 @@ class TestGetAccount(MockHttpTest): 'x-auth-token': 'asdf'}), ]) + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=-") + c.get_account('http://www.test.com/v1/acct', 'asdf', + delimiter='-') + self.assertRequests([ + ('GET', '/v1/acct?format=json&delimiter=-', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + class TestHeadAccount(MockHttpTest): @@ -680,9 +754,10 @@ class TestPostAccount(MockHttpTest): c.http_connection = self.fake_http_connection(200, headers={ 'X-Account-Meta-Color': 'blue', }, body='foo') + headers = {'x-account-meta-shape': 'square'} resp_headers, body = c.post_account( 'http://www.tests.com/path/to/account', 'asdf', - {'x-account-meta-shape': 'square'}, query_string='bar=baz', + headers, query_string='bar=baz', data='some data') self.assertEqual('blue', resp_headers.get('x-account-meta-color')) self.assertEqual('foo', body) @@ -691,6 +766,8 @@ class TestPostAccount(MockHttpTest): 'some data', {'x-auth-token': 'asdf', 'x-account-meta-shape': 'square'}) ]) + # Check that we didn't mutate the request ehader dict + self.assertEqual(headers, {'x-account-meta-shape': 'square'}) def test_server_error(self): body = 'c' * 65 @@ -1160,9 +1237,20 @@ class TestHeadObject(MockHttpTest): }), ]) + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + class TestPutObject(MockHttpTest): + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) @@ -1220,6 +1308,7 @@ class TestPutObject(MockHttpTest): self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, UserWarning)) + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_server_error(self): body = 'c' * 60 headers = {'foo': 'bar'} @@ -1234,7 +1323,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', 'content-type': ''}), + 'x-auth-token': 'asdf', + 'content-type': ''}), ]) def test_query_string(self): @@ -1375,7 +1465,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - def test_no_content_type(self): + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_no_content_type_old_requests(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response @@ -1385,6 +1476,17 @@ class TestPutObject(MockHttpTest): request_header = resp.requests_params['headers'] self.assertEqual(request_header['content-type'], b'') + @mock.patch('swiftclient.requests.__version__', '2.4.0') + def test_no_content_type_new_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertNotIn('content-type', request_header) + def test_content_type_in_headers(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) @@ -1419,6 +1521,11 @@ class TestPostObject(MockHttpTest): 'X-Object-Meta-Test': 'mymeta', 'X-Delete-At': delete_at}), ]) + # Check that the request header dict didn't get mutated + self.assertEqual(args[-1], { + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at, + }) def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') @@ -1794,6 +1901,57 @@ class TestHTTPConnection(MockHttpTest): self.assertFalse(resp.read()) self.assertTrue(resp.closed) + @unittest.skipIf(six.PY3, 'python2 specific test') + def test_response_python2_headers(self): + '''Test utf-8 headers in Python 2. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + '\xd8\xaa-unicode': '\xd8\xaa-value', + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), + ('empty-header', ''), + ('etag', '"%s"' % EMPTY_ETAG)]), + dict(resp.getheaders())) + + @unittest.skipIf(six.PY2, 'python3 specific test') + def test_response_python3_headers(self): + '''Test latin1-encoded headers in Python 3. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + b'\xd8\xaa-unicode'.decode('iso-8859-1'): + b'\xd8\xaa-value'.decode('iso-8859-1'), + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-unicode')) + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\u062a-unicode', '\u062a-value'), + ('empty-header', ''), + ('etag', ('"%s"' % EMPTY_ETAG))]), + dict(resp.getheaders())) + class TestConnection(MockHttpTest): @@ -1933,6 +2091,71 @@ class TestConnection(MockHttpTest): self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + def test_resp_read_on_server_error(self): conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) @@ -2238,8 +2461,7 @@ class TestConnection(MockHttpTest): 'http://auth.example.com', 'user', 'password', timeout=33.0, os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): with mock.patch.multiple('swiftclient.client', http_connection=shim_connection, sleep=mock.DEFAULT): @@ -2322,6 +2544,9 @@ class TestConnection(MockHttpTest): def read(self, *args, **kwargs): return '' + def close(self): + pass + def local_http_connection(url, proxy=None, cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, timeout=None): @@ -2424,16 +2649,17 @@ class TestConnection(MockHttpTest): def test_head_object(self): headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): with mock.patch('swiftclient.client.get_auth', lambda *a, **k: ('http://url:8080/v1/a', 'token')): conn = c.Connection() conn.head_object('c1', 'o1', - headers=headers) + headers=headers, query_string=query_string) self.assertEqual(1, len(self.request_log), self.request_log) self.assertRequests([ - ('HEAD', '/v1/a/c1/o1', '', { + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', }), @@ -2671,6 +2897,16 @@ class TestLogging(MockHttpTest): self.assertIn('X-Storage-Token', output) self.assertIn(unicode_token_value, output) + @mock.patch('swiftclient.client.logger.debug') + def test_unicode_path(self, mock_log): + path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + c.http_log(['GET', path], {}, + MockHttpResponse(status=200, headers=[]), '') + request_log_line = mock_log.mock_calls[0] + self.assertEqual('REQ: %s', request_log_line[1][0]) + self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + request_log_line[1][1]) + class TestCloseConnection(MockHttpTest): @@ -2680,6 +2916,9 @@ class TestCloseConnection(MockHttpTest): self.assertIsNone(conn.http_conn) conn.close() self.assertIsNone(conn.http_conn) + # Can re-close + conn.close() + self.assertIsNone(conn.http_conn) def test_close_ok(self): url = 'http://www.test.com' @@ -2690,7 +2929,7 @@ class TestCloseConnection(MockHttpTest): self.assertEqual(len(conn.http_conn), 2) http_conn_obj = conn.http_conn[1] self.assertIsInstance(http_conn_obj, c.HTTPConnection) - self.assertFalse(hasattr(http_conn_obj, 'close')) + self.assertTrue(hasattr(http_conn_obj, 'close')) conn.close() @@ -2981,10 +3220,11 @@ class TestServiceToken(MockHttpTest): self.assertEqual(conn.attempts, 1) def test_service_token_post_container(self): + headers = {'X-Container-Meta-Color': 'blue'} with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(201)): conn = self.get_connection() - conn.post_container('container1', {}) + conn.post_container('container1', headers) self.assertEqual(1, len(self.request_log), self.request_log) for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) @@ -2994,6 +3234,8 @@ class TestServiceToken(MockHttpTest): self.assertEqual('http://storage_url.com/container1', actual['full_path']) self.assertEqual(conn.attempts, 1) + # Check that we didn't mutate the request header dict + self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) def test_service_token_put_container(self): with mock.patch('swiftclient.client.http_connection', diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index adead00..97abc44 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,6 +14,7 @@ # limitations under the License. import gzip +import json import unittest import mock import six @@ -152,6 +153,54 @@ class TestTempURL(unittest.TestCase): self.assertIsInstance(url, type(self.url)) @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + + @mock.patch('hmac.HMAC') def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', @@ -590,3 +639,41 @@ class TestGetBody(unittest.TestCase): {'content-encoding': 'gzip'}, buf.getvalue()) self.assertEqual({'test': u'\u2603'}, result) + + +class JSONTracker(object): + def __init__(self, data): + self.data = data + self.calls = [] + + def __iter__(self): + for item in self.data: + self.calls.append(('read', item)) + yield item + + def write(self, s): + self.calls.append(('write', s)) + + +class TestJSONableIterable(unittest.TestCase): + def test_json_dump_iterencodes(self): + t = JSONTracker([1, 'fish', 2, 'fish']) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('read', 1), + ('write', '[1'), + ('read', 'fish'), + ('write', ', "fish"'), + ('read', 2), + ('write', ', 2'), + ('read', 'fish'), + ('write', ', "fish"'), + ('write', ']'), + ]) + + def test_json_dump_empty_iter(self): + t = JSONTracker([]) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('write', '[]'), + ]) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f7e48d3..025a234 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -79,6 +79,10 @@ class StubResponse(object): self.body = body self.headers = headers or {} + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, + self.body, self.headers) + def fake_http_connect(*code_iter, **kwargs): """ @@ -103,7 +107,6 @@ def fake_http_connect(*code_iter, **kwargs): self.etag = etag self.content = self.body = body self.timestamp = timestamp - self._is_closed = True self.headers = headers or {} self.request = None @@ -163,6 +166,9 @@ def fake_http_connect(*code_iter, **kwargs): def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) + def close(self): + pass + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) @@ -241,7 +247,8 @@ class MockHttpTest(unittest.TestCase): parsed, _conn = _orig_http_connection(url, proxy=proxy) class RequestsWrapper(object): - pass + def close(self): + pass conn = RequestsWrapper() def request(method, path, *args, **kwargs): @@ -554,9 +561,22 @@ class FakeKeystone(object): pass -def _make_fake_import_keystone_client(fake_import): - def _fake_import_keystone_client(auth_version): - fake_import.auth_version = auth_version - return fake_import, fake_import +class FakeStream(object): + def __init__(self, size): + self.bytes_read = 0 + self.size = size + + def read(self, size=-1): + if self.bytes_read == self.size: + return b'' - return _fake_import_keystone_client + if size == -1 or size + self.bytes_read > self.size: + remaining = self.size - self.bytes_read + self.bytes_read = self.size + return b'A' * remaining + + self.bytes_read += size + return b'A' * size + + def __len__(self): + return self.size diff --git a/tools/swift.bash_completion b/tools/swift.bash_completion new file mode 100644 index 0000000..2f98a6b --- /dev/null +++ b/tools/swift.bash_completion @@ -0,0 +1,32 @@ +declare -a _swift_opts # lazy init + +_swift_get_current_opt() +{ + local opt + for opt in ${_swift_opts[@]} ; do + if [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt\$") > 0 ]] || [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt ") > 0 ]] ; then + echo $opt + return 0 + fi + done + echo "" + return 0 +} + +_swift() +{ + local opt cur prev sflags + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [ "x$_swift_opts" == "x" ] ; then + _swift_opts=(`swift bash_completion "$sbc" | sed -e "s/-[-A-Za-z0-9_]*//g" -e "s/ */ /g"`) + fi + + opt="$(_swift_get_current_opt)" + COMPREPLY=($(compgen -W "$(swift bash_completion $opt)" -- ${cur})) + + return 0 +} +complete -F _swift swift diff --git a/tools/tox_install.sh b/tools/tox_install.sh deleted file mode 100755 index 15aa9de..0000000 --- a/tools/tox_install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Client constraint file contains this client version pin that is in conflict -# with installing the client from source. We should remove the version pin in -# the constraints file before applying it for from-source installation. - -set -e - -if [[ -z "$CONSTRAINTS_FILE" ]]; then - echo 'WARNING: expected $CONSTRAINTS_FILE to be set' >&2 - PIP_FLAGS=(-U) -else - # NOTE(tonyb): Place this in the tox enviroment's log dir so it will get - # published to logs.openstack.org for easy debugging. - localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - - if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" - fi - curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - - pip install -c"$localfile" openstack-requirements - - # This is the main purpose of the script: Allow local installation of - # the current repo. It is listed in constraints file and thus any - # install will be constrained and we need to unconstrain it. - edit-constraints "$localfile" -- "$CLIENT_NAME" - PIP_FLAGS=(-c"$localfile" -U) -fi - -pip install "${PIP_FLAGS[@]}" "$@" @@ -1,54 +1,74 @@ [tox] -envlist = py27,py34,py35,pypy,pep8 +envlist = py37,py36,py35,py27,pypy,pep8 minversion = 2.0 skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} +install_command = python -m pip install -U {opts} {packages} +list_dependencies_command = python -m pip freeze setenv = - LANG=en_US.utf8 + LANG=en_US.utf-8 VIRTUAL_ENV={envdir} - BRANCH_NAME=master - CLIENT_NAME=python-swiftclient - CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt .[keystone] commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ - -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' - python setup.py testr --testr-args="{posargs}" + -print0) | xargs -0 rm -rf' + stestr run {posargs} whitelist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] +basepython = python3 commands = - flake8 swiftclient tests + python -m flake8 swiftclient tests [testenv:venv] +basepython = python3 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage - coverage report +basepython = python3 +setenv = + PYTHON=coverage run --source swiftclient --parallel-mode +commands = + stestr run + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [testenv:func] +basepython = python3 setenv = - {[testenv]setenv} OS_TEST_PATH=tests.functional + PYTHON=coverage run --source swiftclient --parallel-mode whitelist_externals = coverage rm commands = - python setup.py testr --coverage --testr-args="--concurrency=1" + stestr run --concurrency=1 + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage +[testenv:py2func] +basepython=python2 +setenv = {[testenv:func]setenv} +whitelist_externals = {[testenv:func]whitelist_externals} +commands = {[testenv:func]commands} + [testenv:docs] +basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx + python setup.py build_sphinx -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: @@ -59,11 +79,16 @@ commands= # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = H101,H301,H306,H401,H403,H404,H405 +# W504: line break after binary operator +ignore = H101,H301,H306,H401,H403,H404,H405,W504 +# H106: Don’t put vim configuration in source files +# H203: Use assertIs(Not)None to check for None +enable-extensions=H106,H203 show-source = True exclude = .venv,.tox,dist,doc,*egg [testenv:bindep] +basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -71,3 +96,16 @@ exclude = .venv,.tox,dist,doc,*egg usedevelop = False deps = bindep commands = bindep test + +[testenv:releasenotes] +basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt +commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + -r{toxinidir}/test-requirements.txt + .[keystone] |