diff options
39 files changed, 1468 insertions, 510 deletions
@@ -2,9 +2,13 @@ set -e export OS_TEST_PATH='tests.functional' -python setup.py testr --coverage --testr-args="--concurrency=1" +export PYTHON='coverage run --source swiftclient --parallel-mode' +stestr run --concurrency=1 RET=$? +coverage combine +coverage html -d cover +coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage exit $RET @@ -7,6 +7,7 @@ dist/ .DS_Store *.log .testrepository +.stestr/ subunit.log build swiftclient/versioninfo @@ -16,3 +17,4 @@ cover/ coverage.xml doc/build doc/source/api/ +.idea @@ -1,4 +1,4 @@ [gerrit] -host=review.openstack.org +host=review.opendev.org port=29418 project=openstack/python-swiftclient.git @@ -58,7 +58,8 @@ Madhuri Kumari <madhuri.rai07@gmail.com> madhuri <madhuri@madhuri-VirtualBox.(no Morgan Fainberg <morgan.fainberg@gmail.com> <m@metacloud.com> Hua Zhang <zhuadl@cn.ibm.com> <zhuadl@cn.ibm.com> Yummy Bian <yummy.bian@gmail.com> <yummy.bian@gmail.com> -Alistair Coles <alistair.coles@hpe.com> <alistair.coles@hp.com> +Alistair Coles <alistairncoles@gmail.com> <alistair.coles@hp.com> +Alistair Coles <alistairncoles@gmail.com> <alistair.coles@hpe.com> Tong Li <litong01@us.ibm.com> <litong01@us.ibm.com> Paul Luse <paul.e.luse@intel.com> <paul.e.luse@intel.com> Yuan Zhou <yuan.zhou@intel.com> <yuan.zhou@intel.com> @@ -94,3 +95,5 @@ Andreas Jaeger <aj@suse.de> <aj@suse.com> Shashi Kant <shashi.kant@nectechnologies.in> Nandini Tata <nandini.tata@intel.com> <nandini.tata.15@gmail.com> Flavio Percoco <flaper87@gmail.com> +Timur Alperovich <timuralp@swiftstack.com> <timur@timuralp.com> +Thiago da Silva <thiagodasilva@gmail.com> <thiago@redhat.com> diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..5228f20 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_path=${OS_TEST_PATH:-./tests/unit} +top_dir=./ + diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index f3fca90..0000000 --- a/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..ac1d966 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,59 @@ +- job: + name: swiftclient-swift-functional + parent: swift-dsvm-functional + description: | + Run swift's functional tests with python-swiftclient + installed from source instead as package from PyPI. + # Ensure that we install python-swiftclient from git and + # do not install from pypi. This is needed since the parent + # job sets zuul_work_dir to the swift directory and uses tox + # for installation. + required-projects: + - opendev.org/openstack/python-swiftclient + +- job: + name: swiftclient-functional + parent: swift-dsvm-functional + description: | + Run functional tests of python-swiftclient with + python-swiftclient installed from source instead as package from + PyPI. + required-projects: + - opendev.org/openstack/python-swiftclient + vars: + # Override value from parent job to use swiftclient tests + zuul_work_dir: "{{ zuul.projects['opendev.org/openstack/python-swiftclient'].src_dir }}" + +- job: + name: swiftclient-functional-py2 + parent: swiftclient-functional + description: | + Run functional tests of python-swiftclient under Python 2 + vars: + tox_envlist: py2func + +- project: + templates: + - check-requirements + - lib-forward-testing + - openstack-lower-constraints-jobs + - openstack-pypy-jobs-nonvoting + - openstack-python-jobs + - openstack-python35-jobs + - openstack-python36-jobs + - openstack-python37-jobs + - publish-openstack-docs-pti + - release-notes-jobs-python3 + check: + jobs: + - swiftclient-swift-functional + - swiftclient-functional + - swiftclient-functional-py2 + gate: + jobs: + - swiftclient-swift-functional + - swiftclient-functional + - swiftclient-functional-py2 + post: + jobs: + - openstack-tox-cover @@ -2,7 +2,7 @@ Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) Alexandra Settle (alexandra.settle@rackspace.com) Alexis Lee (lxsli@hpe.com) -Alistair Coles (alistair.coles@hpe.com) +Alistair Coles (alistairncoles@gmail.com) Andreas Jaeger (aj@suse.de) Andrew Welleck (awellec@us.ibm.com) Andy McCrae (andy.mccrae@gmail.com) @@ -12,6 +12,7 @@ Ben McCann (ben@benmccann.com) Cedric Brandily (zzelle@gmail.com) Chaozhe.Chen (chaozhe.chen@easystack.cn) Charles Hsu (charles0126@gmail.com) +Chen (dstbtgagt@foxmail.com) Cheng Li (shcli@cn.ibm.com) Chmouel Boudjnah (chmouel@enovance.com) Chris Buccella (chris.buccella@antallagon.com) @@ -35,6 +36,7 @@ Dirk Mueller (dirk@dmllr.de) Donagh McCabe (donagh.mccabe@hpe.com) Doug Hellmann (doug@doughellmann.com) EdLeafe (ed@leafe.com) +Erik Olof Gunnar Andersson (eandersson@blizzard.com) Fabien Boucher (fabien.boucher@enovance.com) Feng Liu (mefengliu23@gmail.com) Flavio Percoco (flaper87@gmail.com) @@ -71,6 +73,7 @@ Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) +lingyongxu (lyxu@fiberhome.com) liuyamin (liuyamin@fiberhome.com) Luis de Bethencourt (luis@debethencourt.com) M V P Nitesh (m.nitesh@nectechnologies.in) @@ -83,10 +86,13 @@ Matthew Oliver (matt@oliver.net.au) Matthieu Huin (mhu@enovance.com) Mike Widman (mwidman@endurancewindpower.com) Min Min Ren (rminmin@cn.ibm.com) +mmcardle (mark.mcardle@sohonet.com) Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) Nelson Marcos (nelsonmarcos@gmail.com) +Nguyen Hai (nguyentrihai93@gmail.com) +Nguyen Hai Truong (truongnh@vn.fujitsu.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -98,6 +104,7 @@ Peter Lisak (peter.lisak@firma.seznam.cz) Petr Kovar (pkovar@redhat.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) Pratik Mallya (pratik.mallya@gmail.com) +qingszhao (zhao.daqing@99cloud.net) Qiu Yu (qiuyu@ebaysf.com) Ray Chen (oldsharp@163.com) ricolin (rico.l@inwinstack.com) @@ -110,6 +117,7 @@ Sean Dague (sean@dague.net) Sergey Gotliv (sgotliv@redhat.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Shane Wang (shane.wang@intel.com) +shangxiaobj (shangxiaobj@inspur.com) Shashi Kant (shashi.kant@nectechnologies.in) Shashirekha Gundur (shashirekha.j.gundur@intel.com) shu-mutou (shu-mutou@rf.jp.nec.com) @@ -118,24 +126,29 @@ Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Steve Martinelli (stevemar@ca.ibm.com) Steven Hardy (shardy@redhat.com) Stuart McLaren (stuart.mclaren@hpe.com) +sunjia (sunjia@inspur.com) Sushil Kumar (sushil.kumar2@globallogic.com) tanlin (lin.tan@intel.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) TheSriram (sriram@klusterkloud.com) -Thiago da Silva (thiago@redhat.com) +Thiago da Silva (thiagodasilva@gmail.com) Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) Tim Burke (tim.burke@gmail.com) Timur Alperovich (timuralp@swiftstack.com) Tong Li (litong01@us.ibm.com) Tony Breeds (tony@bakeyournoodle.com) +Tovin Seven (vinhnt@vn.fujitsu.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) Vitaly Gridnev (vgridnev@mirantis.com) +Vu Cong Tuan (tuanvc@vn.fujitsu.com) +wangqi (wang.qi@99cloud.net) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) +wu.chunyang (wu.chunyang@99cloud.net) YangLei (yanglyy@cn.ibm.com) yangxurong (yangxurong@huawei.com) You Yamagata (bi.yamagata@gmail.com) @@ -149,3 +162,5 @@ zhang-jinnan (ben.os@99cloud.net) zhangyanxian (zhangyanxianmail@163.com) zheng yin (yin.zheng@easystack.cn) Zhenguo Niu (zhenguo@unitedstack.com) +ZhijunWei (wzj334965317@outlook.com) +zhubx007 (zhu.boxiang@99cloud.net) @@ -1,3 +1,48 @@ +3.7.0 +----- + +* Added the delimiter keyword parameter to `get_account()` to match the + functionality of `get_container()`. + +* Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + +* Various other minor bug fixes and improvements. + + +3.6.0 +----- + +* Add the `--prompt` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + `--key`, `--os-password` or an environment variable will be ignored. + +* Added bash completion support to the `swift` CLI. Enable this by sourcing + the included `tools/swift.bash_completion` file. Make it permanent by + including this file in the system's `/etc/bash_completion.d` directory. + +* Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported in Swift 2.19.0 or later. + +* The client.py SDK now supports a `query_string` option on the + `head_object()` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + +* The client.py SDK now respects `region_name` when using sessions. + +* Added a `.close()` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + +* Fixed a bug where using `--debug` in the CLI with unicode account names + would cause a client crash. + +* Make OS_AUTH_URL work in DevStack (for testing) by default. + +* Dropped Python 3.4 testing. + +* Various other minor bug fixes and improvements. + + 3.5.0 ----- @@ -2,8 +2,8 @@ Team and repository tags ======================== -.. image:: https://governance.openstack.org/badges/python-swiftclient.svg - :target: https://governance.openstack.org/reference/tags/index.html +.. image:: https://governance.openstack.org/tc/badges/python-swiftclient.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html .. Change things from this point on @@ -11,13 +11,9 @@ Python bindings to the OpenStack Object Storage API =================================================== .. image:: https://img.shields.io/pypi/v/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ + :target: https://pypi.org/project/python-swiftclient/ :alt: Latest Version -.. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ - :alt: Downloads - This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). @@ -41,16 +37,17 @@ __ https://github.com/openstack/swift * `Source`_ * `Specs`_ * `How to Contribute`_ +* `Release Notes`_ -.. _PyPI: https://pypi.python.org/pypi/python-swiftclient +.. _PyPI: https://pypi.org/project/python-swiftclient .. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient -.. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient +.. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ - +.. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient .. contents:: Contents: :local: diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..6cdad2a --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,5 @@ +keystoneauth1>=3.4.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.20.0 # Apache-2.0 diff --git a/doc/source/_static/.gitignore b/doc/source/_static/.gitignore new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/doc/source/_static/.gitignore diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index b57c25e..1762989 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -139,6 +139,10 @@ swift optional arguments compression should be disabled by default by the system SSL library. +``--prompt`` + Prompt user to enter a password which overrides any password supplied via + ``--key``, ``--os-password`` or environment variables. + Authentication ~~~~~~~~~~~~~~ @@ -241,13 +245,10 @@ storage URL options shown below: --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ list -.. We need the backslash below in order to indent the note -\ - - .. note:: +.. note:: - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. CLI commands ~~~~~~~~~~~~ @@ -735,15 +736,15 @@ is passed, the Unix timestamp when the temporary URL will expire. But beyond that, ``time`` can also be specified as an ISO 8601 timestamp in one of following formats: - i) Complete date: YYYY-MM-DD (eg 1997-07-16) +i) Complete date: YYYY-MM-DD (e.g. 1997-07-16) - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) +ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (e.g. 1997-07-16T19:20:30) - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) +iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (e.g. 1997-07-16T19:20:30Z) Please be aware that if you don't provide the UTC designator (i.e., Z) the timestamp is generated using your local timezone. If only a date is @@ -877,17 +878,14 @@ Download an object from a container: testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] -.. We need the backslash below in order to indent the note -\ - - .. note:: +.. note:: - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name <object-name> is an option that will upload - file and name object to <object-name> or upload directory and use <object-name> as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. For example: diff --git a/doc/source/conf.py b/doc/source/conf.py index 3505f13..85dd81e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -31,8 +31,11 @@ sys.path.insert(0, ROOT) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'oslosphinx'] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'openstackdocstheme'] autoclass_content = 'both' autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] @@ -50,17 +53,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Swiftclient' copyright = u'2013-2016 OpenStack, LLC.' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -import swiftclient.version -release = swiftclient.version.version_string -version = swiftclient.version.version_string - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -104,7 +98,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -#html_theme = 'nature' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/source/index.rst b/doc/source/index.rst index 3c2cb1e..ab05c6b 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -39,17 +39,17 @@ Indices and tables License ~~~~~~~ - Copyright 2013 OpenStack, LLC. +Copyright 2013 OpenStack, LLC. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +* http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst index 926b1b9..d6f9819 100644 --- a/doc/source/introduction.rst +++ b/doc/source/introduction.rst @@ -16,41 +16,41 @@ Use Cases Alongside the command line tool, the ``python-swiftclient`` includes two levels of API: - * A low level client API that provides simple Python wrappers around the - various authentication mechanisms and the individual HTTP requests. - * A high level service API that provides methods for performing common - operations in parallel on a thread pool. +* A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. +* A high level service API that provides methods for performing common + operations in parallel on a thread pool. Example use cases: - * Uploading and retrieving data - Use the command line tool if you are simply uploading and downloading - files and directories to and from your filesystem. The command line tool - can be integrated into a shell script to automate tasks. - - * Integrating into an automated Python workflow - Use the ``SwiftService`` API to perform operations offered by the CLI - if your use case requires integration with a Python-based workflow. - This method offers greater control and flexibility over individual object - operations, such as the metadata set on each object. The ``SwiftService`` - class provides methods to perform multiple sets of operations against a - swift object store using a configurable shared thread pool. A single - instance of the ``SwiftService`` class can be shared between multiple - threads in your own code. - - * Developing an application in Python to access a swift object store - Use the ``SwiftService`` API to develop Python applications that use - swift to store and retrieve objects. A ``SwiftService`` instance provides - a configurable thread pool for performing all operations supported by the - CLI. - - * Fine-grained control over threading or the requests being performed - Use the ``Connection`` API if your use case requires fine grained control - over advanced features or you wish to use your own existing threading - model. Examples of advanced features requiring the use of the - ``Connection`` API include creating an SLO manifest that references - already existing objects, or fine grained control over the query strings - supplied with each HTTP request. +* Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + +* Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + +* Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + +* Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. Important considerations ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,19 +66,19 @@ the proper use case, and not treat the storage like a traditional filesystem. There are two main restrictions to bear in mind when designing an application that uses an object store: - * You cannot rename objects. Due to fact that the name of an object is one - of the factors that determines where the object and its replicas are stored, - renaming would require multiple copies of the data to be moved between - physical storage devices. If you want to rename an object you must upload - to the new location, or make a server side copy request to the new location, - and then delete the original. - - * You cannot modify objects. Objects are stored in multiple locations and - are checked for integrity based on the MD5 sum calculated during - upload. In order to modify the contents of an object, the entire desired - contents must be re-uploaded. In certain special cases it is possible to - work around this restriction using large objects, but no general - file-like access is available to modify a stored object. +* You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + +* You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. Objects cannot be locked ------------------------ diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index 8efd43a..cc4dcc2 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -26,10 +26,10 @@ the auth version based on the combination of options specified, but supplying options from multiple different auth versions can cause unexpected behaviour. - .. note:: +.. note:: - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. Keystone V3 ~~~~~~~~~~~ @@ -109,17 +109,17 @@ in this dictionary are described below, along with their defaults: Options ~~~~~~~ - ``retries``: ``5`` +``retries``: ``5`` The number of times that the library should attempt to retry HTTP actions before giving up and reporting a failure. - ``container_threads``: ``10`` +``container_threads``: ``10`` - ``object_dd_threads``: ``10`` +``object_dd_threads``: ``10`` - ``object_uu_threads``: ``10`` +``object_uu_threads``: ``10`` - ``segment_threads``: ``10`` +``segment_threads``: ``10`` The above options determine the size of the available thread pools for performing swift operations. Container operations (such as listing a container) operate in the container threads, and a similar pattern @@ -131,86 +131,86 @@ Options ``uu`` and ``dd``. This stands for "upload/update" and "download/delete", and the corresponding actions will be run on separate threads pools. - ``segment_size``: ``None`` +``segment_size``: ``None`` If specified, this option enables uploading of large objects. Should the object being uploaded be larger than 5G in size, this option is mandatory otherwise the upload will fail. This option should be specified as a size in bytes. - ``use_slo``: ``False`` +``use_slo``: ``False`` Used in combination with the above option, ``use_slo`` will upload large objects as static rather than dynamic. Only static large objects provide error checking for the downloaded object, so we recommend this option. - ``segment_container``: ``None`` +``segment_container``: ``None`` Allows the user to select the container into which large object segments will be uploaded. We do not recommend changing this value as it could make locating orphaned segments more difficult in the case of errors. - ``leave_segments``: ``False`` +``leave_segments``: ``False`` Setting this option to true means that when deleting or overwriting a large object, its segments will be left in the object store and must be cleaned up manually. This option can be useful when sharing large object segments between multiple objects in more advanced scenarios, but must be treated with care, as it could lead to ever increasing storage usage. - ``changed``: ``None`` +``changed``: ``None`` This option affects uploads and simply means that those objects which already exist in the object store will not be overwritten if the ``mtime`` and size of the source is the same as the existing object. - ``skip_identical``: ``False`` +``skip_identical``: ``False`` A slightly more thorough case of the above, but rather than ``mtime`` and size uses an object's ``MD5 sum``. - ``yes_all``: ``False`` +``yes_all``: ``False`` This options affects only download and delete, and in each case must be specified in order to download/delete the entire contents of an account. This option has no effect on any other calls. - ``no_download``: ``False`` +``no_download``: ``False`` This option only affects download and means that all operations proceed as normal with the exception that no data is written to disk. - ``header``: ``[]`` +``header``: ``[]`` Used with upload and post operations to set headers on objects. Headers are specified as colon separated strings, e.g. "content-type:text/plain". - ``meta``: ``[]`` +``meta``: ``[]`` Used to set metadata on an object similarly to headers. .. note:: Setting metadata is a destructive operation, so when updating one of many metadata values all desired metadata for an object must be re-applied. - ``long``: ``False`` +``long``: ``False`` Affects only list operations, and results in more metrics being made available in the results at the expense of lower performance. - ``fail_fast``: ``False`` +``fail_fast``: ``False`` Applies to delete and upload operations, and attempts to abort queued tasks in the event of errors. - ``prefix``: ``None`` +``prefix``: ``None`` Affects list operations; only objects with the given prefix will be returned/affected. It is not advisable to set at the service level, as those operations that call list to discover objects on which they should operate will also be affected. - ``delimiter``: ``None`` +``delimiter``: ``None`` Affects list operations, and means that listings only contain results up to the first instance of the delimiter in the object name. This is useful for working with objects containing '/' in their names to simulate folder structures. - ``dir_marker``: ``False`` +``dir_marker``: ``False`` Affects uploads, and allows empty 'pseudofolder' objects to be created when the source of an upload is ``None``. - ``checksum``: ``True`` +``checksum``: ``True`` Affects uploads and downloads. If set check md5 sum for the transfer. - ``shuffle``: ``False`` +``shuffle``: ``False`` When downloading objects, the default behaviour of the CLI is to shuffle lists of objects in order to spread the load on storage drives when multiple clients are downloading the same files to multiple locations (e.g. in the @@ -220,12 +220,12 @@ Options are downloaded in lexically-sorted order. Setting this option to ``True`` gives the same shuffling behaviour as the CLI. - ``destination``: ``None`` +``destination``: ``None`` When copying objects, this specifies the destination where the object will be copied to. The default of None means copy will be the same as source. - ``fresh_metadata``: ``None`` +``fresh_metadata``: ``None`` When copying objects, this specifies that the object metadata on the source will *not* be applied to the destination object - the destination object will have a new fresh set of metadata that includes diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..ae61948 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,45 @@ +alabaster==0.7.10 +Babel==2.3.4 +certifi==2018.1.18 +chardet==3.0.4 +coverage==4.0 +docutils==0.11 +dulwich==0.15.0 +extras==1.0.0 +fixtures==3.0.0 +flake8==2.2.4 +futures==3.0.0 +hacking==0.10.0 +idna==2.6 +imagesize==0.7.1 +iso8601==0.1.8 +Jinja2==2.10 +keystoneauth1==3.4.0 +linecache2==1.0.0 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==1.2.0 +netaddr==0.7.10 +openstackdocstheme==1.20.0 +oslo.config==1.2.0 +pbr==2.0.0 +pep8==1.5.7 +PrettyTable==0.7 +pyflakes==0.8.1 +Pygments==2.2.0 +python-keystoneclient==0.7.0 +python-mimeparse==1.6.0 +python-subunit==1.0.0 +pytz==2013.6 +PyYAML==3.12 +reno==2.5.0 +requests==1.1.0 +six==1.9.0 +snowballstemmer==1.2.1 +sphinx==1.6.2 +sphinxcontrib-websupport==1.0.1 +stestr==2.0.0 +testtools==2.2.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 diff --git a/releasenotes/notes/360_notes-1ec385df13a3a735.yaml b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml new file mode 100644 index 0000000..8d82b06 --- /dev/null +++ b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml @@ -0,0 +1,40 @@ +--- +features: + - | + Add the ``--prompt`` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + ``--key`` , ``--os-password`` or an environment variable will be ignored. + + - | + Added bash completion support to the ``swift`` CLI. Enable this by sourcing + the included ``tools/swift.bash_completion`` file. Make it permanent by + including this file in the system's ``/etc/bash_completion.d`` directory. + + - | + Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported are Swift 2.19.0 or later. + + - | + The client.py SDK now supports a ``query_string`` option on the + ``head_object()`` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + + - | + The client.py SDK now respects ``region_name`` when using sessions. + + - | + Added a ``.close()`` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + + - | + Fixed a bug where using ``--debug`` in the CLI with unicode account names + would cause a client crash. + + - | + Make OS_AUTH_URL work in DevStack (for testing) by default. + + - | + Dropped Python 3.4 testing. + + - | + Various other minor bug fixes and improvements. diff --git a/releasenotes/notes/361_notes-59e020e68bcdd709.yaml b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml new file mode 100644 index 0000000..f6b4892 --- /dev/null +++ b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Added the delimiter keyword parameter to ``get_account()`` to match the + functionality of ``get_container()``. + + - | + Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + + - | + Various other minor bug fixes and improvements. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index b27aa96..c71f41d 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -65,15 +65,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Swift Client Release Notes' copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year -# Release notes are version independent. -# The short X.Y version. -version = '' -# The full version, including alpha/beta/rc tags. -release = '' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -173,11 +166,6 @@ html_theme = 'openstackdocs' # # html_extra_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' -html_last_updated_fmt = '%Y-%m-%d %H:%M' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index a5240ea..27f675e 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,8 @@ :maxdepth: 1 current + stein + rocky queens pike ocata diff --git a/releasenotes/source/rocky.rst b/releasenotes/source/rocky.rst new file mode 100644 index 0000000..40dd517 --- /dev/null +++ b/releasenotes/source/rocky.rst @@ -0,0 +1,6 @@ +=================================== + Rocky Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/rocky diff --git a/releasenotes/source/stein.rst b/releasenotes/source/stein.rst new file mode 100644 index 0000000..efaceb6 --- /dev/null +++ b/releasenotes/source/stein.rst @@ -0,0 +1,6 @@ +=================================== + Stein Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/stein diff --git a/requirements.txt b/requirements.txt index 6d31e09..1c2ce33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD -requests>=1.1 -six>=1.5.2 +futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +requests>=1.1.0 +six>=1.9.0 @@ -4,7 +4,7 @@ summary = OpenStack Object Storage API Client Library description-file = README.rst author = OpenStack -author-email = openstack-dev@lists.openstack.org +author-email = openstack-discuss@lists.openstack.org home-page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack @@ -17,8 +17,9 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 [global] setup-hooks = diff --git a/swiftclient/client.py b/swiftclient/client.py index 7db75f0..f071182 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -17,6 +17,7 @@ OpenStack Swift client library used internally """ import socket +import re import requests import logging import warnings @@ -38,6 +39,7 @@ from swiftclient.utils import ( # Default is 100, increase to 256 http_client._MAXHEADERS = 256 +VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -58,6 +60,19 @@ except ImportError: def createLock(self): self.lock = None +ksexceptions = ksclient_v2 = ksclient_v3 = None +try: + from keystoneclient import exceptions as ksexceptions + # prevent keystoneclient warning us that it has no log handlers + logging.getLogger('keystoneclient').addHandler(NullHandler()) + from keystoneclient.v2_0 import client as ksclient_v2 +except ImportError: + pass +try: + from keystoneclient.v3 import client as ksclient_v3 +except ImportError: + pass + # requests version 1.2.3 try to encode headers in ascii, preventing # utf-8 encoded header to be 'prepared' if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): @@ -149,7 +164,7 @@ def http_log(args, kwargs, resp, body): elif element in ('GET', 'POST', 'PUT'): string_parts.append(' -X %s' % element) else: - string_parts.append(' %s' % element) + string_parts.append(' %s' % parse_header_string(element)) if 'headers' in kwargs: headers = scrub_headers(kwargs['headers']) for element in headers: @@ -233,8 +248,8 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) - and header.startswith(USER_METADATA_TYPE)): + if (isinstance(header, six.string_types) and + header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) ret[header] = value @@ -271,6 +286,9 @@ class _ObjectBody(object): def __next__(self): return self.next() + def close(self): + self.resp.close() + class _RetryBody(_ObjectBody): """ @@ -302,7 +320,7 @@ class _RetryBody(_ObjectBody): self.obj = obj self.query_string = query_string self.response_dict = response_dict - self.headers = headers if headers is not None else {} + self.headers = dict(headers) if headers is not None else {} self.bytes_read = 0 def read(self, length=None): @@ -384,6 +402,7 @@ class HTTPConnection(object): self.request_session = requests.Session() # Don't use requests's default headers self.request_session.headers = None + self.resp = None if self.parsed_url.scheme not in ('http', 'https'): raise ClientException('Unsupported scheme "%s" in url "%s"' % (self.parsed_url.scheme, url)) @@ -453,11 +472,23 @@ class HTTPConnection(object): self.resp.status = self.resp.status_code old_getheader = self.resp.raw.getheader + def _decode_header(string): + if string is None or six.PY2: + return string + return string.encode('iso-8859-1').decode('utf-8') + + def _encode_header(string): + if string is None or six.PY2: + return string + return string.encode('utf-8').decode('iso-8859-1') + def getheaders(): - return self.resp.headers.items() + return [(_decode_header(k), _decode_header(v)) + for k, v in self.resp.headers.items()] def getheader(k, v=None): - return old_getheader(k.lower(), v) + return _decode_header(old_getheader( + _encode_header(k.lower()), _encode_header(v))) def releasing_read(*args, **kwargs): chunk = self.resp.raw.read(*args, **kwargs) @@ -476,6 +507,11 @@ class HTTPConnection(object): return self.resp + def close(self): + if self.resp: + self.resp.close() + self.request_session.close() + def http_connection(*arg, **kwarg): """:returns: tuple of (parsed url, connection object)""" @@ -497,6 +533,8 @@ def get_auth_1_0(url, user, key, snet, **kwargs): conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() + resp.close() + conn.close() http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') @@ -511,8 +549,9 @@ def get_auth_1_0(url, user, key, snet, **kwargs): netloc = parsed[1] parsed[1] = 'snet-' + netloc url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) + + token = resp.getheader('x-storage-token', resp.getheader('x-auth-token')) + return url, token def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): @@ -522,25 +561,6 @@ def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): return get_auth_keystone(auth_url, user, key, os_options, **kwargs) -def _import_keystone_client(auth_version): - # the attempted imports are encapsulated in this function to allow - # mocking for tests - try: - if auth_version in AUTH_VERSIONS_V3: - from keystoneclient.v3 import client as ksclient - else: - from keystoneclient.v2_0 import client as ksclient - from keystoneclient import exceptions - # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) - return ksclient, exceptions - except ImportError: - raise ClientException(''' -Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth -version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment -variables to be set or overridden with -A, -U, or -K.''') - - def get_auth_keystone(auth_url, user, key, os_options, **kwargs): """ Authenticate against a keystone server. @@ -555,7 +575,10 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Add the version suffix in case of versionless Keystone endpoints. If # auth_version is also unset it is likely that it is v3 - if len(urlparse(auth_url).path) <= 1: + if not VERSIONFUL_AUTH_PATH.match( + urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]): + # Normalize auth_url to end in a slash because urljoin + auth_url = auth_url.rstrip('/') + '/' if auth_version and auth_version in AUTH_VERSIONS_V2: auth_url = urljoin(auth_url, "v2.0") else: @@ -565,8 +588,21 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Legacy default if not set if auth_version is None: - auth_version = 'v2.0' - ksclient, exceptions = _import_keystone_client(auth_version) + auth_version = '2' + + ksclient = None + if auth_version in AUTH_VERSIONS_V3: + if ksclient_v3 is not None: + ksclient = ksclient_v3 + else: + if ksclient_v2 is not None: + ksclient = ksclient_v2 + + if ksclient is None: + raise ClientException(''' +Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth +version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment +variables to be set or overridden with -A, -U, or -K.''') try: _ksclient = ksclient.Client( @@ -587,13 +623,13 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): cert=kwargs.get('cert'), key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) - except exceptions.Unauthorized: + except ksexceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' if auth_version in AUTH_VERSIONS_V3: msg = ('Unauthorized. Check username/id, password, ' 'tenant name/id and user/tenant domain name/id.') raise ClientException(msg) - except exceptions.AuthorizationFailure as err: + except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) service_type = os_options.get('service_type') or 'object-store' endpoint_type = os_options.get('endpoint_type') or 'publicURL' @@ -606,7 +642,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): service_type=service_type, endpoint_type=endpoint_type, **filter_kwargs) - except exceptions.EndpointNotFound: + except ksexceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) return endpoint, _ksclient.auth_token @@ -644,8 +680,10 @@ def get_auth(auth_url, user, key, **kwargs): if session: service_type = os_options.get('service_type', 'object-store') interface = os_options.get('endpoint_type', 'public') + region_name = os_options.get('region_name') storage_url = session.get_endpoint(service_type=service_type, - interface=interface) + interface=interface, + region_name=region_name) token = session.get_token() elif auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -668,9 +706,9 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') - or os_options.get('project_name') - or os_options.get('project_id')): + if not (os_options.get('tenant_name') or os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') @@ -719,7 +757,7 @@ def store_response(resp, response_dict): def get_account(url, token, marker=None, limit=None, prefix=None, end_marker=None, http_conn=None, full_listing=False, - service_token=None, headers=None): + service_token=None, headers=None, delimiter=None): """ Get a listing of containers for the account. @@ -735,6 +773,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param delimiter: delimiter query :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -748,14 +787,14 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if not http_conn: http_conn = http_connection(url) if full_listing: - rv = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, headers=req_headers) + rv = get_account(url, token, marker, limit, prefix, end_marker, + http_conn, headers=req_headers, delimiter=delimiter) listing = rv[1] while listing: marker = listing[-1]['name'] listing = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, - headers=req_headers)[1] + end_marker, http_conn, headers=req_headers, + delimiter=delimiter)[1] if listing: rv[1].extend(listing) return rv @@ -767,6 +806,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) @@ -847,13 +888,15 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, path = parsed.path if query_string: path += '?' + query_string - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request(method, path, data, headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request(method, path, data, req_headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) + http_log((url, method,), {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -895,12 +938,6 @@ def get_container(url, token, container, marker=None, limit=None, """ if not http_conn: http_conn = http_connection(url) - if headers: - headers = dict(headers) - else: - headers = {} - headers['X-Auth-Token'] = token - headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -935,17 +972,20 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&path=%s' % quote(path) if query_string: qs += '&%s' % query_string.lstrip('?') + req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) method = 'GET' - conn.request(method, '%s?%s' % (cont_path, qs), '', headers) + conn.request(method, '%s?%s' % (cont_path, qs), '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%(url)s%(cont_path)s?%(qs)s' % {'url': url.replace(parsed.path, ''), 'cont_path': cont_path, 'qs': qs}, method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container GET failed', body) @@ -1018,23 +1058,23 @@ def put_container(url, token, container, headers=None, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'PUT' - if not headers: - headers = {} - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + if 'content-length' not in (k.lower() for k in req_headers): + req_headers['Content-Length'] = '0' if query_string: path += '?' + query_string.lstrip('?') - conn.request(method, path, '', headers) + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() store_response(resp, response_dict) http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container PUT failed', body) @@ -1061,16 +1101,18 @@ def post_container(url, token, container, headers, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' - conn.request(method, path, '', headers) + req_headers['Content-Length'] = '0' + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1188,7 +1230,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None, headers=None): + service_token=None, headers=None, query_string=None): """ Get object info @@ -1209,6 +1251,8 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if query_string: + path += '?' + query_string if headers: headers = dict(headers) else: @@ -1365,14 +1409,16 @@ def post_object(url, token, container, name, headers, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request('POST', path, '', headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request('POST', path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1540,7 +1586,7 @@ class Connection(object): os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, retry_on_ratelimit=False, - timeout=None, session=None): + timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1576,6 +1622,8 @@ class Connection(object): after a backoff. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. + :param force_auth_retry: reset auth info even if client got unexpected + error except 401 Unauthorized. """ self.session = session self.authurl = authurl @@ -1608,16 +1656,14 @@ class Connection(object): self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit self.timeout = timeout + self.force_auth_retry = force_auth_retry def close(self): - if (self.http_conn and isinstance(self.http_conn, tuple) - and len(self.http_conn) > 1): + if (self.http_conn and isinstance(self.http_conn, tuple) and + len(self.http_conn) > 1): conn = self.http_conn[1] - if hasattr(conn, 'close') and callable(conn.close): - # XXX: Our HTTPConnection object has no close, should be - # trying to close the requests.Session here? - conn.close() - self.http_conn = None + conn.close() + self.http_conn = None def get_auth(self): self.url, self.token = get_auth(self.authurl, self.user, self.key, @@ -1677,10 +1723,10 @@ class Connection(object): try: if not self.url or not self.token: self.url, self.token = self.get_auth() - self.http_conn = None + self.close() if self.service_auth and not self.service_token: self.url, self.service_token = self.get_service_auth() - self.http_conn = None + self.close() self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() @@ -1722,6 +1768,10 @@ class Connection(object): pass else: raise + + if self.force_auth_retry: + self.url = self.token = self.service_token = None + sleep(backoff) backoff = min(backoff * 2, self.max_backoff) if reset_func: @@ -1732,14 +1782,16 @@ class Connection(object): return self._retry(None, head_account, headers=headers) def get_account(self, marker=None, limit=None, prefix=None, - end_marker=None, full_listing=False, headers=None): + end_marker=None, full_listing=False, headers=None, + delimiter=None): """Wrapper for :func:`get_account`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_account, marker=marker, limit=limit, prefix=prefix, end_marker=end_marker, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + delimiter=delimiter) def post_account(self, headers, response_dict=None, query_string=None, data=None): @@ -1785,9 +1837,10 @@ class Connection(object): query_string=query_string, headers=headers) - def head_object(self, container, obj, headers=None): + def head_object(self, container, obj, headers=None, query_string=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj, headers=headers) + return self._retry(None, head_object, container, obj, headers=headers, + query_string=query_string) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): @@ -1832,7 +1885,9 @@ class Connection(object): reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) + + def reset_func(*a, **kw): + seek(orig_pos) elif reset: reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, @@ -1865,8 +1920,7 @@ class Connection(object): url = url or self.url if not url: url, _ = self.get_auth() - scheme = urlparse(url).scheme - netloc = urlparse(url).netloc - url = scheme + '://' + netloc + '/info' - http_conn = self.http_connection(url) - return get_capabilities(http_conn) + parsed = urlparse(urljoin(url, '/info')) + if not self.http_conn: + self.http_conn = self.http_connection(url) + return get_capabilities((parsed, self.http_conn[1])) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 5e03ed7..fcf0ed9 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -175,6 +175,14 @@ class ConnectionThreadPoolExecutor(ThreadPoolExecutor): super(ConnectionThreadPoolExecutor, self).__init__(max_workers) def submit(self, fn, *args, **kwargs): + """ + Schedules the callable, `fn`, to be executed + + :param fn: the callable to be invoked + :param args: the positional arguments for the callable + :param kwargs: the keyword arguments for the callable + :returns: a Future object representing the execution of the callable + """ def conn_fn(): priority = None conn = None diff --git a/swiftclient/service.py b/swiftclient/service.py index ed5e9e9..06de091 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -17,6 +17,7 @@ import logging import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,7 +45,7 @@ from swiftclient.command_helpers import ( from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -144,6 +145,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), "os_user_domain_name": environ.get('OS_USER_DOMAIN_NAME'), @@ -172,6 +174,7 @@ def _build_default_global_options(): 'container_threads': 10 } + _default_global_options = _build_default_global_options() _default_local_options = { @@ -261,7 +264,8 @@ def get_conn(options): insecure=options['insecure'], cert=options['os_cert'], cert_key=options['os_key'], - ssl_compression=options['ssl_compression']) + ssl_compression=options['ssl_compression'], + force_auth_retry=options['force_auth_retry']) def mkdirs(path): @@ -422,8 +426,8 @@ class _SwiftReader(object): '{1} != {2}'.format( self._path, etag, self._expected_md5)) - if (self._content_length is not None - and self._actual_read != self._content_length): + if (self._content_length is not None and + self._actual_read != self._content_length): raise SwiftError('Error downloading {0}: read_length != ' 'content_length, {1:d} != {2:d}'.format( self._path, self._actual_read, @@ -449,7 +453,9 @@ class SwiftService(object): **_default_local_options ) process_options(self._options) - create_connection = lambda: get_conn(self._options) + + def create_connection(): + return get_conn(self._options) self.thread_manager = MultiThreadingManager( create_connection, segment_threads=self._options['segment_threads'], @@ -1242,8 +1248,8 @@ class SwiftService(object): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if ('x-object-meta-mtime' in headers and not no_file - and not options['ignore_mtime']): + if ('x-object-meta-mtime' in headers and not no_file and + not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: @@ -2034,8 +2040,8 @@ class SwiftService(object): new_slo_manifest_paths = set() segment_size = int(0 if options['segment_size'] is None else options['segment_size']) - if (options['changed'] or options['skip_identical'] - or not options['leave_segments']): + if (options['changed'] or options['skip_identical'] or + not options['leave_segments']): try: headers = conn.head_object(container, obj) is_slo = config_true_value( @@ -2056,9 +2062,9 @@ class SwiftService(object): cl = int(headers.get('content-length')) mt = headers.get('x-object-meta-mtime') - if (path is not None and options['changed'] - and cl == getsize(path) - and mt == put_headers['x-object-meta-mtime']): + if (path is not None and options['changed'] and + cl == getsize(path) and + mt == put_headers['x-object-meta-mtime']): res.update({ 'success': True, 'status': 'skipped-changed' @@ -2067,11 +2073,9 @@ class SwiftService(object): if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -2093,8 +2097,8 @@ class SwiftService(object): # a segment job if we're reading from a stream - we may fail if we # go over the single object limit, but this gives us a nice way # to create objects from memory - if (path is not None and segment_size - and (getsize(path) > segment_size)): + if (path is not None and segment_size and + (getsize(path) > segment_size)): res['large_object'] = True seg_container = container + '_segments' if options['segment_container']: @@ -2161,8 +2165,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, segment_results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - seg['segment_location'] for seg in segment_results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2219,8 +2224,9 @@ class SwiftService(object): response = self._upload_slo_manifest( conn, results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - r['segment_location'] for r in results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) res['large_object'] = True else: res['response_dict'] = ret @@ -2260,11 +2266,10 @@ class SwiftService(object): fp.close() if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2276,10 +2281,8 @@ class SwiftService(object): if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): @@ -2423,8 +2426,8 @@ class SwiftService(object): # Cancel the remaining container deletes, but yield # any pending results - if (not cancelled and options['fail_fast'] - and not res['success']): + if (not cancelled and options['fail_fast'] and + not res['success']): cancelled = True def _bulk_delete_page_size(self, objects): @@ -2471,17 +2474,18 @@ class SwiftService(object): def _delete_segment(conn, container, obj, results_queue=None): results_dict = {} try: - conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} + conn.delete_object(container, obj, response_dict=results_dict) except Exception as err: - traceback, err_time = report_traceback() - logger.exception(err) - res = { - 'success': False, - 'error': err, - 'traceback': traceback, - 'error_timestamp': err_time - } + if not isinstance(err, ClientException) or err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 6ccc16d..a322a1c 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,11 +17,13 @@ from __future__ import print_function, unicode_literals import argparse +import getpass import io import json import logging import signal import socket +import warnings from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join @@ -49,13 +51,14 @@ except ImportError: BASENAME = 'swift' commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', - 'capabilities', 'info', 'tempurl', 'auth') + 'capabilities', 'info', 'tempurl', 'auth', 'bash_completion') def immediate_exit(signum, frame): stderr.write(" Aborted\n") os_exit(2) + st_delete_options = '''[--all] [--leave-segments] [--object-threads <threads>] [--container-threads <threads>] @@ -88,7 +91,7 @@ Optional arguments: '''.strip("\n") -def st_delete(parser, args, output_manager): +def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') @@ -112,6 +115,11 @@ def st_delete(parser, args, output_manager): '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' 'Its value must be a positive integer. Default is 10.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options['yes_all']) or (args and options['yes_all']): @@ -279,7 +287,7 @@ Optional arguments: '''.strip("\n") -def st_download(parser, args, output_manager): +def st_download(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' @@ -342,6 +350,11 @@ def st_download(parser, args, output_manager): 'to store the access and modified timestamp for the downloaded file. ' 'With this option, the header is ignored and the timestamps are ' 'created freshly.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': @@ -492,7 +505,7 @@ Optional arguments: '''.strip('\n') -def st_list(parser, args, output_manager): +def st_list(parser, args, output_manager, return_parser=False): def _print_stats(options, stats, human): total_count = total_bytes = 0 @@ -569,6 +582,11 @@ def st_list(parser, args, output_manager): '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for listing.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if options['delimiter'] and not args: @@ -627,7 +645,7 @@ Optional arguments: '''.strip('\n') -def st_stat(parser, args, output_manager): +def st_stat(parser, args, output_manager, return_parser=False): parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') @@ -636,6 +654,10 @@ def st_stat(parser, args, output_manager): default=[], help='Adds a custom request header to use for stat.') + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] @@ -723,7 +745,7 @@ Optional arguments: '''.strip('\n') -def st_post(parser, args, output_manager): +def st_post(parser, args, output_manager, return_parser=False): parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' @@ -748,6 +770,11 @@ def st_post(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (options['read_acl'] or options['write_acl'] or options['sync_to'] or @@ -820,7 +847,7 @@ Optional arguments: '''.strip('\n') -def st_copy(parser, args, output_manager): +def st_copy(parser, args, output_manager, return_parser=False): parser.add_argument( '-d', '--destination', help='The container and name of the ' 'destination object') @@ -837,6 +864,11 @@ def st_copy(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] @@ -946,7 +978,7 @@ Optional arguments: '''.strip('\n') -def st_upload(parser, args, output_manager): +def st_upload(parser, args, output_manager, return_parser=False): DEFAULT_STDIN_SEGMENT = 10 * 1024 * 1024 parser.add_argument( @@ -1004,6 +1036,11 @@ def st_upload(parser, args, output_manager): parser.add_argument( '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -1183,7 +1220,7 @@ Optional arguments: st_info_help = st_capabilities_help -def st_capabilities(parser, args, output_manager): +def st_capabilities(parser, args, output_manager, return_parser=False): def _print_compo_cap(name, capabilities): for feature, options in sorted(capabilities.items(), key=lambda x: x[0]): @@ -1196,6 +1233,11 @@ def st_capabilities(parser, args, output_manager): parser.add_argument('--json', action='store_true', help='print capability information in json') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1244,7 +1286,12 @@ Display auth related authentication variables in shell friendly format. '''.strip('\n') -def st_auth(parser, args, thread_manager): +def st_auth(parser, args, thread_manager, return_parser=False): + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if options['verbose'] > 1: if options['auth_version'] in ('1', '1.0'): @@ -1323,10 +1370,12 @@ Optional arguments: generated. --iso8601 If present, the generated temporary URL will contain an ISO 8601 UTC timestamp instead of a Unix timestamp. + --ip-range If present, the temporary URL will be restricted to the + given ip or ip range. '''.strip('\n') -def st_tempurl(parser, args, thread_manager): +def st_tempurl(parser, args, thread_manager, return_parser=False): parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, @@ -1346,6 +1395,16 @@ def st_tempurl(parser, args, thread_manager): help=("If present, the temporary URL will contain an ISO 8601 UTC " "timestamp instead of a Unix timestamp."), ) + parser.add_argument( + '--ip-range', action='store', + default=None, + help=("If present, the temporary URL will be restricted to the " + "given ip or ip range."), + ) + + # We return the parser to build up the bash_completion + if return_parser: + return parser (options, args) = parse_args(parser, args) args = args[1:] @@ -1365,7 +1424,8 @@ def st_tempurl(parser, args, thread_manager): path = generate_temp_url(parsed.path, timestamp, key, method, absolute=options['absolute_expiry'], iso8601=options['iso8601'], - prefix=options['prefix_based']) + prefix=options['prefix_based'], + ip_range=options['ip_range']) except ValueError as err: thread_manager.error(err) return @@ -1377,6 +1437,66 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg(url) +st_bash_completion_help = '''Retrieve command specific flags used by bash_completion. + +Optional positional arguments: + <command> Swift client command to filter the flags by. +'''.strip('\n') + + +st_bash_completion_options = '''[command] +''' + + +def st_bash_completion(parser, args, thread_manager, return_parser=False): + if return_parser: + return parser + + global commands + com = args[1] if len(args) > 1 else None + + if com: + if com in commands: + fn_commands = ["st_%s" % com] + else: + print("") + return + else: + fn_commands = [fn for fn in globals().keys() + if fn.startswith('st_') and + not fn.endswith('_options') and + not fn.endswith('_help')] + + subparsers = parser.add_subparsers() + subcommands = {} + if not com: + subcommands['base'] = parser + for command in fn_commands: + cmd = command[3:] + if com: + subparser = subparsers.add_parser( + cmd, help=globals()['%s_help' % command]) + add_default_args(subparser) + subparser = globals()[command]( + subparser, args, thread_manager, True) + subcommands[cmd] = subparser + else: + subcommands[cmd] = None + + cmds = set() + opts = set() + for sc_str, sc in list(subcommands.items()): + cmds.add(sc_str) + if sc: + for option in sc._optionals._option_string_actions: + opts.add(option) + + for cmd_to_remove in (com, 'bash_completion', 'base'): + if cmd_to_remove in cmds: + cmds.remove(cmd_to_remove) + print(' '.join(cmds | opts)) + + class HelpFormatter(argparse.HelpFormatter): def _format_action_invocation(self, action): if not action.option_strings: @@ -1410,6 +1530,30 @@ class HelpFormatter(argparse.HelpFormatter): return action.dest +def prompt_for_password(): + """ + Prompt the user for a password. + + :raise SystemExit: if a password cannot be entered without it being echoed + to the terminal. + :return: the entered password. + """ + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=getpass.GetPassWarning, + append=True) + try: + # temporarily set signal handling back to default to avoid user + # Ctrl-c leaving terminal in weird state + signal.signal(signal.SIGINT, signal.SIG_DFL) + return getpass.getpass() + except EOFError: + return None + except getpass.GetPassWarning: + exit('Input stream incompatible with --prompt option') + finally: + signal.signal(signal.SIGINT, immediate_exit) + + def parse_args(parser, args, enforce_requires=True): options, args = parser.parse_known_args(args or ['-h']) options = vars(options) @@ -1435,6 +1579,10 @@ def parse_args(parser, args, enforce_requires=True): if args and args[0] == 'tempurl': return options, args + # do this before process_options sets default auth version + if enforce_requires and options['prompt']: + options['key'] = options['os_password'] = prompt_for_password() + # Massage auth version; build out os_options subdict process_options(options) @@ -1469,88 +1617,7 @@ adding "-V 2" is necessary for this.'''.strip('\n')) return options, args -def main(arguments=None): - argv = sys_argv if arguments is None else arguments - - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] - - version = client_version - parser = argparse.ArgumentParser( - add_help=False, formatter_class=HelpFormatter, usage=''' -%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] - [--debug] [--info] [--quiet] [--auth <auth_url>] - [--auth-version <auth_version> | - --os-identity-api-version <auth_version> ] - [--user <username>] - [--key <api_key>] [--retries <num_retries>] - [--os-username <auth-user-name>] [--os-password <auth-password>] - [--os-user-id <auth-user-id>] - [--os-user-domain-id <auth-user-domain-id>] - [--os-user-domain-name <auth-user-domain-name>] - [--os-tenant-id <auth-tenant-id>] - [--os-tenant-name <auth-tenant-name>] - [--os-project-id <auth-project-id>] - [--os-project-name <auth-project-name>] - [--os-project-domain-id <auth-project-domain-id>] - [--os-project-domain-name <auth-project-domain-name>] - [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] - [--os-storage-url <storage-url>] [--os-region-name <region-name>] - [--os-service-type <service-type>] - [--os-endpoint-type <endpoint-type>] - [--os-cacert <ca-certificate>] [--insecure] - [--os-cert <client-certificate-file>] - [--os-key <client-certificate-key-file>] - [--no-ssl-compression] - <subcommand> [--help] [<subcommand options>] - -Command-line interface to the OpenStack Swift API. - -Positional arguments: - <subcommand> - delete Delete a container or objects within a container. - download Download objects from containers. - list Lists the containers for the account or the objects - for a container. - post Updates meta information for the account, container, - or object; creates containers if not present. - copy Copies object, optionally adds meta - stat Displays information for the account, container, - or object. - upload Uploads files or directories to the given container. - capabilities List cluster capabilities. - tempurl Create a temporary URL. - auth Display auth related environment variables. - -Examples: - %(prog)s download --help - - %(prog)s -A https://api.example.com/v1.0 \\ - -U user -K api_key stat -v - - %(prog)s --os-auth-url https://api.example.com/v2.0 \\ - --os-tenant-name tenant \\ - --os-username user --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-name project1 --os-project-domain-name domain1 \\ - --os-username user --os-user-domain-name domain1 \\ - --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-id 0123456789abcdef0123456789abcdef \\ - --os-user-id abcdef0123456789abcdef0123456789 \\ - --os-password password list - - %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ - list - - %(prog)s list --lh -'''.strip('\n')) - parser.add_argument('--version', action='version', - version='python-swiftclient %s' % version) - parser.add_argument('-h', '--help', action='store_true') - +def add_default_args(parser): default_auth_version = '1.0' for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): try: @@ -1610,6 +1677,17 @@ Examples: help='This option is deprecated and not used anymore. ' 'SSL compression should be disabled by default ' 'by the system SSL library.') + parser.add_argument('--force-auth-retry', + action='store_true', dest='force_auth_retry', + default=False, + help='Force a re-auth attempt on ' + 'any error other than 401 unauthorized') + parser.add_argument('--prompt', + action='store_true', dest='prompt', + default=False, + help='Prompt user to enter a password which overrides ' + 'any password supplied via --key, --os-password ' + 'or environment variables.') os_grp = parser.add_argument_group("OpenStack authentication options") os_grp.add_argument('--os-username', @@ -1752,6 +1830,100 @@ Examples: default=environ.get('OS_KEY'), help='Specify a client certificate key file (for ' 'client auth). Defaults to env[OS_KEY].') + + +def main(arguments=None): + argv = sys_argv if arguments is None else arguments + + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] + [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] + [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] + [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] + [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + [--force-auth-retry] + <subcommand> [--help] [<subcommand options>] + +Command-line interface to the OpenStack Swift API. + +Positional arguments: + <subcommand> + delete Delete a container or objects within a container. + download Download objects from containers. + list Lists the containers for the account or the objects + for a container. + post Updates meta information for the account, container, + or object; creates containers if not present. + copy Copies object, optionally adds meta + stat Displays information for the account, container, + or object. + upload Uploads files or directories to the given container. + capabilities List cluster capabilities. + tempurl Create a temporary URL. + auth Display auth related environment variables. + bash_completion Outputs option and flag cli data ready for + bash_completion. + +Examples: + %(prog)s download --help + + %(prog)s -A https://api.example.com/v1.0 \\ + -U user -K api_key stat -v + + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ + --os-username user --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-name project1 --os-project-domain-name domain1 \\ + --os-username user --os-user-domain-name domain1 \\ + --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-id 0123456789abcdef0123456789abcdef \\ + --os-user-id abcdef0123456789abcdef0123456789 \\ + --os-password password list + + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ + list + + %(prog)s list --lh +'''.strip('\n')) + + version = client_version + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') + + add_default_args(parser) + options, args = parse_args(parser, argv[1:], enforce_requires=False) if options['help'] or options['os_help']: @@ -1772,9 +1944,14 @@ Examples: parser.usage = globals()['st_%s_help' % args[0]] if options['insecure']: import requests - from requests.packages.urllib3.exceptions import \ - InsecureRequestWarning - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + try: + from requests.packages.urllib3.exceptions import \ + InsecureRequestWarning + except ImportError: + pass + else: + requests.packages.urllib3.disable_warnings( + InsecureRequestWarning) try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8afcde9..2b208b9 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -69,12 +69,12 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False): + prefix=False, iso8601=False, ip_range=None): """Generates a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object or prefix if - a prefix-based temporary URL should be generated. Example: + a prefix-based temporary URL should be generated. Example: /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. :param seconds: time in seconds or ISO 8601 timestamp. If absolute is False and this is the string representation of an @@ -92,6 +92,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False, :param prefix: if True then a prefix-based temporary URL will be generated. :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp instead of a UNIX timestamp will be created. + :param ip_range: if a valid ip range, restricts the temporary URL to the + range of ips. :raises ValueError: if timestamp or path is not in valid format. :return: the path portion of a temporary URL """ @@ -155,8 +157,21 @@ def generate_temp_url(path, seconds, key, method, absolute=False, expiration = int(time.time() + timestamp) else: expiration = timestamp - hmac_body = u'\n'.join([method.upper(), str(expiration), - ('prefix:' if prefix else '') + path_for_body]) + + hmac_parts = [method.upper(), str(expiration), + ('prefix:' if prefix else '') + path_for_body] + + if ip_range: + if isinstance(ip_range, six.binary_type): + try: + ip_range = ip_range.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + 'ip_range must be representable as UTF-8' + ) + hmac_parts.insert(0, "ip=%s" % ip_range) + + hmac_body = u'\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility if not isinstance(key, six.binary_type): @@ -169,6 +184,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False, temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) + + if ip_range: + temp_url += u'&temp_url_ip_range={}'.format(ip_range) + if prefix: temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller @@ -376,3 +395,11 @@ def n_at_a_time(seq, n): def n_groups(seq, n): items_per_group = ((len(seq) - 1) // n) + 1 return n_at_a_time(seq, items_per_group) + + +def normalize_manifest_path(path): + if six.PY2 and isinstance(path, six.text_type): + path = path.encode('utf-8') + if path.startswith('/'): + return path[1:] + return path diff --git a/test-requirements.txt b/test-requirements.txt index a9a0c7f..b3ca5f8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,6 @@ -hacking>=0.10.0,<0.11 +hacking>=1.1.0,<1.2.0 # Apache-2.0 -coverage>=3.6 -mock>=1.2 -oslosphinx>=4.7.0 # Apache-2.0 -sphinx>=1.1.2,<1.2 -testrepository>=0.0.18 -reno>=1.8.0,!=2.3.1 # Apache-2.0 -openstackdocstheme>=1.16.0 # Apache-2.0 +coverage!=4.4,>=4.0 # Apache-2.0 +keystoneauth1>=3.4.0 # Apache-2.0 +mock>=1.2.0 # BSD +stestr>=2.0.0 # Apache-2.0 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index d60ae06..bae3044 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -46,11 +46,34 @@ class TestFunctional(unittest.TestCase): config.read(config_file) self.config = config if config.has_section('func_test'): - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') + if config.has_option('func_test', 'auth_uri'): + self.auth_url = config.get('func_test', 'auth_uri') + try: + self.auth_version = config.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + self.auth_version = last_piece + else: + raise + else: + auth_host = config.get('func_test', 'auth_host') + auth_port = config.getint('func_test', 'auth_port') + auth_ssl = config.getboolean('func_test', 'auth_ssl') + auth_prefix = config.get('func_test', 'auth_prefix') + self.auth_version = config.get('func_test', 'auth_version') + self.auth_url = "" + if auth_ssl: + self.auth_url += "https://" + else: + self.auth_url += "http://" + self.auth_url += "%s:%s%s" % ( + auth_host, auth_port, auth_prefix) + if self.auth_version == "1": + self.auth_url += 'v1.0' + try: self.account_username = config.get('func_test', 'account_username') @@ -59,15 +82,6 @@ class TestFunctional(unittest.TestCase): username = config.get('func_test', 'username') self.account_username = "%s:%s" % (account, username) self.password = config.get('func_test', 'password') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://" - else: - self.auth_url += "http://" - self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - else: self.skip_tests = True @@ -108,6 +122,7 @@ class TestFunctional(unittest.TestCase): self.conn.delete_container(container) except swiftclient.ClientException: pass + self.conn.close() def _check_account_headers(self, headers): headers_to_check = [ @@ -153,6 +168,18 @@ class TestFunctional(unittest.TestCase): self.assertTrue(len(containers) >= 1) self.assertEqual(self.containername_2, containers[0].get('name')) + # Test prefix + _, containers = self.conn.get_account(prefix='dne') + self.assertEqual(0, len(containers)) + + # Test delimiter + _, containers = self.conn.get_account( + prefix=self.containername, delimiter='_') + self.assertEqual(2, len(containers)) + self.assertEqual(self.containername, containers[0].get('name')) + self.assertTrue( + self.containername_2.startswith(containers[1].get('subdir'))) + def _check_container_headers(self, headers): self.assertTrue(headers.get('content-length')) self.assertTrue(headers.get('x-container-object-count')) @@ -485,9 +512,6 @@ class TestUsingKeystone(TestFunctional): self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - def setUp(self): - super(TestUsingKeystone, self).setUp() - class TestUsingKeystoneV3(TestFunctional): """ @@ -514,6 +538,3 @@ class TestUsingKeystoneV3(TestFunctional): return swiftclient.Connection(self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - - def setUp(self): - super(TestUsingKeystoneV3, self).setUp() diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 110fb01..f729c25 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals -from genericpath import getmtime +import contextlib +from genericpath import getmtime +import getpass import hashlib import json import logging @@ -25,7 +27,6 @@ import unittest import textwrap from time import localtime, mktime, strftime, strptime -from requests.packages.urllib3.exceptions import InsecureRequestWarning import six import sys @@ -36,12 +37,16 @@ import swiftclient.utils from os.path import basename, dirname from .utils import ( - CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, + CaptureOutput, fake_get_auth_keystone, FakeKeystone, StubResponse, MockHttpTest) from swiftclient.utils import ( EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning +except ImportError: + InsecureRequestWarning = None if six.PY2: BUILTIN_OPEN = '__builtin__.open' @@ -112,6 +117,20 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): return args, env +@contextlib.contextmanager +def patch_disable_warnings(): + if InsecureRequestWarning is None: + # If InsecureRequestWarning isn't available, disbale_warnings won't + # be either; they both came in with + # https://github.com/requests/requests/commit/811ee4e and left again + # in https://github.com/requests/requests/commit/8e17600 + yield None + else: + with mock.patch('requests.packages.urllib3.disable_warnings') \ + as patched: + yield patched + + @mock.patch.dict(os.environ, mocked_os_environ) class TestShell(unittest.TestCase): def setUp(self): @@ -780,11 +799,11 @@ class TestShell(unittest.TestCase): response_dict={}) expected_delete_calls = [ mock.call( - b'container1', b'old_seg1', + 'container1', 'old_seg1', response_dict={} ), mock.call( - b'container2', b'old_seg2', + 'container2', 'old_seg2', response_dict={} ) ] @@ -816,6 +835,46 @@ class TestShell(unittest.TestCase): self.assertFalse(connection.return_value.delete_object.mock_calls) @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_upload_delete_dlo_segments(self, connection): # Upload delete existing segments connection.return_value.head_container.return_value = { @@ -1665,7 +1724,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -1674,7 +1733,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True) + iso8601=False, prefix=True, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -1686,7 +1745,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -1695,7 +1754,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False) + iso8601=True, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1704,7 +1763,16 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1767,6 +1835,17 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -1783,7 +1862,7 @@ class TestShell(unittest.TestCase): argv = ["", "tempurl", "GET", "60", '/v1/a/c', "secret_key", "--absolute", '--prefix-based'] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for path %r' % (expected, output.err, '/v1/a/c')) @@ -1793,7 +1872,7 @@ class TestShell(unittest.TestCase): argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', "secret_key", "--absolute"] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for time %r' % (expected, output.err, bad_time)) @@ -1957,8 +2036,8 @@ class TestBase(unittest.TestCase): self._environ_vars = {} keys = list(os.environ.keys()) for k in keys: - if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') - or k.startswith('OS_')): + if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or + k.startswith('OS_')): self._environ_vars[k] = os.environ.pop(k) def _replace_swift_env_vars(self): @@ -2283,17 +2362,66 @@ class TestParsing(TestBase): os_opts = {"password": "secret", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) os_opts = {"username": "user", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) os_opts = {"username": "user", "password": "secret"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') def test_no_tenant_name_or_id_v2(self): os_opts = {"password": "secret", @@ -2446,7 +2574,17 @@ class TestKeystoneOptions(MockHttpTest): cmd_args=cmd_args) ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' ks_token = 'fake_auth_token' + # check correct auth version gets used + key = 'auth-version' fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) + if no_auth: + fake_ks2 = fake_ks3 = None + elif opts.get(key, self.defaults.get(key)) == '2.0': + fake_ks2 = fake_ks + fake_ks3 = None + else: + fake_ks2 = None + fake_ks3 = fake_ks # fake_conn will check that storage_url and auth_token are as expected endpoint = os_opts.get('storage-url', ks_endpoint) token = os_opts.get('auth-token', ks_token) @@ -2454,12 +2592,11 @@ class TestKeystoneOptions(MockHttpTest): storage_url=endpoint, auth_token=token) - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)), \ + with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ + mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ mock.patch('swiftclient.client.http_connection', fake_conn), \ mock.patch.dict(os.environ, env, clear=True), \ - mock.patch('requests.packages.urllib3.disable_warnings') as \ - mock_disable_warnings: + patch_disable_warnings() as mock_disable_warnings: try: swiftclient.shell.main(args) except SystemExit as e: @@ -2467,23 +2604,19 @@ class TestKeystoneOptions(MockHttpTest): except SwiftError as err: self.fail('Unexpected SwiftError: %s' % err) - if 'insecure' in flags: - self.assertEqual([mock.call(InsecureRequestWarning)], - mock_disable_warnings.mock_calls) - else: - self.assertEqual([], mock_disable_warnings.mock_calls) + if InsecureRequestWarning is not None: + if 'insecure' in flags: + self.assertEqual([mock.call(InsecureRequestWarning)], + mock_disable_warnings.mock_calls) + else: + self.assertEqual([], mock_disable_warnings.mock_calls) if no_auth: - # check that keystone client was not used and terminate tests - self.assertIsNone(getattr(fake_ks, 'auth_version')) - self.assertEqual(len(fake_ks.calls), 0) + # We patched out both keystoneclient versions to be None; + # they *can't* have been used and if we tried to, we would + # have raised ClientExceptions return - # check correct auth version was passed to _import_keystone_client - key = 'auth-version' - expected = opts.get(key, self.defaults.get(key)) - self.assertEqual(expected, fake_ks.auth_version) - # check args passed to keystone Client __init__ self.assertEqual(len(fake_ks.calls), 1) actual_args = fake_ks.calls[0] @@ -2854,9 +2987,9 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): self.account = 'AUTH_alice' # keystone returns endpoint for another account - fake_ks = FakeKeystone(endpoint='http://example.com:8080/v1/AUTH_bob', - token='bob_token') - self.fake_ks_import = _make_fake_import_keystone_client(fake_ks) + self.fake_ks = FakeKeystone( + endpoint='http://example.com:8080/v1/AUTH_bob', + token='bob_token') self.cont = 'c1' self.cont_path = '/v1/%s/%s' % (self.account, self.cont) @@ -2886,12 +3019,12 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): Modify response code to 200 if cross account permissions match. """ status = 403 - if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) - and read_ok and method in ('GET', 'HEAD')): + if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and + read_ok and method in ('GET', 'HEAD')): status = 200 elif (path.startswith('/v1/%s/%s%s' - % (self.account, self.cont, self.obj)) - and write_ok and method in ('PUT', 'POST', 'DELETE')): + % (self.account, self.cont, self.obj)) and + write_ok and method in ('PUT', 'POST', 'DELETE')): status = 200 return status return on_request @@ -2935,8 +3068,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -2958,8 +3090,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): on_request=req_handler) args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -2985,8 +3116,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): '--segment-size=10', '--segment-container=%s' % self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3024,8 +3154,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): cmd_args=[self.cont, self.obj, '--leave-segments', '--segment-size=10']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3061,8 +3190,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3119,8 +3247,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3141,8 +3268,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3160,8 +3286,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3185,8 +3310,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): fake_conn = self.fake_http_connection(resp, on_request=req_handler) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3203,8 +3327,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): fake_conn = self.fake_http_connection(403) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 3de5f02..2d45deb 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -26,11 +26,13 @@ import tempfile from hashlib import md5 from six import binary_type from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, - FakeKeystone, _make_fake_import_keystone_client) + FakeKeystone) from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException from swiftclient import client as c import swiftclient.utils import swiftclient @@ -320,8 +322,7 @@ class TestGetAuth(MockHttpTest): # TestConnection.test_timeout_passed_down but is required to check that # get_auth does the right thing when it is not passed a timeout arg fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): c.get_auth('http://www.test.com', 'asdf', 'asdf', os_options=dict(tenant_name='tenant'), auth_version="2.0", timeout=42.0) @@ -578,8 +579,7 @@ class TestGetAuth(MockHttpTest): def test_get_auth_keystone_versionless(self): fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('http://authurl', 'user', 'key', {}) self.assertEqual(1, len(fake_ks.calls)) self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) @@ -587,14 +587,56 @@ class TestGetAuth(MockHttpTest): def test_get_auth_keystone_versionless_auth_version_set(self): fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): c.get_auth_keystone('http://auth_url', 'user', 'key', {}, auth_version='2.0') self.assertEqual(1, len(fake_ks.calls)) self.assertEqual('http://auth_url/v2.0', fake_ks.calls[0].get('auth_url')) + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + def test_auth_with_session(self): mock_session = mock.MagicMock() mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' @@ -662,6 +704,18 @@ class TestGetAccount(MockHttpTest): 'x-auth-token': 'asdf'}), ]) + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=-") + c.get_account('http://www.test.com/v1/acct', 'asdf', + delimiter='-') + self.assertRequests([ + ('GET', '/v1/acct?format=json&delimiter=-', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + class TestHeadAccount(MockHttpTest): @@ -700,9 +754,10 @@ class TestPostAccount(MockHttpTest): c.http_connection = self.fake_http_connection(200, headers={ 'X-Account-Meta-Color': 'blue', }, body='foo') + headers = {'x-account-meta-shape': 'square'} resp_headers, body = c.post_account( 'http://www.tests.com/path/to/account', 'asdf', - {'x-account-meta-shape': 'square'}, query_string='bar=baz', + headers, query_string='bar=baz', data='some data') self.assertEqual('blue', resp_headers.get('x-account-meta-color')) self.assertEqual('foo', body) @@ -711,6 +766,8 @@ class TestPostAccount(MockHttpTest): 'some data', {'x-auth-token': 'asdf', 'x-account-meta-shape': 'square'}) ]) + # Check that we didn't mutate the request ehader dict + self.assertEqual(headers, {'x-account-meta-shape': 'square'}) def test_server_error(self): body = 'c' * 65 @@ -1180,6 +1237,16 @@ class TestHeadObject(MockHttpTest): }), ]) + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + class TestPutObject(MockHttpTest): @@ -1454,6 +1521,11 @@ class TestPostObject(MockHttpTest): 'X-Object-Meta-Test': 'mymeta', 'X-Delete-At': delete_at}), ]) + # Check that the request header dict didn't get mutated + self.assertEqual(args[-1], { + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at, + }) def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') @@ -1829,6 +1901,57 @@ class TestHTTPConnection(MockHttpTest): self.assertFalse(resp.read()) self.assertTrue(resp.closed) + @unittest.skipIf(six.PY3, 'python2 specific test') + def test_response_python2_headers(self): + '''Test utf-8 headers in Python 2. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + '\xd8\xaa-unicode': '\xd8\xaa-value', + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), + ('empty-header', ''), + ('etag', '"%s"' % EMPTY_ETAG)]), + dict(resp.getheaders())) + + @unittest.skipIf(six.PY2, 'python3 specific test') + def test_response_python3_headers(self): + '''Test latin1-encoded headers in Python 3. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + b'\xd8\xaa-unicode'.decode('iso-8859-1'): + b'\xd8\xaa-value'.decode('iso-8859-1'), + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-unicode')) + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\u062a-unicode', '\u062a-value'), + ('empty-header', ''), + ('etag', ('"%s"' % EMPTY_ETAG))]), + dict(resp.getheaders())) + class TestConnection(MockHttpTest): @@ -1968,6 +2091,71 @@ class TestConnection(MockHttpTest): self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + def test_resp_read_on_server_error(self): conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) @@ -2273,8 +2461,7 @@ class TestConnection(MockHttpTest): 'http://auth.example.com', 'user', 'password', timeout=33.0, os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): with mock.patch.multiple('swiftclient.client', http_connection=shim_connection, sleep=mock.DEFAULT): @@ -2357,6 +2544,9 @@ class TestConnection(MockHttpTest): def read(self, *args, **kwargs): return '' + def close(self): + pass + def local_http_connection(url, proxy=None, cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, timeout=None): @@ -2459,16 +2649,17 @@ class TestConnection(MockHttpTest): def test_head_object(self): headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): with mock.patch('swiftclient.client.get_auth', lambda *a, **k: ('http://url:8080/v1/a', 'token')): conn = c.Connection() conn.head_object('c1', 'o1', - headers=headers) + headers=headers, query_string=query_string) self.assertEqual(1, len(self.request_log), self.request_log) self.assertRequests([ - ('HEAD', '/v1/a/c1/o1', '', { + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', }), @@ -2706,6 +2897,16 @@ class TestLogging(MockHttpTest): self.assertIn('X-Storage-Token', output) self.assertIn(unicode_token_value, output) + @mock.patch('swiftclient.client.logger.debug') + def test_unicode_path(self, mock_log): + path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + c.http_log(['GET', path], {}, + MockHttpResponse(status=200, headers=[]), '') + request_log_line = mock_log.mock_calls[0] + self.assertEqual('REQ: %s', request_log_line[1][0]) + self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + request_log_line[1][1]) + class TestCloseConnection(MockHttpTest): @@ -2715,6 +2916,9 @@ class TestCloseConnection(MockHttpTest): self.assertIsNone(conn.http_conn) conn.close() self.assertIsNone(conn.http_conn) + # Can re-close + conn.close() + self.assertIsNone(conn.http_conn) def test_close_ok(self): url = 'http://www.test.com' @@ -2725,7 +2929,7 @@ class TestCloseConnection(MockHttpTest): self.assertEqual(len(conn.http_conn), 2) http_conn_obj = conn.http_conn[1] self.assertIsInstance(http_conn_obj, c.HTTPConnection) - self.assertFalse(hasattr(http_conn_obj, 'close')) + self.assertTrue(hasattr(http_conn_obj, 'close')) conn.close() @@ -3016,10 +3220,11 @@ class TestServiceToken(MockHttpTest): self.assertEqual(conn.attempts, 1) def test_service_token_post_container(self): + headers = {'X-Container-Meta-Color': 'blue'} with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(201)): conn = self.get_connection() - conn.post_container('container1', {}) + conn.post_container('container1', headers) self.assertEqual(1, len(self.request_log), self.request_log) for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) @@ -3029,6 +3234,8 @@ class TestServiceToken(MockHttpTest): self.assertEqual('http://storage_url.com/container1', actual['full_path']) self.assertEqual(conn.attempts, 1) + # Check that we didn't mutate the request header dict + self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) def test_service_token_put_container(self): with mock.patch('swiftclient.client.http_connection', diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index adead00..e54b90c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -152,6 +152,54 @@ class TestTempURL(unittest.TestCase): self.assertIsInstance(url, type(self.url)) @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + + @mock.patch('hmac.HMAC') def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 2def73f..8081501 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -78,6 +78,10 @@ class StubResponse(object): self.body = body self.headers = headers or {} + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, + self.body, self.headers) + def fake_http_connect(*code_iter, **kwargs): """ @@ -102,7 +106,6 @@ def fake_http_connect(*code_iter, **kwargs): self.etag = etag self.content = self.body = body self.timestamp = timestamp - self._is_closed = True self.headers = headers or {} self.request = None @@ -162,6 +165,9 @@ def fake_http_connect(*code_iter, **kwargs): def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) + def close(self): + pass + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) @@ -228,7 +234,8 @@ class MockHttpTest(unittest.TestCase): parsed, _conn = _orig_http_connection(url, proxy=proxy) class RequestsWrapper(object): - pass + def close(self): + pass conn = RequestsWrapper() def request(method, path, *args, **kwargs): @@ -542,14 +549,6 @@ class FakeKeystone(object): pass -def _make_fake_import_keystone_client(fake_import): - def _fake_import_keystone_client(auth_version): - fake_import.auth_version = auth_version - return fake_import, fake_import - - return _fake_import_keystone_client - - class FakeStream(object): def __init__(self, size): self.bytes_read = 0 diff --git a/tools/swift.bash_completion b/tools/swift.bash_completion new file mode 100644 index 0000000..2f98a6b --- /dev/null +++ b/tools/swift.bash_completion @@ -0,0 +1,32 @@ +declare -a _swift_opts # lazy init + +_swift_get_current_opt() +{ + local opt + for opt in ${_swift_opts[@]} ; do + if [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt\$") > 0 ]] || [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt ") > 0 ]] ; then + echo $opt + return 0 + fi + done + echo "" + return 0 +} + +_swift() +{ + local opt cur prev sflags + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [ "x$_swift_opts" == "x" ] ; then + _swift_opts=(`swift bash_completion "$sbc" | sed -e "s/-[-A-Za-z0-9_]*//g" -e "s/ */ /g"`) + fi + + opt="$(_swift_get_current_opt)" + COMPREPLY=($(compgen -W "$(swift bash_completion $opt)" -- ${cur})) + + return 0 +} +complete -F _swift swift @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,pypy,pep8 +envlist = py37,py36,py35,py27,pypy,pep8 minversion = 2.0 skipsdist = True @@ -8,7 +8,7 @@ usedevelop = True install_command = python -m pip install -U {opts} {packages} list_dependencies_command = python -m pip freeze setenv = - LANG=en_US.utf8 + LANG=en_US.utf-8 VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt @@ -16,35 +16,59 @@ deps = -r{toxinidir}/requirements.txt .[keystone] commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ - -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' - python setup.py testr --testr-args="{posargs}" + -print0) | xargs -0 rm -rf' + stestr run {posargs} whitelist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] +basepython = python3 commands = python -m flake8 swiftclient tests [testenv:venv] +basepython = python3 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage - coverage report +basepython = python3 +setenv = + PYTHON=coverage run --source swiftclient --parallel-mode +commands = + stestr run + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [testenv:func] -setenv = OS_TEST_PATH=tests.functional +basepython = python3 +setenv = + OS_TEST_PATH=tests.functional + PYTHON=coverage run --source swiftclient --parallel-mode whitelist_externals = coverage rm commands = - python setup.py testr --coverage --testr-args="--concurrency=1" + stestr run --concurrency=1 + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage +[testenv:py2func] +basepython=python2 +setenv = {[testenv:func]setenv} +whitelist_externals = {[testenv:func]whitelist_externals} +commands = {[testenv:func]commands} + [testenv:docs] +basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx + python setup.py build_sphinx -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: @@ -55,11 +79,16 @@ commands= # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = H101,H301,H306,H401,H403,H404,H405 +# W504: line break after binary operator +ignore = H101,H301,H306,H401,H403,H404,H405,W504 +# H106: Don’t put vim configuration in source files +# H203: Use assertIs(Not)None to check for None +enable-extensions=H106,H203 show-source = True exclude = .venv,.tox,dist,doc,*egg [testenv:bindep] +basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -69,4 +98,14 @@ deps = bindep commands = bindep test [testenv:releasenotes] +basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + -r{toxinidir}/test-requirements.txt + .[keystone] |