diff options
38 files changed, 2953 insertions, 800 deletions
diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..67a39c4 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,7 @@ +- project: + check: + jobs: + - openstack-tox-lower-constraints + gate: + jobs: + - openstack-tox-lower-constraints @@ -42,6 +42,7 @@ Florent Flament (florent.flament-ext@cloudwatt.com) Greg Holt (gholt@rackspace.com) Greg Lange (greglange@gmail.com) groqez (groqez@yopmail.net) +Hangdong Zhang (hdzhang@fiberhome.com) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) hgangwx (hgangwx@cn.ibm.com) Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) @@ -65,11 +66,14 @@ Josh Gachnang (josh@pcsforeducation.com) Juan J. Martinez (juan@memset.com) Jude Job (judeopenstack@gmail.com) Julien Danjou (julien@danjou.info) +Kazufumi Noto (noto.kazufumi@gmail.com) Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) +liuyamin (liuyamin@fiberhome.com) Luis de Bethencourt (luis@debethencourt.com) +M V P Nitesh (m.nitesh@nectechnologies.in) Mahati Chamarthy (mahati.chamarthy@gmail.com) Marek Kaleta (marek.kaleta@firma.seznam.cz) Mark Seger (mark.seger@hpe.com) @@ -82,6 +86,7 @@ Min Min Ren (rminmin@cn.ibm.com) Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) +Nelson Marcos (nelsonmarcos@gmail.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -90,6 +95,7 @@ Paul Belanger (pabelanger@redhat.com) Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br) Pete Zaitcev (zaitcev@kotori.zaitcev.us) Peter Lisak (peter.lisak@firma.seznam.cz) +Petr Kovar (pkovar@redhat.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) Pratik Mallya (pratik.mallya@gmail.com) Qiu Yu (qiuyu@ebaysf.com) @@ -120,12 +126,14 @@ Thiago da Silva (thiago@redhat.com) Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) Tim Burke (tim.burke@gmail.com) +Timur Alperovich (timuralp@swiftstack.com) Tong Li (litong01@us.ibm.com) Tony Breeds (tony@bakeyournoodle.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) +Vitaly Gridnev (vgridnev@mirantis.com) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) YangLei (yanglyy@cn.ibm.com) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ec32d85..0bde968 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,13 +1,13 @@ If you would like to contribute to the development of OpenStack, you must follow the steps in this page: - http://docs.openstack.org/infra/manual/developers.html + https://docs.openstack.org/infra/manual/developers.html Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at: - http://docs.openstack.org/infra/manual/developers.html#development-workflow. + https://docs.openstack.org/infra/manual/developers.html#development-workflow Gerrit is the review system used in the OpenStack projects. We're sorry, but we won't be able to respond to pull requests submitted through @@ -1,3 +1,50 @@ +3.5.0 +----- + +* Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + +* The ``--meta`` option can now be set on the upload command. + +* Updated PyPy test dependency references to be more accurate + on different distros. + +* Various other minor bug fixes and improvements. + +3.4.0 +----- + +* The `swift` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + `--object-name` must be given when content is loaded from stdin. + +* Tolerate RFC-compliant ETags returned from the server. + +* Skip checksum validation on partial downloads. + +* Buffer reads from disk, resulting in much faster upload throughput. + +* Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + +* Added an option to ignore mtime metadata entry (`--ignore-mtime`). + +* When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + +* Expose `--prefix` as an option for st_delete. + +* Imported docs content from openstack-manuals project. + +* Various other minor bug fixes and improvements. + 3.3.0 ----- @@ -2,8 +2,8 @@ Team and repository tags ======================== -.. image:: https://governance.openstack.org/badges/python-swiftclient.svg - :target: https://governance.openstack.org/reference/tags/index.html +.. image:: https://governance.openstack.org/tc/badges/python-swiftclient.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html .. Change things from this point on @@ -11,26 +11,22 @@ Python bindings to the OpenStack Object Storage API =================================================== .. image:: https://img.shields.io/pypi/v/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ + :target: https://pypi.org/project/python-swiftclient/ :alt: Latest Version -.. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ - :alt: Downloads - This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). Development takes place via the usual OpenStack processes as outlined in the `OpenStack wiki`__. -__ http://docs.openstack.org/infra/manual/developers.html +__ https://docs.openstack.org/infra/manual/developers.html This code is based on the original client previously included with `OpenStack's Swift`__ The python-swiftclient is licensed under the Apache License like the rest of OpenStack. -__ http://github.com/openstack/swift +__ https://github.com/openstack/swift * Free software: Apache license * `PyPI`_ - package installation @@ -42,14 +38,14 @@ __ http://github.com/openstack/swift * `Specs`_ * `How to Contribute`_ -.. _PyPI: https://pypi.python.org/pypi/python-swiftclient -.. _Online Documentation: http://docs.openstack.org/developer/python-swiftclient +.. _PyPI: https://pypi.org/project/python-swiftclient +.. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient .. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient -.. _How to Contribute: http://docs.openstack.org/infra/manual/developers.html -.. _Specs: http://specs.openstack.org/openstack/swift-specs/ +.. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html +.. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. contents:: Contents: @@ -1,6 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; -# see http://docs.openstack.org/infra/bindep/ for additional information. +# see https://docs.openstack.org/infra/bindep/ for additional information. -curl -pypy [test] -pypy-dev [test] +pypy [test !platform:fedora] +pypy-dev [test platform:dpkg] +pypy-devel [test platform:rpm !platform:fedora] diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 1f288d6..00e1440 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the remaining args. The \-c or \-\-changed is an option that will only upload files that have changed since the last upload. The \-\-object\-name <object\-name> is an option that will upload file and name object to <object\-name> or upload dir -and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size> -and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more). +and use <object\-name> as object prefix. If the file name is "-", reads the +content from standard input. In this case, \-\-object\-name is required and no +other files may be given. The \-S <size> or \-\-segment\-size <size> and +\-\-leave\-segments and others are options as well (see swift upload \-\-help +for more). .RE \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] @@ -102,6 +105,9 @@ with \-\-no-download actually not to write anything to disk. The \-\-ignore-checksum is an option that turns off checksum validation. You can specify optional headers with the repeatable cURL-like option \-H [\-\-header]. For more details and options see swift download \-\-help. +The \-\-ignore\-mtime option ignores the x\-object\-meta\-mtime metadata entry +on the object (if present) and instead creates the downloaded files with +fresh atime and mtime values. .RE \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] @@ -205,4 +211,4 @@ swift \-A https://127.0.0.1:443/auth/v1.0 \-U swiftops:swiftops \-K swiftops sta .SH DOCUMENTATION .LP More in depth documentation about OpenStack Swift as a whole can be found at -.BI https://docs.openstack.org/developer/swift +.BI https://docs.openstack.org/swift/latest/ diff --git a/doc/source/cli.rst b/doc/source/cli.rst deleted file mode 100644 index 8d80d1b..0000000 --- a/doc/source/cli.rst +++ /dev/null @@ -1,442 +0,0 @@ -==== -CLI -==== - -The ``swift`` tool is a command line utility for communicating with an OpenStack -Object Storage (swift) environment. It allows one to perform several types of -operations. - -Authentication -~~~~~~~~~~~~~~ - -This section covers the options for authenticating with a swift -object store. The combinations of options required for each authentication -version are detailed below, but are just a subset of those that can be used -to successfully authenticate. These are the most common and recommended -combinations. - -You should obtain the details of your authentication version and credentials -from your storage provider. These details should make it clearer which of the -authentication sections below are most likely to allow you to connect to your -storage account. - -Keystone v3 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-name project1 --os-project-domain-name domain1 \ - --os-username user --os-user-domain-name domain1 \ - --os-password password list - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-id 0123456789abcdef0123456789abcdef \ - --os-user-id abcdef0123456789abcdef0123456789 \ - --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following combinations of environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=3 - OS_USERNAME=user - OS_USER_DOMAIN_NAME=domain1 - OS_PASSWORD=password - OS_PROJECT_NAME=project1 - OS_PROJECT_DOMAIN_NAME=domain1 - OS_AUTH_URL=https://api.example.com:5000/v3 - - ST_AUTH_VERSION=3 - OS_USER_ID=abcdef0123456789abcdef0123456789 - OS_PASSWORD=password - OS_PROJECT_ID=0123456789abcdef0123456789abcdef - OS_AUTH_URL=https://api.example.com:5000/v3 - -Keystone v2 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v2.0 \ - --os-tenant-name tenant \ - --os-username user --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=2.0 - OS_USERNAME=user - OS_PASSWORD=password - OS_TENANT_NAME=tenant - OS_AUTH_URL=https://api.example.com:5000/v2.0 - -Legacy auth systems -------------------- - -You can configure swift to work with any number of other authentication systems -that we will not cover in this document. If your storage provider is not using -Keystone to provide access tokens, please contact them for instructions on the -required options. It is likely that the options will need to be specified as -below: - -.. code-block:: bash - - swift -A https://api.example.com/v1.0 -U user -K api_key list - -Specifying the options above manually on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=1.0 - ST_AUTH=https://api.example.com/v1.0 - ST_USER=user - ST_KEY=key - -It is also possible that you need to use a completely separate auth system, in which -case ``swiftclient`` cannot request a token for you. In this case you should make the -authentication request separately and access your storage using the token and -storage URL options shown below: - -.. code-block:: bash - - swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ - list - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. - -CLI commands -~~~~~~~~~~~~ - -Stat ----- - - ``stat [container [object]]`` - - Displays information for the account, container, or object depending on - the arguments given (if any). In verbose mode, the storage URL and the - authentication token are displayed as well. - -List ----- - - ``list [command-options] [container]`` - - Lists the containers for the account or the objects for a container. - The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only - list items beginning with that prefix. The ``-d <delimiter>`` or - ``--delimiter <delimiter>`` is an option (for container listings only) - that will roll up items with the given delimiter (see `OpenStack Swift - general documentation <http://docs.openstack.org/developer/swift/>` for - what this means). - - The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` - and ``ls -lh``, the latter providing sizes in human readable format - (For example: ``3K``, ``12M``, etc). The latter two switches use more - overhead to retrieve the displayed details, which is directly proportional - to the number of container or objects listed. - -Upload ------- - - ``upload [command-options] container file_or_directory [file_or_directory] [...]`` - - Uploads the files and directories specified by the remaining arguments to the - given container. The ``-c`` or ``--changed`` is an option that will only - upload files that have changed since the last upload. The - ``--object-name <object-name>`` is an option that will upload a file and - name object to ``<object-name>`` or upload a directory and use ``<object-name>`` - as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and - ``--leave-segments`` are options as well (see ``--help`` for more). - -Post ----- - - ``post [command-options] [container] [object]`` - - Updates meta information for the account, container, or object depending - on the arguments given. If the container is not found, the ``swiftclient`` - will create it automatically, but this is not true for accounts and - objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl - <read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. - The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, - and is used to define the user metadata items to set in the form ``Name:Value``. - You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` - - For more information about ACL formats see the documentation: - `ACLs <http://docs.openstack.org/developer/swift/misc.html#acls/>`_. - -Download --------- - - ``download [command-options] [container] [object] [object] [...]`` - - Downloads everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. For a - single object download, you may use the ``-o <filename>`` or ``--output <filename>`` - option to redirect the output to a specific file or ``-`` to - redirect to stdout. The ``--ignore-checksum`` is an option that turn off - checksum validation. You can specify optional headers with the repeatable - cURL-like option ``-H [--header <name:value>]``. - -Delete ------- - - ``delete [command-options] [container] [object] [object] [...]`` - - Deletes everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. Segments - of manifest objects will be deleted as well, unless you specify the - ``--leave-segments`` option. - -Copy ----- - - ``copy [command-options] container object`` - - Copies an object to a new destination or adds user metadata to an object. Depending - on the options supplied, you can preserve existing metadata in contrast to the post - command. The ``--destination`` option sets the copy target destination in the form - ``/container/object``. If not set, the object will be copied onto itself which is useful - for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy - an object without existing user meta data, and the ``-m`` or ``--meta`` option - to define user meta data items to set in the form ``Name:Value``. You can repeat - this option. For example: ``copy -m Color:Blue -m Size:Large``. - -Capabilities ------------- - - ``capabilities [proxy-url]`` - - Displays cluster capabilities. The output includes the list of the - activated Swift middlewares as well as relevant options for each ones. - Additionally the command displays relevant options for the Swift core. If - the ``proxy-url`` option is not provided, the storage URL retrieved after - authentication is used as ``proxy-url``. - -Tempurl -------- - - ``tempurl [command-options] [method] [time] [path] [key]`` - - Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to - allow for this temporary URL that is usually ``GET` or ``PUT``. ``time`` option sets - the amount of time the temporary URL will be valid for. - ``time`` can be specified as an integer, denoting the number of seconds - from now on until the URL shall be valid; or, if ``--absolute`` - is passed, the Unix timestamp when the temporary URL will expire. - But beyond that, ``time`` can also be specified as an ISO 8601 timestamp - in one of following formats: - - i) Complete date: YYYY-MM-DD (eg 1997-07-16) - - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) - - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) - - Please be aware that if you don't provide the UTC designator (i.e., Z) - the timestamp is generated using your local timezone. If only a date is - specified, the time part used will equal to ``00:00:00``. - - ``path`` option sets the full path to the Swift object. - Example: ``/v1/AUTH_account/c/o``. ``key`` option is - the secret temporary URL key set on the Swift cluster. To set a key, run - ``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary - URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not - forget to append the desired objectname at the end of the path portion (and before the - query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the - URL by using the ``--iso8601`` option. - -Auth ----- - - ``auth`` - - Display authentication variables in shell friendly format. Command to run to export storage - URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. - Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic - authentication: ``swift auth -v -U test:tester -K testing``. - -Examples -~~~~~~~~ - -In this section we present some example usage of the ``swift`` CLI. To keep the -examples as short as possible, these examples assume that the relevant authentication -options have been set using environment variables. You can obtain the full list of -commands and options available in the ``swift`` CLI by executing the following: - -.. code-block:: bash - - > swift --help - > swift <command> --help - -Simple examples ---------------- - -List the existing swift containers: - -.. code-block:: bash - - > swift list - - container_1 - -Create a new container: - -.. code-block:: bash - - > swift post TestContainer - -Upload an object into a container: - -.. code-block:: bash - - > swift upload TestContainer testSwift.txt - - testSwift.txt - -List the contents of a container: - -.. code-block:: bash - - > swift list TestContainer - - testSwift.txt - -Copy an object to new destination: - -.. code-block:: bash - - > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt - - SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt - -Delete an object from a container: - -.. code-block:: bash - - > swift delete TestContainer testSwift.txt - - testSwift.txt - -Delete a container: - -.. code-block:: bash - - > swift delete TestContainer - - TestContainer - -Display auth related authentication variables in shell friendly format: - -.. code-block:: bash - - > swift auth - - export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 - export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 - -Download an object from a container: - -.. code-block:: bash - - > swift download TestContainer testSwift.txt - - testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name <object-name> is an option that will upload - file and name object to <object-name> or upload directory and use <object-name> as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. - -For example: - -.. code-block:: bash - - > swift upload TestContainer /home/swift/testSwift/testSwift.txt - - home/swift/testSwift/testSwift.txt - - > swift list TestContainer - - home/swift/testSwift/testSwift.txt - -More complex examples ---------------------- - -Swift has a single object size limit of 5GiB. In order to upload files larger -than this, we must create a large object that consists of smaller segments. -The example below shows how to upload a large video file as a static large -object in 1GiB segments: - -.. code-block:: bash - - > swift upload videos --use-slo --segment-size 1G myvideo.mp4 - - myvideo.mp4 segment 8 - myvideo.mp4 segment 4 - myvideo.mp4 segment 2 - myvideo.mp4 segment 7 - myvideo.mp4 segment 0 - myvideo.mp4 segment 1 - myvideo.mp4 segment 3 - myvideo.mp4 segment 6 - myvideo.mp4 segment 5 - myvideo.mp4 - -This command will upload segments to a container named ``videos_segments``, and -create a manifest file describing the entire object in the ``videos`` container. -For more information on large objects, see the documentation `here -<http://docs.openstack.org/developer/swift/overview_large_objects.html>`_. - -.. code-block:: bash - - > swift list videos - - myvideo.mp4 - - > swift list videos_segments - - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 - -Firstly, the key should be set, then generate a temporary URL for a Swift object: - -.. code-block:: bash - - > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" - - > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ - /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 - - /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ - 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..88fafa1 --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,964 @@ +==== +CLI +==== + +The ``swift`` tool is a command line utility for communicating with an OpenStack +Object Storage (swift) environment. It allows one to perform several types of +operations. + + +For help on a specific :command:`swift` command, enter: + +.. code-block:: console + + $ swift COMMAND --help + +.. _swift_command_usage: + +swift usage +~~~~~~~~~~~ + +.. code-block:: console + + Usage: swift [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + <subcommand> [--help] [<subcommand options>] + +**Subcommands:** + +``delete`` + Delete a container or objects within a container. + +``download`` + Download objects from containers. + +``list`` + Lists the containers for the account or the objects + for a container. + +``post`` + Updates meta information for the account, container, + or object; creates containers if not present. + +``copy`` + Copies object, optionally adds meta + +``stat`` + Displays information for the account, container, + or object. + +``upload`` + Uploads files or directories to the given container. + +``capabilities`` + List cluster capabilities. + +``tempurl`` + Create a temporary URL. + +``auth`` + Display auth related environment variables. + +.. _swift_command_options: + +swift optional arguments +~~~~~~~~~~~~~~~~~~~~~~~~ + +``--version`` + show program's version number and exit + +``-h, --help`` + show this help message and exit + +``--os-help`` + Show OpenStack authentication options. + +``-s, --snet`` + Use SERVICENET internal network. + +``-v, --verbose`` + Print more info. + +``--debug`` + Show the curl commands and results of all http queries + regardless of result status. + +``--info`` + Show the curl commands and results of all http queries + which return an error. + +``-q, --quiet`` + Suppress status output. + +``-A AUTH, --auth=AUTH`` + URL for obtaining an auth token. + +``-V AUTH_VERSION, --auth-version=AUTH_VERSION, --os-identity-api-version=AUTH_VERSION`` + Specify a version for authentication. Defaults to + ``env[ST_AUTH_VERSION]``, ``env[OS_AUTH_VERSION]``, + ``env[OS_IDENTITY_API_VERSION]`` or 1.0. + +``-U USER, --user=USER`` + User name for obtaining an auth token. + +``-K KEY, --key=KEY`` + Key for obtaining an auth token. + +``-R RETRIES, --retries=RETRIES`` + The number of times to retry a failed connection. + +``--insecure`` + Allow swiftclient to access servers without having to + verify the SSL certificate. Defaults to + ``env[SWIFTCLIENT_INSECURE]`` (set to 'true' to enable). + +``--no-ssl-compression`` + This option is deprecated and not used anymore. SSL + compression should be disabled by default by the + system SSL library. + +``--prompt`` + Prompt user to enter a password which overrides any password supplied via + ``--key``, ``--os-password`` or environment variables. + +Authentication +~~~~~~~~~~~~~~ + +This section covers the options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are just a subset of those that can be used +to successfully authenticate. These are the most common and recommended +combinations. + +You should obtain the details of your authentication version and credentials +from your storage provider. These details should make it clearer which of the +authentication sections below are most likely to allow you to connect to your +storage account. + +Keystone v3 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-name project1 --os-project-domain-name domain1 \ + --os-username user --os-user-domain-name domain1 \ + --os-password password list + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-id 0123456789abcdef0123456789abcdef \ + --os-user-id abcdef0123456789abcdef0123456789 \ + --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following combinations of environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=3 + OS_USERNAME=user + OS_USER_DOMAIN_NAME=domain1 + OS_PASSWORD=password + OS_PROJECT_NAME=project1 + OS_PROJECT_DOMAIN_NAME=domain1 + OS_AUTH_URL=https://api.example.com:5000/v3 + + ST_AUTH_VERSION=3 + OS_USER_ID=abcdef0123456789abcdef0123456789 + OS_PASSWORD=password + OS_PROJECT_ID=0123456789abcdef0123456789abcdef + OS_AUTH_URL=https://api.example.com:5000/v3 + +Keystone v2 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v2.0 \ + --os-tenant-name tenant \ + --os-username user --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=2.0 + OS_USERNAME=user + OS_PASSWORD=password + OS_TENANT_NAME=tenant + OS_AUTH_URL=https://api.example.com:5000/v2.0 + +Legacy auth systems +------------------- + +You can configure swift to work with any number of other authentication systems +that we will not cover in this document. If your storage provider is not using +Keystone to provide access tokens, please contact them for instructions on the +required options. It is likely that the options will need to be specified as +below: + +.. code-block:: bash + + swift -A https://api.example.com/v1.0 -U user -K api_key list + +Specifying the options above manually on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=1.0 + ST_AUTH=https://api.example.com/v1.0 + ST_USER=user + ST_KEY=key + +It is also possible that you need to use a completely separate auth system, in which +case ``swiftclient`` cannot request a token for you. In this case you should make the +authentication request separately and access your storage using the token and +storage URL options shown below: + +.. code-block:: bash + + swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ + list + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +CLI commands +~~~~~~~~~~~~ + +.. _swift_auth: + +Auth +---- + +.. code-block:: console + + Usage: swift auth + +Display authentication variables in shell friendly format. Command to run to export storage +URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. +Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic +authentication: ``swift auth -v -U test:tester -K testing``. + +.. _swift_stat: + +swift stat +---------- + +.. code-block:: console + + Usage: swift stat [--lh] [--header <header:value>] + [<container> [<object>]] + +Displays information for the account, container, or object depending on +the arguments given (if any). In verbose mode, the storage URL and the +authentication token are displayed as well. + +**Positional arguments:** + +``[container]`` + Name of container to stat from. + +``[object]`` + Name of object to stat. + +**Optional arguments:** + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-H, --header <header:value>`` + Adds a custom request header to use for stat. + +.. _swift_list: + +swift list +---------- + +.. code-block:: console + + Usage: swift list [--long] [--lh] [--totals] [--prefix <prefix>] + [--delimiter <delimiter>] [--header <header:value>] + [<container>] + +Lists the containers for the account or the objects for a container. +The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only +list items beginning with that prefix. The ``-d <delimiter>`` or +``--delimiter <delimiter>`` is an option (for container listings only) +that will roll up items with the given delimiter (see `OpenStack Swift +general documentation <https://docs.openstack.org/swift/latest/>` for +what this means). + +The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` +and ``ls -lh``, the latter providing sizes in human readable format +(For example: ``3K``, ``12M``, etc). The latter two switches use more +overhead to retrieve the displayed details, which is directly proportional +to the number of container or objects listed. + +**Positional arguments:** + +``[container]`` + Name of container to list object in. + +**Optional arguments:** + +``-l, --long`` + Long listing format, similar to ls -l. + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-t, --totals`` + Used with -l or --lh, only report totals. + +``-p <prefix>, --prefix <prefix>`` + Only list items beginning with the prefix. + +``-d <delim>, --delimiter <delim>`` + Roll up items with the given delimiter. For containers + only. See OpenStack Swift API documentation for what + this means. + +``-H, --header <header:value>`` + Adds a custom request header to use for listing. + +.. _swift_upload: + +swift upload +------------ + +.. code-block:: console + + Usage: swift upload [--changed] [--skip-identical] [--segment-size <size>] + [--segment-container <container>] [--leave-segments] + [--object-threads <thread>] [--segment-threads <threads>] + [--header <header>] [--use-slo] [--ignore-checksum] + [--object-name <object-name>] + <container> <file_or_directory> [<file_or_directory>] [...] + +Uploads the files and directories specified by the remaining arguments to the +given container. The ``-c`` or ``--changed`` is an option that will only +upload files that have changed since the last upload. The +``--object-name <object-name>`` is an option that will upload a file and +name object to ``<object-name>`` or upload a directory and use ``<object-name>`` +as object prefix. If the file name is "-", client reads content from standard +input. In this case ``--object-name`` is required to set the name of the object +and no other files may be given. The ``-S <size>`` or ``--segment-size <size>`` +and ``--leave-segments`` are options as well (see ``--help`` for more). + +**Positional arguments:** + +``<container>`` + Name of container to upload to. + +``<file_or_directory>`` + Name of file or directory to upload. Specify multiple + times for multiple uploads. + +**Optional arguments:** + +``-c, --changed`` + Only upload files that have changed since the last + upload. + +``--skip-identical`` + Skip uploading files that are identical on both sides. + +``-S, --segment-size <size>`` + Upload files in segments no larger than <size> (in + Bytes) and then create a "manifest" file that will + download all the segments as if it were the original + file. + +``--segment-container <container>`` + Upload the segments into the specified container. If + not specified, the segments will be uploaded to a + <container>_segments container to not pollute the + main <container> listings. + +``--leave-segments`` + Indicates that you want the older segments of manifest + objects left alone (in the case of overwrites). + +``--object-threads <threads>`` + Number of threads to use for uploading full objects. + Default is 10. + +``--segment-threads <threads>`` + Number of threads to use for uploading object segments. + Default is 10. + +``-H, --header <header:value>`` + Adds a customized request header. This option may be + repeated. Example: -H "content-type:text/plain" + -H "Content-Length: 4000". + +``--use-slo`` + When used in conjunction with --segment-size it will + create a Static Large Object instead of the default + Dynamic Large Object. + +``--object-name <object-name>`` + Upload file and name object to <object-name> or upload + dir and use <object-name> as object prefix instead of + folder name. + +``--ignore-checksum`` + Turn off checksum validation for uploads. + + +.. _swift_post: + +swift post +---------- + +.. code-block:: console + + Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to] + [--sync-key <sync-key>] [--meta <name:value>] + [--header <header>] + [<container> [<object>]] + +Updates meta information for the account, container, or object depending +on the arguments given. If the container is not found, the ``swiftclient`` +will create it automatically, but this is not true for accounts and +objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl +<read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. +The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, +and is used to define the user metadata items to set in the form ``Name:Value``. +You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` + +For more information about ACL formats see the documentation: +`ACLs <https://docs.openstack.org/swift/latest/misc.html#acls>`_. + +**Positional arguments:** + +``[container]`` + Name of container to post to. + +``[object]`` + Name of object to post. + +**Optional arguments:** + +``-r, --read-acl <acl>`` + Read ACL for containers. Quick summary of ACL syntax: + ``.r:*``, ``.r:-.example.com``, ``.r:www.example.com``, + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-w, --write-acl <acl>`` + Write ACL for containers. Quick summary of ACL syntax: + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-t, --sync-to <sync-to>`` + Sync To for containers, for multi-cluster replication. + +``-k, --sync-key <sync-key>`` + Sync Key for containers, for multi-cluster replication. + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. + This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_download: + +swift download +-------------- + +.. code-block:: console + + Usage: swift download [--all] [--marker <marker>] [--prefix <prefix>] + [--output <out_file>] [--output-dir <out_directory>] + [--object-threads <threads>] [--ignore-checksum] + [--container-threads <threads>] [--no-download] + [--skip-identical] [--remove-prefix] + [--header <header:value>] [--no-shuffle] + [<container> [<object>] [...]] + +Downloads everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. For a +single object download, you may use the ``-o <filename>`` or ``--output <filename>`` +option to redirect the output to a specific file or ``-`` to +redirect to stdout. The ``--ignore-checksum`` is an option that turn off +checksum validation. You can specify optional headers with the repeatable +cURL-like option ``-H [--header <name:value>]``. ``--ignore-mtime`` ignores the +``x-object-meta-mtime`` metadata entry on the object (if present) and instead +creates the downloaded files with fresh atime and mtime values. + +**Positional arguments:** + +``<container>`` + Name of container to download from. To download a + whole account, omit this and specify --all. + +``<object>`` + Name of object to download. Specify multiple times + for multiple objects. Omit this to download all + objects from the container. + +**Optional arguments:** + +``-a, --all`` + Indicates that you really want to download + everything in the account. + +``-m, --marker <marker>`` + Marker to use when starting a container or account + download. + +``-p, --prefix <prefix>`` + Only download items beginning with <prefix> + +``-r, --remove-prefix`` + An optional flag for --prefix <prefix>, use this + option to download items without <prefix> + +``-o, --output <out_file>`` + For a single file download, stream the output to + <out_file>. Specifying "-" as <out_file> will + redirect to stdout. + +``-D, --output-dir <out_directory>`` + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. + +``--object-threads <threads>`` + Number of threads to use for downloading objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for downloading containers. + Default is 10. + +``--no-download`` + Perform download(s), but don't actually write anything + to disk. + +``-H, --header <header:value>`` + Adds a customized request header to the query, like + "Range" or "If-Match". This option may be repeated. + + Example: --header "content-type:text/plain" + +``--skip-identical`` + Skip downloading files that are identical on both + sides. + +``--ignore-checksum`` + Turn off checksum validation for downloads. + +``--no-shuffle`` + By default, when downloading a complete account or + container, download order is randomised in order to + reduce the load on individual drives when multiple + clients are executed simultaneously to download the + same set of objects (e.g. a nightly automated download + script to multiple servers). Enable this option to + submit download jobs to the thread pool in the order + they are listed in the object store. + +.. _swift_delete: + +swift delete +------------ + +.. code-block:: console + + Usage: swift delete [--all] [--leave-segments] + [--object-threads <threads>] + [--container-threads <threads>] + [--header <header:value>] + [<container> [<object>] [...]] + +Deletes everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. Segments +of manifest objects will be deleted as well, unless you specify the +``--leave-segments`` option. + +**Positional arguments:** + +``[<container>]`` + Name of container to delete from. + +``[<object>]`` + Name of object to delete. Specify multiple times + for multiple objects. + +**Optional arguments:** + +``-a, --all`` + Delete all containers and objects. + +``--leave-segments`` + Do not delete segments of manifest objects. + +``-H, --header <header:value>`` + Adds a custom request header to use for deleting + objects or an entire container. + + +``--object-threads <threads>`` + Number of threads to use for deleting objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for deleting containers. + Default is 10. + +.. _swift_copy: + +swift copy +---------- + +.. code-block:: console + + Usage: swift copy [--destination </container/object>] [--fresh-metadata] + [--meta <name:value>] [--header <header>] <container> + <object> [<object>] [...] + +Copies an object to a new destination or adds user metadata to an object. Depending +on the options supplied, you can preserve existing metadata in contrast to the post +command. The ``--destination`` option sets the copy target destination in the form +``/container/object``. If not set, the object will be copied onto itself which is useful +for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy +an object without existing user meta data, and the ``-m`` or ``--meta`` option +to define user meta data items to set in the form ``Name:Value``. You can repeat +this option. For example: ``copy -m Color:Blue -m Size:Large``. + +**Positional arguments:** + +``<container>`` + Name of container to copy from. + +``<object>`` + Name of object to copy. Specify multiple times for multiple objects + +**Optional arguments:** + +``-d, --destination </container[/object]>`` + The container and name of the destination object. Name + of destination object can be omitted, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + +``-M, --fresh-metadata`` + Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_capabilities: + +swift capabilities +------------------ + +.. code-block:: console + + Usage: swift capabilities [--json] [<proxy_url>] + +Displays cluster capabilities. The output includes the list of the +activated Swift middlewares as well as relevant options for each ones. +Additionally the command displays relevant options for the Swift core. If +the ``proxy-url`` option is not provided, the storage URL retrieved after +authentication is used as ``proxy-url``. + +**Optional positional arguments:** + +``<proxy_url>`` + Proxy URL of the cluster to retrieve capabilities. + +``--json`` + Print the cluster capabilities in JSON format. + +.. _swift_tempurl: + +swift tempurl +------------- + +.. code-block:: console + + Usage: swift tempurl [--absolute] [--prefix-based] + <method> <seconds> <path> <key> + +Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to +allow for this temporary URL that is usually ``GET`` or ``PUT``. ``time`` option sets +the amount of time the temporary URL will be valid for. +``time`` can be specified as an integer, denoting the number of seconds +from now on until the URL shall be valid; or, if ``--absolute`` +is passed, the Unix timestamp when the temporary URL will expire. +But beyond that, ``time`` can also be specified as an ISO 8601 timestamp +in one of following formats: + + i) Complete date: YYYY-MM-DD (eg 1997-07-16) + + ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (eg 1997-07-16T19:20:30) + + iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (eg 1997-07-16T19:20:30Z) + +Please be aware that if you don't provide the UTC designator (i.e., Z) +the timestamp is generated using your local timezone. If only a date is +specified, the time part used will equal to ``00:00:00``. + +``path`` option sets the full path to the Swift object. +Example: ``/v1/AUTH_account/c/o``. ``key`` option is +the secret temporary URL key set on the Swift cluster. To set a key, run +``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary +URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not +forget to append the desired objectname at the end of the path portion (and before the +query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the +URL by using the ``--iso8601`` option. + +**Positional arguments:** + +``<method>`` + An HTTP method to allow for this temporary URL. + Usually 'GET' or 'PUT'. + +``<seconds>`` + The amount of time in seconds the temporary URL will be + valid for; or, if --absolute is passed, the Unix + timestamp when the temporary URL will expire. + +``<path>`` + The full path to the Swift object. + + Example: /v1/AUTH_account/c/o + or: http://saio:8080/v1/AUTH_account/c/o + +``<key>`` + The secret temporary URL key set on the Swift cluster. + To set a key, run 'swift post -m + "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"' + +**Optional arguments:** + +``--absolute`` + Interpret the <seconds> positional argument as a Unix + timestamp rather than a number of seconds in the + future. + +``--prefix-based`` + If present, a prefix-based tempURL will be generated. + +Examples +~~~~~~~~ + +In this section we present some example usage of the ``swift`` CLI. To keep the +examples as short as possible, these examples assume that the relevant authentication +options have been set using environment variables. You can obtain the full list of +commands and options available in the ``swift`` CLI by executing the following: + +.. code-block:: bash + + > swift --help + > swift <command> --help + +Simple examples +--------------- + +List the existing swift containers: + +.. code-block:: bash + + > swift list + + container_1 + +Create a new container: + +.. code-block:: bash + + > swift post TestContainer + +Upload an object into a container: + +.. code-block:: bash + + > swift upload TestContainer testSwift.txt + + testSwift.txt + +List the contents of a container: + +.. code-block:: bash + + > swift list TestContainer + + testSwift.txt + +Copy an object to new destination: + +.. code-block:: bash + + > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt + + SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt + +Delete an object from a container: + +.. code-block:: bash + + > swift delete TestContainer testSwift.txt + + testSwift.txt + +Delete a container: + +.. code-block:: bash + + > swift delete TestContainer + + TestContainer + +Display auth related authentication variables in shell friendly format: + +.. code-block:: bash + + > swift auth + + export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 + export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 + +Download an object from a container: + +.. code-block:: bash + + > swift download TestContainer testSwift.txt + + testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. + +For example: + +.. code-block:: bash + + > swift upload TestContainer /home/swift/testSwift/testSwift.txt + + home/swift/testSwift/testSwift.txt + + > swift list TestContainer + + home/swift/testSwift/testSwift.txt + +More complex examples +--------------------- + +Swift has a single object size limit of 5GiB. In order to upload files larger +than this, we must create a large object that consists of smaller segments. +The example below shows how to upload a large video file as a static large +object in 1GiB segments: + +.. code-block:: bash + + > swift upload videos --use-slo --segment-size 1G myvideo.mp4 + + myvideo.mp4 segment 8 + myvideo.mp4 segment 4 + myvideo.mp4 segment 2 + myvideo.mp4 segment 7 + myvideo.mp4 segment 0 + myvideo.mp4 segment 1 + myvideo.mp4 segment 3 + myvideo.mp4 segment 6 + myvideo.mp4 segment 5 + myvideo.mp4 + +This command will upload segments to a container named ``videos_segments``, and +create a manifest file describing the entire object in the ``videos`` container. +For more information on large objects, see the documentation `here +<https://docs.openstack.org/swift/latest/overview_large_objects.html>`_. + +.. code-block:: bash + + > swift list videos + + myvideo.mp4 + + > swift list videos_segments + + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 + +Firstly, the key should be set, then generate a temporary URL for a Swift object: + +.. code-block:: bash + + > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" + + > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ + /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 + + /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ + 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/conf.py b/doc/source/conf.py index 3505f13..f56b643 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -31,8 +31,11 @@ sys.path.insert(0, ROOT) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'oslosphinx'] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'openstackdocstheme'] autoclass_content = 'both' autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] @@ -104,7 +107,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -#html_theme = 'nature' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/source/index.rst b/doc/source/index.rst index f123b7b..3c2cb1e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,7 +16,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 - cli + cli/index service-api client-api diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 0000000..6488b28 --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,46 @@ +alabaster==0.7.10 +Babel==2.3.4 +certifi==2018.1.18 +chardet==3.0.4 +coverage==4.0 +docutils==0.11 +dulwich==0.15.0 +extras==1.0.0 +fixtures==3.0.0 +flake8==2.2.4 +futures==3.0.0 +hacking==0.10.0 +idna==2.6 +imagesize==0.7.1 +iso8601==0.1.8 +Jinja2==2.10 +keystoneauth1==3.4.0 +linecache2==1.0.0 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==1.2.0 +netaddr==0.7.10 +openstackdocstheme==1.18.1 +oslo.config==1.2.0 +oslosphinx==4.7.0 +pbr==2.0.0 +pep8==1.5.7 +PrettyTable==0.7 +pyflakes==0.8.1 +Pygments==2.2.0 +python-keystoneclient==3.8.0 +python-mimeparse==1.6.0 +python-subunit==1.0.0 +pytz==2013.6 +PyYAML==3.12 +reno==2.5.0 +requests==2.14.2 +six==1.10.0 +snowballstemmer==1.2.1 +sphinx==1.6.2 +sphinxcontrib-websupport==1.0.1 +testrepository==0.0.18 +testtools==2.2.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 diff --git a/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml new file mode 100644 index 0000000..0aae5cf --- /dev/null +++ b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml @@ -0,0 +1,20 @@ +--- +features: + + - The ``swift`` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + ``--object-name`` must be given when content is loaded from stdin. + - Tolerate RFC-compliant ETags returned from the server. + - Skip checksum validation on partial downloads. + - Buffer reads from disk, resulting in much faster upload throughput. + - > + Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + - Added an option to ignore mtime metadata entry (``--ignore-mtime``). + - > + When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + - Expose `--prefix` as an option for st_delete. + - Imported docs content from openstack-manuals project. + - Various other minor bug fixes and improvements. diff --git a/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml new file mode 100644 index 0000000..2e6b4ea --- /dev/null +++ b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + + - | + The ``--meta`` option can now be set on the upload command. + + - | + Updated PyPy test dependency references to be more accurate + on different distros. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..b27aa96 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,356 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# swift documentation build configuration file, created by +# sphinx-quickstart on Mon Oct 3 17:01:55 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import datetime + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'reno.sphinxext', + 'openstackdocstheme', +] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Swift Client Release Notes' +copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year + +# Release notes are version independent. +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +# todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# "<project> v<release> documentation" by default. +# +# html_title = u'swift v2.10.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'SwiftClientReleaseNotesdoc' + +# -- Options for LaTeX output --------------------------------------------- + +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', + +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', + +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', + +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# (master_doc, 'swift.tex', u'swift Documentation', +# u'swift', 'manual'), +# ] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +# man_pages = [ +# (master_doc, 'swift', u'swift Documentation', +# [author], 1) +# ] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +# texinfo_documents = [ +# (master_doc, 'swift', u'swift Documentation', +# author, 'swift', 'One line description of project.', +# 'Miscellaneous'), +# ] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + +locale_dirs = ['locale/'] + +# -- Options for openstackdocstheme ------------------------------------------- +repository_name = 'openstack/python-swiftclient' +bug_project = 'python-swiftclient' +bug_tag = '' diff --git a/releasenotes/source/current.rst b/releasenotes/source/current.rst new file mode 100644 index 0000000..87a748f --- /dev/null +++ b/releasenotes/source/current.rst @@ -0,0 +1,5 @@ +==================================== + Current (Unreleased) Release Notes +==================================== + +.. release-notes:: diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..a5240ea --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,12 @@ +============================ + Swift Client Release Notes +============================ + +.. toctree:: + :maxdepth: 1 + + current + queens + pike + ocata + newton diff --git a/releasenotes/source/newton.rst b/releasenotes/source/newton.rst new file mode 100644 index 0000000..59418a3 --- /dev/null +++ b/releasenotes/source/newton.rst @@ -0,0 +1,6 @@ +============================= + Newton Series Release Notes +============================= + +.. release-notes:: + :branch: stable/newton diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst new file mode 100644 index 0000000..726307b --- /dev/null +++ b/releasenotes/source/ocata.rst @@ -0,0 +1,6 @@ +============================ + Ocata Series Release Notes +============================ + +.. release-notes:: + :branch: stable/ocata diff --git a/releasenotes/source/pike.rst b/releasenotes/source/pike.rst new file mode 100644 index 0000000..e2c4806 --- /dev/null +++ b/releasenotes/source/pike.rst @@ -0,0 +1,6 @@ +=========================== + Pike Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/pike diff --git a/releasenotes/source/queens.rst b/releasenotes/source/queens.rst new file mode 100644 index 0000000..36ac616 --- /dev/null +++ b/releasenotes/source/queens.rst @@ -0,0 +1,6 @@ +=================================== + Queens Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/queens diff --git a/requirements.txt b/requirements.txt index 6d31e09..6b52791 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD -requests>=1.1 -six>=1.5.2 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +requests>=2.14.2 # Apache-2.0 +six>=1.10.0 # MIT @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://docs.openstack.org/developer/python-swiftclient +home-page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -33,7 +33,7 @@ data_files = [extras] keystone = - python-keystoneclient>=0.7.0 + python-keystoneclient>=3.8.0 # Apache-2.0 [entry_points] console_scripts = @@ -17,10 +17,16 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools, sys -if sys.version_info < (2, 7): - sys.exit('Sorry, Python < 2.7 is not supported for' - ' python-swiftclient>=3.0') +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass setuptools.setup( - setup_requires=['pbr'], + setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/swiftclient/client.py b/swiftclient/client.py index 80b6eda..cc2a124 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -17,6 +17,7 @@ OpenStack Swift client library used internally """ import socket +import re import requests import logging import warnings @@ -25,7 +26,7 @@ from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError from six.moves import http_client from six.moves.urllib.parse import quote as _quote, unquote -from six.moves.urllib.parse import urlparse, urlunparse +from six.moves.urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time import six @@ -38,6 +39,7 @@ from swiftclient.utils import ( # Default is 100, increase to 256 http_client._MAXHEADERS = 256 +VERSIONFUL_AUTH_PATH = re.compile('v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -550,9 +552,25 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): insecure = kwargs.get('insecure', False) timeout = kwargs.get('timeout', None) - auth_version = kwargs.get('auth_version', '2.0') + auth_version = kwargs.get('auth_version', None) debug = logger.isEnabledFor(logging.DEBUG) + # Add the version suffix in case of versionless Keystone endpoints. If + # auth_version is also unset it is likely that it is v3 + if not VERSIONFUL_AUTH_PATH.match( + urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]): + # Normalize auth_url to end in a slash because urljoin + auth_url = auth_url.rstrip('/') + '/' + if auth_version and auth_version in AUTH_VERSIONS_V2: + auth_url = urljoin(auth_url, "v2.0") + else: + auth_url = urljoin(auth_url, "v3") + auth_version = '3' + logger.debug("Versionless auth_url - using %s as endpoint" % auth_url) + + # Legacy default if not set + if auth_version is None: + auth_version = '2' ksclient, exceptions = _import_keystone_client(auth_version) try: @@ -631,8 +649,10 @@ def get_auth(auth_url, user, key, **kwargs): if session: service_type = os_options.get('service_type', 'object-store') interface = os_options.get('endpoint_type', 'public') + region_name = os_options.get('region_name') storage_url = session.get_endpoint(service_type=service_type, - interface=interface) + interface=interface, + region_name=region_name) token = session.get_token() elif auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -1175,7 +1195,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None, headers=None): + service_token=None, headers=None, query_string=None): """ Get object info @@ -1196,6 +1216,8 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if query_string: + path += '?' + query_string if headers: headers = dict(headers) else: @@ -1286,8 +1308,10 @@ def put_object(url, token=None, container=None, name=None, contents=None, if content_type is not None: headers['Content-Type'] = content_type elif 'Content-Type' not in headers: - # python-requests sets application/x-www-form-urlencoded otherwise - headers['Content-Type'] = '' + if StrictVersion(requests.__version__) < StrictVersion('2.4.0'): + # python-requests sets application/x-www-form-urlencoded otherwise + # if using python3. + headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' @@ -1525,7 +1549,7 @@ class Connection(object): os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, retry_on_ratelimit=False, - timeout=None, session=None): + timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1561,6 +1585,8 @@ class Connection(object): after a backoff. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. + :param force_auth_retry: reset auth info even if client got unexpected + error except 401 Unauthorized. """ self.session = session self.authurl = authurl @@ -1593,6 +1619,7 @@ class Connection(object): self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit self.timeout = timeout + self.force_auth_retry = force_auth_retry def close(self): if (self.http_conn and isinstance(self.http_conn, tuple) @@ -1707,6 +1734,10 @@ class Connection(object): pass else: raise + + if self.force_auth_retry: + self.url = self.token = self.service_token = None + sleep(backoff) backoff = min(backoff * 2, self.max_backoff) if reset_func: @@ -1770,9 +1801,10 @@ class Connection(object): query_string=query_string, headers=headers) - def head_object(self, container, obj, headers=None): + def head_object(self, container, obj, headers=None, query_string=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj, headers=headers) + return self._retry(None, head_object, container, obj, headers=headers, + query_string=query_string) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): diff --git a/swiftclient/service.py b/swiftclient/service.py index 223641b..eedad46 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -50,6 +50,7 @@ from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") @@ -143,6 +144,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), "os_user_domain_name": environ.get('OS_USER_DOMAIN_NAME'), @@ -203,6 +205,7 @@ _default_local_options = { 'shuffle': False, 'destination': None, 'fresh_metadata': False, + 'ignore_mtime': False, } POLICY = 'X-Storage-Policy' @@ -259,7 +262,8 @@ def get_conn(options): insecure=options['insecure'], cert=options['os_cert'], cert_key=options['os_key'], - ssl_compression=options['ssl_compression']) + ssl_compression=options['ssl_compression'], + force_auth_retry=options['force_auth_retry']) def mkdirs(path): @@ -378,10 +382,24 @@ class _SwiftReader(object): self._actual_read = 0 self._content_length = None self._actual_md5 = None - self._expected_etag = headers.get('etag') - - if ('x-object-manifest' not in headers - and 'x-static-large-object' not in headers and checksum): + self._expected_md5 = headers.get('etag', '') + + if len(self._expected_md5) > 1 and self._expected_md5[0] == '"' \ + and self._expected_md5[-1] == '"': + self._expected_md5 = self._expected_md5[1:-1] + + # Some headers indicate the MD5 of the response + # definitely *won't* match the ETag + bad_md5_headers = set([ + 'content-range', + 'x-object-manifest', + 'x-static-large-object', + ]) + if bad_md5_headers.intersection(headers): + # This isn't a useful checksum + self._expected_md5 = '' + + if self._expected_md5 and checksum: self._actual_md5 = md5() if 'content-length' in headers: @@ -399,12 +417,12 @@ class _SwiftReader(object): self._check_contents() def _check_contents(self): - if self._actual_md5 and self._expected_etag: + if self._actual_md5 and self._expected_md5: etag = self._actual_md5.hexdigest() - if etag != self._expected_etag: + if etag != self._expected_md5: raise SwiftError('Error downloading {0}: md5sum != etag, ' '{1} != {2}'.format( - self._path, etag, self._expected_etag)) + self._path, etag, self._expected_md5)) if (self._content_length is not None and self._actual_read != self._content_length): @@ -1111,14 +1129,14 @@ class SwiftService(object): if options['skip_identical']: filename = out_file if out_file else path try: - fp = open(filename, 'rb') + fp = open(filename, 'rb', DISK_BUFFER) except IOError: pass else: with fp: md5sum = md5() while True: - data = fp.read(65536) + data = fp.read(DISK_BUFFER) if not data: break md5sum.update(data) @@ -1126,7 +1144,7 @@ class SwiftService(object): try: start_time = time() - get_args = {'resp_chunk_size': 65536, + get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} if options['skip_identical']: @@ -1209,10 +1227,10 @@ class SwiftService(object): if not no_file: if out_file: - fp = open(out_file, 'wb') + fp = open(out_file, 'wb', DISK_BUFFER) else: if basename(path): - fp = open(path, 'wb') + fp = open(path, 'wb', DISK_BUFFER) else: pseudodir = True @@ -1226,7 +1244,8 @@ class SwiftService(object): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if 'x-object-meta-mtime' in headers and not no_file: + if ('x-object-meta-mtime' in headers and not no_file + and not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: @@ -1485,7 +1504,8 @@ class SwiftService(object): if hasattr(s, 'read'): # We've got a file like object to upload to o file_future = self.thread_manager.object_uu_pool.submit( - self._upload_object_job, container, s, o, object_options + self._upload_object_job, container, s, o, object_options, + results_queue=rq ) details['file'] = s details['object'] = o @@ -1717,7 +1737,7 @@ class SwiftService(object): } fp = None try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) @@ -1767,6 +1787,132 @@ class SwiftService(object): if fp is not None: fp.close() + @staticmethod + def _put_object(conn, container, name, content, headers=None, md5=None): + """ + Upload object into a given container and verify the resulting ETag, if + the md5 optional parameter is passed. + + :param conn: The Swift connection to use for uploads. + :param container: The container to put the object into. + :param name: The name of the object. + :param content: Object content. + :param headers: Headers (optional) to associate with the object. + :param md5: MD5 sum of the content. If passed in, will be used to + verify the returned ETag. + + :returns: A dictionary as the response from calling put_object. + The keys are: + - status + - reason + - headers + On error, the dictionary contains the following keys: + - success (with value False) + - error - the encountered exception (object) + - error_timestamp + - response_dict - results from the put_object call, as + documented above + - attempts - number of attempts made + """ + if headers is None: + headers = {} + else: + headers = dict(headers) + if md5 is not None: + headers['etag'] = md5 + results = {} + try: + etag = conn.put_object( + container, name, content, content_length=len(content), + headers=headers, response_dict=results) + if md5 is not None and etag != md5: + raise SwiftError('Upload verification failed for {0}: md5 ' + 'mismatch {1} != {2}'.format(name, md5, etag)) + results['success'] = True + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + return { + 'success': False, + 'error': err, + 'error_timestamp': err_time, + 'response_dict': results, + 'attempts': conn.attempts, + 'traceback': traceback + } + return results + + @staticmethod + def _upload_stream_segment(conn, container, object_name, + segment_container, segment_name, + segment_size, segment_index, + headers, fd): + """ + Upload a segment from a stream, buffering it in memory first. The + resulting object is placed either as a segment in the segment + container, or if it is smaller than a single segment, as the given + object name. + + :param conn: Swift Connection to use. + :param container: Container in which the object would be placed. + :param object_name: Name of the final object (used in case the stream + is smaller than the segment_size) + :param segment_container: Container to hold the object segments. + :param segment_name: The name of the segment. + :param segment_size: Minimum segment size. + :param segment_index: The segment index. + :param headers: Headers to attach to the segment/object. + :param fd: File-like handle for the content. Must implement read(). + + :returns: Dictionary, containing the following keys: + - complete -- whether the stream is exhausted + - segment_size - the actual size of the segment (may be + smaller than the passed in segment_size) + - segment_location - path to the segment + - segment_index - index of the segment + - segment_etag - the ETag for the segment + """ + buf = [] + dgst = md5() + bytes_read = 0 + while bytes_read < segment_size: + data = fd.read(segment_size - bytes_read) + if not data: + break + bytes_read += len(data) + dgst.update(data) + buf.append(data) + buf = b''.join(buf) + segment_hash = dgst.hexdigest() + + if not buf and segment_index > 0: + # Happens if the segment size aligns with the object size + return {'complete': True, + 'segment_size': 0, + 'segment_index': None, + 'segment_etag': None, + 'segment_location': None, + 'success': True} + + if segment_index == 0 and len(buf) < segment_size: + ret = SwiftService._put_object( + conn, container, object_name, buf, headers, segment_hash) + ret['segment_location'] = '/%s/%s' % (container, object_name) + else: + ret = SwiftService._put_object( + conn, segment_container, segment_name, buf, headers, + segment_hash) + ret['segment_location'] = '/%s/%s' % ( + segment_container, segment_name) + + ret.update( + dict(complete=len(buf) < segment_size, + segment_size=len(buf), + segment_index=segment_index, + segment_etag=segment_hash, + for_object=object_name)) + return ret + def _get_chunk_data(self, conn, container, obj, headers, manifest=None): chunks = [] if 'x-object-manifest' in headers: @@ -1794,8 +1940,10 @@ class SwiftService(object): return chunks def _is_identical(self, chunk_data, path): + if path is None: + return False try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) except IOError: return False @@ -1804,7 +1952,7 @@ class SwiftService(object): to_read = chunk['bytes'] md5sum = md5() while to_read: - data = fp.read(min(65536, to_read)) + data = fp.read(min(DISK_BUFFER, to_read)) if not data: return False md5sum.update(data) @@ -1814,6 +1962,47 @@ class SwiftService(object): # Each chunk is verified; check that we're at the end of the file return not fp.read(1) + @staticmethod + def _upload_slo_manifest(conn, segment_results, container, obj, headers): + """ + Upload an SLO manifest, given the results of uploading each segment, to + the specified container. + + :param segment_results: List of response_dict structures, as populated + by _upload_segment_job. Specifically, each + entry must container the following keys: + - segment_location + - segment_etag + - segment_size + - segment_index + :param container: The container to put the manifest into. + :param obj: The name of the manifest object to use. + :param headers: Optional set of headers to attach to the manifest. + """ + if headers is None: + headers = {} + segment_results.sort(key=lambda di: di['segment_index']) + for seg in segment_results: + seg_loc = seg['segment_location'].lstrip('/') + if isinstance(seg_loc, text_type): + seg_loc = seg_loc.encode('utf-8') + + manifest_data = json.dumps([ + { + 'path': d['segment_location'], + 'etag': d['segment_etag'], + 'size_bytes': d['segment_size'] + } for d in segment_results + ]) + + response = {} + conn.put_object( + container, obj, manifest_data, + headers=headers, + query_string='multipart-manifest=put', + response_dict=response) + return response + def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): if obj.startswith('./') or obj.startswith('.\\'): @@ -1898,6 +2087,8 @@ class SwiftService(object): return res # Merge the command line header options to the put_headers + put_headers.update(split_headers( + options['meta'], 'X-Object-Meta-')) put_headers.update(split_headers(options['header'], '')) # Don't do segment job if object is not big enough, and never do @@ -1969,30 +2160,11 @@ class SwiftService(object): res['segment_results'] = segment_results if options['use_slo']: - segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): - seg_loc = seg_loc.encode('utf-8') - new_slo_manifest_paths.add(seg_loc) - - manifest_data = json.dumps([ - { - 'path': d['segment_location'], - 'etag': d['segment_etag'], - 'size_bytes': d['segment_size'] - } for d in segment_results - ]) - - put_headers['x-static-large-object'] = 'true' - mr = {} - conn.put_object( - container, obj, manifest_data, - headers=put_headers, - query_string='multipart-manifest=put', - response_dict=mr - ) - res['manifest_response_dict'] = mr + response = self._upload_slo_manifest( + conn, segment_results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths = { + seg['segment_location'] for seg in segment_results} else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2010,6 +2182,51 @@ class SwiftService(object): response_dict=mr ) res['manifest_response_dict'] = mr + elif options['use_slo'] and segment_size and not path: + segment = 0 + results = [] + while True: + segment_name = '%s/slo/%s/%s/%08d' % ( + obj, put_headers['x-object-meta-mtime'], + segment_size, segment + ) + seg_container = container + '_segments' + if options['segment_container']: + seg_container = options['segment_container'] + ret = self._upload_stream_segment( + conn, container, obj, + seg_container, + segment_name, + segment_size, + segment, + put_headers, + stream + ) + if not ret['success']: + return ret + if (ret['complete'] and segment == 0) or\ + ret['segment_size'] > 0: + results.append(ret) + if results_queue is not None: + # Don't insert the 0-sized segments or objects + # themselves + if ret['segment_location'] != '/%s/%s' % ( + container, obj) and ret['segment_size'] > 0: + results_queue.put(ret) + if ret['complete']: + break + segment += 1 + if results[0]['segment_location'] != '/%s/%s' % ( + container, obj): + response = self._upload_slo_manifest( + conn, results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths = { + r['segment_location'] for r in results} + res['large_object'] = True + else: + res['response_dict'] = ret + res['large_object'] = False else: res['large_object'] = False obr = {} @@ -2017,7 +2234,7 @@ class SwiftService(object): try: if path is not None: content_length = getsize(path) - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) contents = LengthWrapper(fp, content_length, md5=options['checksum']) @@ -2043,7 +2260,6 @@ class SwiftService(object): finally: if fp is not None: fp.close() - if old_manifest or old_slo_manifest_paths: drs = [] delobjsmap = {} @@ -2257,17 +2473,18 @@ class SwiftService(object): def _delete_segment(conn, container, obj, results_queue=None): results_dict = {} try: - conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} + conn.delete_object(container, obj, response_dict=results_dict) except Exception as err: - traceback, err_time = report_traceback() - logger.exception(err) - res = { - 'success': False, - 'error': err, - 'traceback': traceback, - 'error_timestamp': err_time - } + if not isinstance(err, ClientException) or err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 841ed6e..ff5b2be 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,16 +17,19 @@ from __future__ import print_function, unicode_literals import argparse +import getpass +import io import json import logging import signal import socket +import warnings from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 from six.moves.urllib.parse import unquote, urlparse -from sys import argv as sys_argv, exit, stderr +from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime from swiftclient import RequestException @@ -48,7 +51,7 @@ except ImportError: BASENAME = 'swift' commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', - 'capabilities', 'info', 'tempurl', 'auth') + 'capabilities', 'info', 'tempurl', 'auth', 'bash_completion') def immediate_exit(signum, frame): @@ -87,7 +90,7 @@ Optional arguments: '''.strip("\n") -def st_delete(parser, args, output_manager): +def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') @@ -111,6 +114,11 @@ def st_delete(parser, args, output_manager): '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' 'Its value must be a positive integer. Default is 10.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options['yes_all']) or (args and options['yes_all']): @@ -272,10 +280,13 @@ Optional arguments: script to multiple servers). Enable this option to submit download jobs to the thread pool in the order they are listed in the object store. + --ignore-mtime Ignore the 'X-Object-Meta-Mtime' header when + downloading an object. Instead, create atime and mtime + with fresh timestamps. '''.strip("\n") -def st_download(parser, args, output_manager): +def st_download(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' @@ -332,6 +343,17 @@ def st_download(parser, args, output_manager): 'nightly automated download script to multiple servers). Enable this ' 'option to submit download jobs to the thread pool in the order they ' 'are listed in the object store.') + parser.add_argument( + '--ignore-mtime', action='store_true', dest='ignore_mtime', + default=False, help='By default, the object-meta-mtime header is used ' + 'to store the access and modified timestamp for the downloaded file. ' + 'With this option, the header is ignored and the timestamps are ' + 'created freshly.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': @@ -482,7 +504,7 @@ Optional arguments: '''.strip('\n') -def st_list(parser, args, output_manager): +def st_list(parser, args, output_manager, return_parser=False): def _print_stats(options, stats, human): total_count = total_bytes = 0 @@ -559,6 +581,11 @@ def st_list(parser, args, output_manager): '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for listing.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if options['delimiter'] and not args: @@ -617,7 +644,7 @@ Optional arguments: '''.strip('\n') -def st_stat(parser, args, output_manager): +def st_stat(parser, args, output_manager, return_parser=False): parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') @@ -626,6 +653,10 @@ def st_stat(parser, args, output_manager): default=[], help='Adds a custom request header to use for stat.') + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] @@ -713,7 +744,7 @@ Optional arguments: '''.strip('\n') -def st_post(parser, args, output_manager): +def st_post(parser, args, output_manager, return_parser=False): parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' @@ -738,6 +769,11 @@ def st_post(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (options['read_acl'] or options['write_acl'] or options['sync_to'] or @@ -810,7 +846,7 @@ Optional arguments: '''.strip('\n') -def st_copy(parser, args, output_manager): +def st_copy(parser, args, output_manager, return_parser=False): parser.add_argument( '-d', '--destination', help='The container and name of the ' 'destination object') @@ -827,6 +863,11 @@ def st_copy(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] @@ -881,8 +922,8 @@ def st_copy(parser, args, output_manager): st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>] [--segment-container <container>] [--leave-segments] [--object-threads <thread>] [--segment-threads <threads>] - [--header <header>] [--use-slo] [--ignore-checksum] - [--object-name <object-name>] + [--meta <name:value>] [--header <header>] [--use-slo] + [--ignore-checksum] [--object-name <object-name>] <container> <file_or_directory> [<file_or_directory>] [...] ''' @@ -892,7 +933,9 @@ Uploads specified files and directories to the given container. Positional arguments: <container> Name of container to upload to. <file_or_directory> Name of file or directory to upload. Specify multiple - times for multiple uploads. + times for multiple uploads. If "-" is specified, reads + content from standard input (--object-name is required + in this case). Optional arguments: -c, --changed Only upload files that have changed since the last @@ -916,6 +959,9 @@ Optional arguments: --segment-threads <threads> Number of threads to use for uploading object segments. Default is 10. + -m, --meta <name:value> + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large -H, --header <header:value> Adds a customized request header. This option may be repeated. Example: -H "content-type:text/plain" @@ -931,7 +977,9 @@ Optional arguments: '''.strip('\n') -def st_upload(parser, args, output_manager): +def st_upload(parser, args, output_manager, return_parser=False): + DEFAULT_STDIN_SEGMENT = 10 * 1024 * 1024 + parser.add_argument( '-c', '--changed', action='store_true', dest='changed', default=False, help='Only upload files that have changed since ' @@ -967,6 +1015,10 @@ def st_upload(parser, args, output_manager): help='Number of threads to use for uploading object segments. ' 'Its value must be a positive integer. Default is 10.') parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' ' This option may be repeated. Example: -H "content-type:text/plain" ' @@ -983,6 +1035,11 @@ def st_upload(parser, args, output_manager): parser.add_argument( '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -993,6 +1050,11 @@ def st_upload(parser, args, output_manager): else: container = args[0] files = args[1:] + from_stdin = '-' in files + if from_stdin and len(files) > 1: + output_manager.error( + 'upload from stdin cannot be used along with other files') + return if options['object_name'] is not None: if len(files) > 1: @@ -1000,6 +1062,10 @@ def st_upload(parser, args, output_manager): return else: orig_path = files[0] + elif from_stdin: + output_manager.error( + 'object-name must be specified with uploads from stdin') + return if options['segment_size']: try: @@ -1032,12 +1098,26 @@ def st_upload(parser, args, output_manager): st_upload_help) return + if from_stdin: + if not options['use_slo']: + options['use_slo'] = True + if not options['segment_size']: + options['segment_size'] = DEFAULT_STDIN_SEGMENT + options['object_uu_threads'] = options['object_threads'] with SwiftService(options=options) as swift: try: objs = [] dir_markers = [] for f in files: + if f == '-': + fd = io.open(stdin.fileno(), mode='rb') + objs.append(SwiftUploadObject( + fd, object_name=options['object_name'])) + # We ensure that there is exactly one "file" to upload in + # this case -- stdin + break + if isfile(f): objs.append(f) elif isdir(f): @@ -1051,7 +1131,7 @@ def st_upload(parser, args, output_manager): # Now that we've collected all the required files and dir markers # build the tuples for the call to upload - if options['object_name'] is not None: + if options['object_name'] is not None and not from_stdin: objs = [ SwiftUploadObject( o, object_name=o.replace( @@ -1139,7 +1219,7 @@ Optional arguments: st_info_help = st_capabilities_help -def st_capabilities(parser, args, output_manager): +def st_capabilities(parser, args, output_manager, return_parser=False): def _print_compo_cap(name, capabilities): for feature, options in sorted(capabilities.items(), key=lambda x: x[0]): @@ -1152,6 +1232,11 @@ def st_capabilities(parser, args, output_manager): parser.add_argument('--json', action='store_true', help='print capability information in json') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1200,7 +1285,12 @@ Display auth related authentication variables in shell friendly format. '''.strip('\n') -def st_auth(parser, args, thread_manager): +def st_auth(parser, args, thread_manager, return_parser=False): + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if options['verbose'] > 1: if options['auth_version'] in ('1', '1.0'): @@ -1279,10 +1369,12 @@ Optional arguments: generated. --iso8601 If present, the generated temporary URL will contain an ISO 8601 UTC timestamp instead of a Unix timestamp. + --ip-range If present, the temporary URL will be restricted to the + given ip or ip range. '''.strip('\n') -def st_tempurl(parser, args, thread_manager): +def st_tempurl(parser, args, thread_manager, return_parser=False): parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, @@ -1302,6 +1394,16 @@ def st_tempurl(parser, args, thread_manager): help=("If present, the temporary URL will contain an ISO 8601 UTC " "timestamp instead of a Unix timestamp."), ) + parser.add_argument( + '--ip-range', action='store', + default=None, + help=("If present, the temporary URL will be restricted to the " + "given ip or ip range."), + ) + + # We return the parser to build up the bash_completion + if return_parser: + return parser (options, args) = parse_args(parser, args) args = args[1:] @@ -1321,7 +1423,8 @@ def st_tempurl(parser, args, thread_manager): path = generate_temp_url(parsed.path, timestamp, key, method, absolute=options['absolute_expiry'], iso8601=options['iso8601'], - prefix=options['prefix_based']) + prefix=options['prefix_based'], + ip_range=options['ip_range']) except ValueError as err: thread_manager.error(err) return @@ -1333,6 +1436,65 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg(url) +st_bash_completion_help = '''Retrieve command specific flags used by bash_completion. + +Optional positional arguments: + <command> Swift client command to filter the flags by. +'''.strip('\n') + + +st_bash_completion_options = '''[command] +''' + + +def st_bash_completion(parser, args, thread_manager, return_parser=False): + if return_parser: + return parser + + global commands + com = args[1] if len(args) > 1 else None + + if com: + if com in commands: + fn_commands = ["st_%s" % com] + else: + print("") + return + else: + fn_commands = [fn for fn in globals().keys() + if fn.startswith('st_') and not fn.endswith('_options') + and not fn.endswith('_help')] + + subparsers = parser.add_subparsers() + subcommands = {} + if not com: + subcommands['base'] = parser + for command in fn_commands: + cmd = command[3:] + if com: + subparser = subparsers.add_parser( + cmd, help=globals()['%s_help' % command]) + add_default_args(subparser) + subparser = globals()[command]( + subparser, args, thread_manager, True) + subcommands[cmd] = subparser + else: + subcommands[cmd] = None + + cmds = set() + opts = set() + for sc_str, sc in list(subcommands.items()): + cmds.add(sc_str) + if sc: + for option in sc._optionals._option_string_actions: + opts.add(option) + + for cmd_to_remove in (com, 'bash_completion', 'base'): + if cmd_to_remove in cmds: + cmds.remove(cmd_to_remove) + print(' '.join(cmds | opts)) + + class HelpFormatter(argparse.HelpFormatter): def _format_action_invocation(self, action): if not action.option_strings: @@ -1366,6 +1528,30 @@ class HelpFormatter(argparse.HelpFormatter): return action.dest +def prompt_for_password(): + """ + Prompt the user for a password. + + :raise SystemExit: if a password cannot be entered without it being echoed + to the terminal. + :return: the entered password. + """ + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=getpass.GetPassWarning, + append=True) + try: + # temporarily set signal handling back to default to avoid user + # Ctrl-c leaving terminal in weird state + signal.signal(signal.SIGINT, signal.SIG_DFL) + return getpass.getpass() + except EOFError: + return None + except getpass.GetPassWarning: + exit('Input stream incompatible with --prompt option') + finally: + signal.signal(signal.SIGINT, immediate_exit) + + def parse_args(parser, args, enforce_requires=True): options, args = parser.parse_known_args(args or ['-h']) options = vars(options) @@ -1391,6 +1577,10 @@ def parse_args(parser, args, enforce_requires=True): if args and args[0] == 'tempurl': return options, args + # do this before process_options sets default auth version + if enforce_requires and options['prompt']: + options['key'] = options['os_password'] = prompt_for_password() + # Massage auth version; build out os_options subdict process_options(options) @@ -1425,88 +1615,7 @@ adding "-V 2" is necessary for this.'''.strip('\n')) return options, args -def main(arguments=None): - argv = sys_argv if arguments is None else arguments - - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] - - version = client_version - parser = argparse.ArgumentParser( - add_help=False, formatter_class=HelpFormatter, usage=''' -%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] - [--debug] [--info] [--quiet] [--auth <auth_url>] - [--auth-version <auth_version> | - --os-identity-api-version <auth_version> ] - [--user <username>] - [--key <api_key>] [--retries <num_retries>] - [--os-username <auth-user-name>] [--os-password <auth-password>] - [--os-user-id <auth-user-id>] - [--os-user-domain-id <auth-user-domain-id>] - [--os-user-domain-name <auth-user-domain-name>] - [--os-tenant-id <auth-tenant-id>] - [--os-tenant-name <auth-tenant-name>] - [--os-project-id <auth-project-id>] - [--os-project-name <auth-project-name>] - [--os-project-domain-id <auth-project-domain-id>] - [--os-project-domain-name <auth-project-domain-name>] - [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] - [--os-storage-url <storage-url>] [--os-region-name <region-name>] - [--os-service-type <service-type>] - [--os-endpoint-type <endpoint-type>] - [--os-cacert <ca-certificate>] [--insecure] - [--os-cert <client-certificate-file>] - [--os-key <client-certificate-key-file>] - [--no-ssl-compression] - <subcommand> [--help] [<subcommand options>] - -Command-line interface to the OpenStack Swift API. - -Positional arguments: - <subcommand> - delete Delete a container or objects within a container. - download Download objects from containers. - list Lists the containers for the account or the objects - for a container. - post Updates meta information for the account, container, - or object; creates containers if not present. - copy Copies object, optionally adds meta - stat Displays information for the account, container, - or object. - upload Uploads files or directories to the given container. - capabilities List cluster capabilities. - tempurl Create a temporary URL. - auth Display auth related environment variables. - -Examples: - %(prog)s download --help - - %(prog)s -A https://api.example.com/v1.0 \\ - -U user -K api_key stat -v - - %(prog)s --os-auth-url https://api.example.com/v2.0 \\ - --os-tenant-name tenant \\ - --os-username user --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-name project1 --os-project-domain-name domain1 \\ - --os-username user --os-user-domain-name domain1 \\ - --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-id 0123456789abcdef0123456789abcdef \\ - --os-user-id abcdef0123456789abcdef0123456789 \\ - --os-password password list - - %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ - list - - %(prog)s list --lh -'''.strip('\n')) - parser.add_argument('--version', action='version', - version='python-swiftclient %s' % version) - parser.add_argument('-h', '--help', action='store_true') - +def add_default_args(parser): default_auth_version = '1.0' for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): try: @@ -1566,6 +1675,17 @@ Examples: help='This option is deprecated and not used anymore. ' 'SSL compression should be disabled by default ' 'by the system SSL library.') + parser.add_argument('--force-auth-retry', + action='store_true', dest='force_auth_retry', + default=False, + help='Force a re-auth attempt on ' + 'any error other than 401 unauthorized') + parser.add_argument('--prompt', + action='store_true', dest='prompt', + default=False, + help='Prompt user to enter a password which overrides ' + 'any password supplied via --key, --os-password ' + 'or environment variables.') os_grp = parser.add_argument_group("OpenStack authentication options") os_grp.add_argument('--os-username', @@ -1708,6 +1828,100 @@ Examples: default=environ.get('OS_KEY'), help='Specify a client certificate key file (for ' 'client auth). Defaults to env[OS_KEY].') + + +def main(arguments=None): + argv = sys_argv if arguments is None else arguments + + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] + [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] + [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] + [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] + [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + [--force-auth-retry] + <subcommand> [--help] [<subcommand options>] + +Command-line interface to the OpenStack Swift API. + +Positional arguments: + <subcommand> + delete Delete a container or objects within a container. + download Download objects from containers. + list Lists the containers for the account or the objects + for a container. + post Updates meta information for the account, container, + or object; creates containers if not present. + copy Copies object, optionally adds meta + stat Displays information for the account, container, + or object. + upload Uploads files or directories to the given container. + capabilities List cluster capabilities. + tempurl Create a temporary URL. + auth Display auth related environment variables. + bash_completion Outputs option and flag cli data ready for + bash_completion. + +Examples: + %(prog)s download --help + + %(prog)s -A https://api.example.com/v1.0 \\ + -U user -K api_key stat -v + + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ + --os-username user --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-name project1 --os-project-domain-name domain1 \\ + --os-username user --os-user-domain-name domain1 \\ + --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-id 0123456789abcdef0123456789abcdef \\ + --os-user-id abcdef0123456789abcdef0123456789 \\ + --os-password password list + + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ + list + + %(prog)s list --lh +'''.strip('\n')) + + version = client_version + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') + + add_default_args(parser) + options, args = parse_args(parser, argv[1:], enforce_requires=False) if options['help'] or options['os_help']: diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8afcde9..5c17c61 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -69,7 +69,7 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False): + prefix=False, iso8601=False, ip_range=None): """Generates a temporary URL that gives unauthenticated access to the Swift object. @@ -92,6 +92,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False, :param prefix: if True then a prefix-based temporary URL will be generated. :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp instead of a UNIX timestamp will be created. + :param ip_range: if a valid ip range, restricts the temporary URL to the + range of ips. :raises ValueError: if timestamp or path is not in valid format. :return: the path portion of a temporary URL """ @@ -155,8 +157,21 @@ def generate_temp_url(path, seconds, key, method, absolute=False, expiration = int(time.time() + timestamp) else: expiration = timestamp - hmac_body = u'\n'.join([method.upper(), str(expiration), - ('prefix:' if prefix else '') + path_for_body]) + + hmac_parts = [method.upper(), str(expiration), + ('prefix:' if prefix else '') + path_for_body] + + if ip_range: + if isinstance(ip_range, six.binary_type): + try: + ip_range = ip_range.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + 'ip_range must be representable as UTF-8' + ) + hmac_parts.insert(0, "ip=%s" % ip_range) + + hmac_body = u'\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility if not isinstance(key, six.binary_type): @@ -169,6 +184,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False, temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) + + if ip_range: + temp_url += u'&temp_url_ip_range={}'.format(ip_range) + if prefix: temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller diff --git a/test-requirements.txt b/test-requirements.txt index 0ce9cec..634851e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,10 @@ -hacking>=0.10.0,<0.11 +hacking<0.11,>=0.10.0 -coverage>=3.6 -mock>=1.2 +coverage!=4.4,>=4.0 # Apache-2.0 +keystoneauth1>=3.4.0 # Apache-2.0 +mock>=1.2.0 # BSD oslosphinx>=4.7.0 # Apache-2.0 -sphinx>=1.1.2,<1.2 +sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD testrepository>=0.0.18 +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.18.1 # Apache-2.0 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index d60ae06..0380d96 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -485,9 +485,6 @@ class TestUsingKeystone(TestFunctional): self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - def setUp(self): - super(TestUsingKeystone, self).setUp() - class TestUsingKeystoneV3(TestFunctional): """ @@ -514,6 +511,3 @@ class TestUsingKeystoneV3(TestFunctional): return swiftclient.Connection(self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - - def setUp(self): - super(TestUsingKeystoneV3, self).setUp() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 260f1cb..12fbaa0 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -36,6 +36,8 @@ from swiftclient.service import ( SwiftService, SwiftError, SwiftUploadObject ) +from tests.unit import utils as test_utils + clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -119,25 +121,36 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) def test_create_with_large_object_headers(self): # md5 should not be initialized if large object headers are present - sr = self.sr('path', 'body', {'x-object-manifest': 'test'}) + sr = self.sr('path', 'body', {'x-object-manifest': 'test', + 'etag': '"%s"' % ('0' * 32)}) + self.assertEqual(sr._path, 'path') + self.assertEqual(sr._body, 'body') + self.assertIsNone(sr._content_length) + self.assertFalse(sr._expected_md5) + self.assertIsNone(sr._actual_md5) + + sr = self.sr('path', 'body', {'x-static-large-object': 'test', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) - sr = self.sr('path', 'body', {'x-static-large-object': 'test'}) + def test_create_with_content_range_header(self): + # md5 should not be initialized if large object headers are present + sr = self.sr('path', 'body', {'content-range': 'bytes 0-3/10', + 'etag': '"%s"' % ('0' * 32)}) self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_ignore_checksum(self): @@ -146,7 +159,7 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertIsNone(sr._content_length) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) self.assertIsNone(sr._actual_md5) def test_create_with_content_length(self): @@ -155,10 +168,9 @@ class TestSwiftReader(unittest.TestCase): self.assertEqual(sr._path, 'path') self.assertEqual(sr._body, 'body') self.assertEqual(sr._content_length, 5) - self.assertIsNone(sr._expected_etag) + self.assertFalse(sr._expected_md5) - self.assertIsNotNone(sr._actual_md5) - self.assertIs(type(sr._actual_md5), self.md5_type) + self.assertIsNone(sr._actual_md5) # Check Contentlength raises error if it isn't an integer self.assertRaises(SwiftError, self.sr, 'path', 'body', @@ -175,11 +187,17 @@ class TestSwiftReader(unittest.TestCase): # Check error is raised if expected etag doesn't match calculated md5. # md5 for a SwiftReader that has done nothing is # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing - sr = self.sr('path', BytesIO(b'body'), {'etag': 'doesntmatch'}) + sr = self.sr('path', BytesIO(b'body'), + {'etag': md5(b'doesntmatch').hexdigest()}) self.assertRaises(SwiftError, _consume, sr) sr = self.sr('path', BytesIO(b'body'), - {'etag': '841a2d689ad86bd1611447453c22c6fc'}) + {'etag': md5(b'body').hexdigest()}) + _consume(sr) + + # Should still work if etag was quoted + sr = self.sr('path', BytesIO(b'body'), + {'etag': '"%s"' % md5(b'body').hexdigest()}) _consume(sr) # Check error is raised if SwiftReader doesn't read the same length @@ -191,11 +209,13 @@ class TestSwiftReader(unittest.TestCase): _consume(sr) # Check that the iterator generates expected length and etag values - sr = self.sr('path', ['abc'.encode()] * 3, {}) + sr = self.sr('path', ['abc'.encode()] * 3, + {'content-length': 9, + 'etag': md5('abc'.encode() * 3).hexdigest()}) _consume(sr) self.assertEqual(sr._actual_read, 9) self.assertEqual(sr._actual_md5.hexdigest(), - '97ac82a5b825239e782d0339e2d7b910') + md5('abc'.encode() * 3).hexdigest()) class _TestServiceBase(unittest.TestCase): @@ -1070,6 +1090,83 @@ class TestService(unittest.TestCase): self.assertEqual(upload_obj_resp['path'], obj['path']) self.assertTrue(mock_open.return_value.closed) + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + class TestServiceUpload(_TestServiceBase): @@ -1128,14 +1225,9 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='テスト/dummy.dat', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 10, - 'segment_container': None, - 'use_slo': False, - 'checksum': True}) + options=dict(s._options, + segment_size=10, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1213,6 +1305,141 @@ class TestServiceUpload(_TestServiceBase): self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + def test_etag_mismatch_with_ignore_checksum(self): def _consuming_conn(*a, **kw): contents = a[2] @@ -1332,12 +1559,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1387,12 +1610,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = float(r['headers']['x-object-meta-mtime']) self.assertEqual(mtime, expected_mtime) @@ -1434,12 +1653,8 @@ class TestServiceUpload(_TestServiceBase): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) self.assertIs(r['success'], False) self.assertIn('md5 mismatch', str(r.get('error'))) @@ -1933,7 +2148,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) written_content.write.assert_called_once_with(b'objcontent') mock_conn.get_object.assert_called_once_with( @@ -1977,7 +2192,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) mock_utime.assert_called_once_with( 'test_o', (1454113727.682512, 1454113727.682512)) written_content.write.assert_called_once_with(b'objcontent') @@ -2023,7 +2238,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual(0, len(mock_utime.mock_calls)) written_content.write.assert_called_once_with(b'objcontent') @@ -2033,6 +2248,52 @@ class TestServiceDownload(_TestServiceBase): ) self.assertEqual(expected_r, actual_r) + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index fba9be7..8c995e5 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals + from genericpath import getmtime +import getpass import hashlib import json import logging @@ -27,6 +29,7 @@ from time import localtime, mktime, strftime, strptime from requests.packages.urllib3.exceptions import InsecureRequestWarning import six +import sys import swiftclient from swiftclient.service import SwiftError @@ -473,7 +476,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) makedirs.reset_mock() @@ -490,7 +493,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks @@ -507,7 +510,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) self.assertEqual([], makedirs.mock_calls) @@ -553,7 +556,7 @@ class TestShell(unittest.TestCase): mock_shuffle.assert_any_call(['container']) mock_shuffle.assert_any_call(['object']) mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -577,7 +580,7 @@ class TestShell(unittest.TestCase): argv = ["", "download", "--all", "--no-shuffle"] swiftclient.shell.main(argv) self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -610,7 +613,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([ mock.call('pseudo'), ], mock_mkdir.mock_calls) @@ -623,7 +626,8 @@ class TestShell(unittest.TestCase): connection.return_value.put_object.return_value = EMPTY_ETAG connection.return_value.attempts = 0 argv = ["", "upload", "container", self.tmpfile, - "-H", "X-Storage-Policy:one"] + "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] swiftclient.shell.main(argv) connection.return_value.put_container.assert_called_once_with( 'container', @@ -636,7 +640,8 @@ class TestShell(unittest.TestCase): mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, - 'X-Storage-Policy': 'one'}, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) # upload to pseudo-folder (via <container> param) @@ -717,10 +722,9 @@ class TestShell(unittest.TestCase): mock.ANY, headers={ 'x-object-meta-mtime': mock.ANY, - 'x-static-large-object': 'true' }, query_string='multipart-manifest=put', - response_dict={}) + response_dict=mock.ANY) @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): @@ -906,6 +910,44 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 0) @mock.patch('swiftclient.service.Connection') @@ -1625,7 +1667,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -1634,7 +1676,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True) + iso8601=False, prefix=True, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -1646,7 +1688,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -1655,7 +1697,7 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False) + iso8601=True, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1664,7 +1706,16 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1727,6 +1778,17 @@ class TestShell(unittest.TestCase): swiftclient.shell.main(argv) self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', @@ -2243,17 +2305,66 @@ class TestParsing(TestBase): os_opts = {"password": "secret", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) os_opts = {"username": "user", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) os_opts = {"username": "user", "password": "secret"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') def test_no_tenant_name_or_id_v2(self): os_opts = {"password": "secret", diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index d4a704e..009a026 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -14,6 +14,7 @@ # limitations under the License. import gzip +import json import logging import mock import six @@ -25,11 +26,13 @@ import tempfile from hashlib import md5 from six import binary_type from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, FakeKeystone, _make_fake_import_keystone_client) from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException from swiftclient import client as c import swiftclient.utils import swiftclient @@ -83,7 +86,7 @@ class TestClientException(unittest.TestCase): class MockHttpResponse(object): - def __init__(self, status=0, headers=None, verify=False, need_items=None): + def __init__(self, status=0, headers=None, verify=False): self.status = status self.status_code = status self.reason = "OK" @@ -92,7 +95,6 @@ class MockHttpResponse(object): self.verify = verify self.md5sum = md5() self.headers = {'etag': '"%s"' % EMPTY_ETAG} - self.need_items = need_items if headers: self.headers.update(headers) self.closed = False @@ -119,9 +121,7 @@ class MockHttpResponse(object): return self.headers.get(name, default) def getheaders(self): - if self.need_items: - return dict(self.headers).items() - return dict(self.headers) + return dict(self.headers).items() def fake_response(self): return self @@ -577,6 +577,73 @@ class TestGetAuth(MockHttpTest): self.assertTrue(url.startswith("http")) self.assertTrue(token) + def test_get_auth_keystone_versionless(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://authurl', 'user', 'key', {}) + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionless_auth_version_set(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://auth_url', 'user', 'key', + {}, auth_version='2.0') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v2.0', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + def test_auth_with_session(self): mock_session = mock.MagicMock() mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' @@ -1162,9 +1229,20 @@ class TestHeadObject(MockHttpTest): }), ]) + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + class TestPutObject(MockHttpTest): + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) @@ -1222,6 +1300,7 @@ class TestPutObject(MockHttpTest): self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, UserWarning)) + @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_server_error(self): body = 'c' * 60 headers = {'foo': 'bar'} @@ -1236,7 +1315,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', 'content-type': ''}), + 'x-auth-token': 'asdf', + 'content-type': ''}), ]) def test_query_string(self): @@ -1377,7 +1457,8 @@ class TestPutObject(MockHttpTest): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - def test_no_content_type(self): + @mock.patch('swiftclient.requests.__version__', '2.2.0') + def test_no_content_type_old_requests(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response @@ -1387,6 +1468,17 @@ class TestPutObject(MockHttpTest): request_header = resp.requests_params['headers'] self.assertEqual(request_header['content-type'], b'') + @mock.patch('swiftclient.requests.__version__', '2.4.0') + def test_no_content_type_new_requests(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn) + request_header = resp.requests_params['headers'] + self.assertNotIn('content-type', request_header) + def test_content_type_in_headers(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) @@ -1935,6 +2027,71 @@ class TestConnection(MockHttpTest): self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + def test_resp_read_on_server_error(self): conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) @@ -2319,7 +2476,7 @@ class TestConnection(MockHttpTest): return 'header' def getheaders(self): - return {"key1": "value1", "key2": "value2"} + return [('key1', 'value1'), ('key2', 'value2')] def read(self, *args, **kwargs): return '' @@ -2426,16 +2583,17 @@ class TestConnection(MockHttpTest): def test_head_object(self): headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): with mock.patch('swiftclient.client.get_auth', lambda *a, **k: ('http://url:8080/v1/a', 'token')): conn = c.Connection() conn.head_object('c1', 'o1', - headers=headers) + headers=headers, query_string=query_string) self.assertEqual(1, len(self.request_log), self.request_log) self.assertRequests([ - ('HEAD', '/v1/a/c1/o1', '', { + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', }), @@ -2569,6 +2727,33 @@ class TestLogging(MockHttpTest): c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') self.assertEqual(exc_context.exception.http_status, 404) + def test_content_encoding_gzip_body_is_logged_decoded(self): + buf = six.BytesIO() + gz = gzip.GzipFile(fileobj=buf, mode='w') + data = {"test": u"\u2603"} + decoded_body = json.dumps(data).encode('utf-8') + gz.write(decoded_body) + gz.close() + # stub a gzip encoded body + body = buf.getvalue() + headers = {'content-encoding': 'gzip'} + # ... and make a content-encoding gzip error response + stub_response = StubResponse(500, body, headers) + with mock.patch('swiftclient.client.logger.info') as mock_log: + # ... if the client gets such a response + c.http_connection = self.fake_http_connection(stub_response) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 500) + # it will log the decoded body + self.assertEqual([ + mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + '-X GET -H "X-Auth-Token: ..."'), + mock.call('RESP STATUS: %s %s', 500, 'Fake'), + mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), + mock.call('RESP BODY: %s', decoded_body) + ], mock_log.mock_calls) + def test_redact_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' @@ -2611,44 +2796,6 @@ class TestLogging(MockHttpTest): self.assertNotIn(unicode_token_value, output) self.assertNotIn(set_cookie_value, output) - def test_logging_body(self): - with mock.patch('swiftclient.client.logger.debug') as mock_log: - token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' - token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') - unicode_token_encoded = unicode_token_value.encode('utf8') - set_cookie_value = 'X-Auth-Token=%s' % token_value - set_cookie_encoded = set_cookie_value.encode('utf8') - buf = six.BytesIO() - gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) - gz.close() - c.http_log( - ['GET'], - {'headers': { - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded - }}, - MockHttpResponse( - status=200, - headers={ - 'X-Auth-Token': token_encoded, - 'X-Storage-Token': unicode_token_encoded, - 'content-encoding': 'gzip', - 'Etag': b'mock_etag', - 'Set-Cookie': set_cookie_encoded - }, - need_items=True, - ), - buf.getvalue(), - ) - self.assertEqual( - mock.call( - 'RESP BODY: %s', u'{"test": "\u2603"}'.encode('utf8')), - mock_log.mock_calls[3]) - def test_show_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index adead00..e54b90c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -152,6 +152,54 @@ class TestTempURL(unittest.TestCase): self.assertIsInstance(url, type(self.url)) @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + + @mock.patch('hmac.HMAC') def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, '2014-05-13T17:53:20Z', diff --git a/tests/unit/utils.py b/tests/unit/utils.py index c05146e..2def73f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -548,3 +548,24 @@ def _make_fake_import_keystone_client(fake_import): return fake_import, fake_import return _fake_import_keystone_client + + +class FakeStream(object): + def __init__(self, size): + self.bytes_read = 0 + self.size = size + + def read(self, size=-1): + if self.bytes_read == self.size: + return b'' + + if size == -1 or size + self.bytes_read > self.size: + remaining = self.size - self.bytes_read + self.bytes_read = self.size + return b'A' * remaining + + self.bytes_read += size + return b'A' * size + + def __len__(self): + return self.size diff --git a/tools/swift.bash_completion b/tools/swift.bash_completion new file mode 100644 index 0000000..2f98a6b --- /dev/null +++ b/tools/swift.bash_completion @@ -0,0 +1,32 @@ +declare -a _swift_opts # lazy init + +_swift_get_current_opt() +{ + local opt + for opt in ${_swift_opts[@]} ; do + if [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt\$") > 0 ]] || [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt ") > 0 ]] ; then + echo $opt + return 0 + fi + done + echo "" + return 0 +} + +_swift() +{ + local opt cur prev sflags + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [ "x$_swift_opts" == "x" ] ; then + _swift_opts=(`swift bash_completion "$sbc" | sed -e "s/-[-A-Za-z0-9_]*//g" -e "s/ */ /g"`) + fi + + opt="$(_swift_get_current_opt)" + COMPREPLY=($(compgen -W "$(swift bash_completion $opt)" -- ${cur})) + + return 0 +} +complete -F _swift swift diff --git a/tools/tox_install.sh b/tools/tox_install.sh deleted file mode 100755 index 15aa9de..0000000 --- a/tools/tox_install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Client constraint file contains this client version pin that is in conflict -# with installing the client from source. We should remove the version pin in -# the constraints file before applying it for from-source installation. - -set -e - -if [[ -z "$CONSTRAINTS_FILE" ]]; then - echo 'WARNING: expected $CONSTRAINTS_FILE to be set' >&2 - PIP_FLAGS=(-U) -else - # NOTE(tonyb): Place this in the tox enviroment's log dir so it will get - # published to logs.openstack.org for easy debugging. - localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - - if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" - fi - curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - - pip install -c"$localfile" openstack-requirements - - # This is the main purpose of the script: Allow local installation of - # the current repo. It is listed in constraints file and thus any - # install will be constrained and we need to unconstrain it. - edit-constraints "$localfile" -- "$CLIENT_NAME" - PIP_FLAGS=(-c"$localfile" -U) -fi - -pip install "${PIP_FLAGS[@]}" "$@" @@ -5,13 +5,11 @@ skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} +install_command = python -m pip install -U {opts} {packages} +list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} - BRANCH_NAME=master - CLIENT_NAME=python-swiftclient - CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -25,7 +23,7 @@ passenv = SWIFT_* *_proxy [testenv:pep8] commands = - flake8 swiftclient tests + python -m flake8 swiftclient tests [testenv:venv] commands = {posargs} @@ -35,9 +33,7 @@ commands = python setup.py testr --coverage coverage report [testenv:func] -setenv = - {[testenv]setenv} - OS_TEST_PATH=tests.functional +setenv = OS_TEST_PATH=tests.functional whitelist_externals = coverage rm @@ -71,3 +67,13 @@ exclude = .venv,.tox,dist,doc,*egg usedevelop = False deps = bindep commands = bindep test + +[testenv:releasenotes] +commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + -r{toxinidir}/test-requirements.txt + .[keystone] |