diff options
| -rw-r--r-- | AUTHORS | 7 | ||||
| -rw-r--r-- | ChangeLog | 28 | ||||
| -rw-r--r-- | README.rst | 2 | ||||
| -rw-r--r-- | bindep.txt | 5 | ||||
| -rw-r--r-- | doc/manpages/swift.1 | 12 | ||||
| -rw-r--r-- | doc/source/cli.rst | 442 | ||||
| -rw-r--r-- | doc/source/cli/index.rst | 960 | ||||
| -rw-r--r-- | doc/source/index.rst | 2 | ||||
| -rw-r--r-- | releasenotes/notes/340_notes-1777780bbfdb4d96.yaml | 20 | ||||
| -rw-r--r-- | releasenotes/source/conf.py | 356 | ||||
| -rw-r--r-- | releasenotes/source/current.rst | 5 | ||||
| -rw-r--r-- | releasenotes/source/index.rst | 11 | ||||
| -rw-r--r-- | releasenotes/source/newton.rst | 6 | ||||
| -rw-r--r-- | releasenotes/source/ocata.rst | 6 | ||||
| -rw-r--r-- | releasenotes/source/pike.rst | 6 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | swiftclient/client.py | 17 | ||||
| -rw-r--r-- | swiftclient/service.py | 25 | ||||
| -rwxr-xr-x | swiftclient/shell.py | 35 | ||||
| -rw-r--r-- | test-requirements.txt | 2 | ||||
| -rw-r--r-- | tests/unit/test_service.py | 52 | ||||
| -rw-r--r-- | tests/unit/test_shell.py | 53 | ||||
| -rw-r--r-- | tests/unit/test_swiftclient.py | 20 | ||||
| -rwxr-xr-x | tools/tox_install.sh | 41 | ||||
| -rw-r--r-- | tox.ini | 9 |
25 files changed, 1625 insertions, 499 deletions
@@ -65,11 +65,14 @@ Josh Gachnang (josh@pcsforeducation.com) Juan J. Martinez (juan@memset.com) Jude Job (judeopenstack@gmail.com) Julien Danjou (julien@danjou.info) +Kazufumi Noto (noto.kazufumi@gmail.com) Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) +liuyamin (liuyamin@fiberhome.com) Luis de Bethencourt (luis@debethencourt.com) +M V P Nitesh (m.nitesh@nectechnologies.in) Mahati Chamarthy (mahati.chamarthy@gmail.com) Marek Kaleta (marek.kaleta@firma.seznam.cz) Mark Seger (mark.seger@hpe.com) @@ -82,6 +85,7 @@ Min Min Ren (rminmin@cn.ibm.com) Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) +Nelson Marcos (nelsonmarcos@gmail.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -90,6 +94,7 @@ Paul Belanger (pabelanger@redhat.com) Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br) Pete Zaitcev (zaitcev@kotori.zaitcev.us) Peter Lisak (peter.lisak@firma.seznam.cz) +Petr Kovar (pkovar@redhat.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) Pratik Mallya (pratik.mallya@gmail.com) Qiu Yu (qiuyu@ebaysf.com) @@ -120,12 +125,14 @@ Thiago da Silva (thiago@redhat.com) Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) Tim Burke (tim.burke@gmail.com) +Timur Alperovich (timuralp@swiftstack.com) Tong Li (litong01@us.ibm.com) Tony Breeds (tony@bakeyournoodle.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) +Vitaly Gridnev (vgridnev@mirantis.com) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) YangLei (yanglyy@cn.ibm.com) @@ -1,3 +1,31 @@ +3.4.0 +----- + +* The `swift` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + `--object-name` must be given when content is loaded from stdin. + +* Tolerate RFC-compliant ETags returned from the server. + +* Skip checksum validation on partial downloads. + +* Buffer reads from disk, resulting in much faster upload throughput. + +* Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + +* Added an option to ignore mtime metadata entry (`--ignore-mtime`). + +* When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + +* Expose `--prefix` as an option for st_delete. + +* Imported docs content from openstack-manuals project. + +* Various other minor bug fixes and improvements. + 3.3.0 ----- @@ -43,7 +43,7 @@ __ http://github.com/openstack/swift * `How to Contribute`_ .. _PyPI: https://pypi.python.org/pypi/python-swiftclient -.. _Online Documentation: http://docs.openstack.org/developer/python-swiftclient +.. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient @@ -2,5 +2,6 @@ # see http://docs.openstack.org/infra/bindep/ for additional information. curl -pypy [test] -pypy-dev [test] +pypy [test !platform:fedora] +pypy-dev [test platform:dpkg] +pypy-devel [test platform:rpm !platform:fedora] diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 1f288d6..00e1440 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -63,8 +63,11 @@ Uploads to the given container the files and directories specified by the remaining args. The \-c or \-\-changed is an option that will only upload files that have changed since the last upload. The \-\-object\-name <object\-name> is an option that will upload file and name object to <object\-name> or upload dir -and use <object\-name> as object prefix. The \-S <size> or \-\-segment\-size <size> -and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more). +and use <object\-name> as object prefix. If the file name is "-", reads the +content from standard input. In this case, \-\-object\-name is required and no +other files may be given. The \-S <size> or \-\-segment\-size <size> and +\-\-leave\-segments and others are options as well (see swift upload \-\-help +for more). .RE \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] @@ -102,6 +105,9 @@ with \-\-no-download actually not to write anything to disk. The \-\-ignore-checksum is an option that turns off checksum validation. You can specify optional headers with the repeatable cURL-like option \-H [\-\-header]. For more details and options see swift download \-\-help. +The \-\-ignore\-mtime option ignores the x\-object\-meta\-mtime metadata entry +on the object (if present) and instead creates the downloaded files with +fresh atime and mtime values. .RE \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] @@ -205,4 +211,4 @@ swift \-A https://127.0.0.1:443/auth/v1.0 \-U swiftops:swiftops \-K swiftops sta .SH DOCUMENTATION .LP More in depth documentation about OpenStack Swift as a whole can be found at -.BI https://docs.openstack.org/developer/swift +.BI https://docs.openstack.org/swift/latest/ diff --git a/doc/source/cli.rst b/doc/source/cli.rst deleted file mode 100644 index 8d80d1b..0000000 --- a/doc/source/cli.rst +++ /dev/null @@ -1,442 +0,0 @@ -==== -CLI -==== - -The ``swift`` tool is a command line utility for communicating with an OpenStack -Object Storage (swift) environment. It allows one to perform several types of -operations. - -Authentication -~~~~~~~~~~~~~~ - -This section covers the options for authenticating with a swift -object store. The combinations of options required for each authentication -version are detailed below, but are just a subset of those that can be used -to successfully authenticate. These are the most common and recommended -combinations. - -You should obtain the details of your authentication version and credentials -from your storage provider. These details should make it clearer which of the -authentication sections below are most likely to allow you to connect to your -storage account. - -Keystone v3 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-name project1 --os-project-domain-name domain1 \ - --os-username user --os-user-domain-name domain1 \ - --os-password password list - - swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ - --os-project-id 0123456789abcdef0123456789abcdef \ - --os-user-id abcdef0123456789abcdef0123456789 \ - --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following combinations of environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=3 - OS_USERNAME=user - OS_USER_DOMAIN_NAME=domain1 - OS_PASSWORD=password - OS_PROJECT_NAME=project1 - OS_PROJECT_DOMAIN_NAME=domain1 - OS_AUTH_URL=https://api.example.com:5000/v3 - - ST_AUTH_VERSION=3 - OS_USER_ID=abcdef0123456789abcdef0123456789 - OS_PASSWORD=password - OS_PROJECT_ID=0123456789abcdef0123456789abcdef - OS_AUTH_URL=https://api.example.com:5000/v3 - -Keystone v2 ------------ - -.. code-block:: bash - - swift --os-auth-url https://api.example.com:5000/v2.0 \ - --os-tenant-name tenant \ - --os-username user --os-password password list - -Manually specifying the options above on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=2.0 - OS_USERNAME=user - OS_PASSWORD=password - OS_TENANT_NAME=tenant - OS_AUTH_URL=https://api.example.com:5000/v2.0 - -Legacy auth systems -------------------- - -You can configure swift to work with any number of other authentication systems -that we will not cover in this document. If your storage provider is not using -Keystone to provide access tokens, please contact them for instructions on the -required options. It is likely that the options will need to be specified as -below: - -.. code-block:: bash - - swift -A https://api.example.com/v1.0 -U user -K api_key list - -Specifying the options above manually on the command line can be avoided by -setting the following environment variables: - -.. code-block:: bash - - ST_AUTH_VERSION=1.0 - ST_AUTH=https://api.example.com/v1.0 - ST_USER=user - ST_KEY=key - -It is also possible that you need to use a completely separate auth system, in which -case ``swiftclient`` cannot request a token for you. In this case you should make the -authentication request separately and access your storage using the token and -storage URL options shown below: - -.. code-block:: bash - - swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ - list - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. - -CLI commands -~~~~~~~~~~~~ - -Stat ----- - - ``stat [container [object]]`` - - Displays information for the account, container, or object depending on - the arguments given (if any). In verbose mode, the storage URL and the - authentication token are displayed as well. - -List ----- - - ``list [command-options] [container]`` - - Lists the containers for the account or the objects for a container. - The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only - list items beginning with that prefix. The ``-d <delimiter>`` or - ``--delimiter <delimiter>`` is an option (for container listings only) - that will roll up items with the given delimiter (see `OpenStack Swift - general documentation <http://docs.openstack.org/developer/swift/>` for - what this means). - - The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` - and ``ls -lh``, the latter providing sizes in human readable format - (For example: ``3K``, ``12M``, etc). The latter two switches use more - overhead to retrieve the displayed details, which is directly proportional - to the number of container or objects listed. - -Upload ------- - - ``upload [command-options] container file_or_directory [file_or_directory] [...]`` - - Uploads the files and directories specified by the remaining arguments to the - given container. The ``-c`` or ``--changed`` is an option that will only - upload files that have changed since the last upload. The - ``--object-name <object-name>`` is an option that will upload a file and - name object to ``<object-name>`` or upload a directory and use ``<object-name>`` - as object prefix. The ``-S <size>`` or ``--segment-size <size>`` and - ``--leave-segments`` are options as well (see ``--help`` for more). - -Post ----- - - ``post [command-options] [container] [object]`` - - Updates meta information for the account, container, or object depending - on the arguments given. If the container is not found, the ``swiftclient`` - will create it automatically, but this is not true for accounts and - objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl - <read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. - The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, - and is used to define the user metadata items to set in the form ``Name:Value``. - You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` - - For more information about ACL formats see the documentation: - `ACLs <http://docs.openstack.org/developer/swift/misc.html#acls/>`_. - -Download --------- - - ``download [command-options] [container] [object] [object] [...]`` - - Downloads everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. For a - single object download, you may use the ``-o <filename>`` or ``--output <filename>`` - option to redirect the output to a specific file or ``-`` to - redirect to stdout. The ``--ignore-checksum`` is an option that turn off - checksum validation. You can specify optional headers with the repeatable - cURL-like option ``-H [--header <name:value>]``. - -Delete ------- - - ``delete [command-options] [container] [object] [object] [...]`` - - Deletes everything in the account (with ``--all``), or everything in a - container, or a list of objects depending on the arguments given. Segments - of manifest objects will be deleted as well, unless you specify the - ``--leave-segments`` option. - -Copy ----- - - ``copy [command-options] container object`` - - Copies an object to a new destination or adds user metadata to an object. Depending - on the options supplied, you can preserve existing metadata in contrast to the post - command. The ``--destination`` option sets the copy target destination in the form - ``/container/object``. If not set, the object will be copied onto itself which is useful - for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy - an object without existing user meta data, and the ``-m`` or ``--meta`` option - to define user meta data items to set in the form ``Name:Value``. You can repeat - this option. For example: ``copy -m Color:Blue -m Size:Large``. - -Capabilities ------------- - - ``capabilities [proxy-url]`` - - Displays cluster capabilities. The output includes the list of the - activated Swift middlewares as well as relevant options for each ones. - Additionally the command displays relevant options for the Swift core. If - the ``proxy-url`` option is not provided, the storage URL retrieved after - authentication is used as ``proxy-url``. - -Tempurl -------- - - ``tempurl [command-options] [method] [time] [path] [key]`` - - Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to - allow for this temporary URL that is usually ``GET` or ``PUT``. ``time`` option sets - the amount of time the temporary URL will be valid for. - ``time`` can be specified as an integer, denoting the number of seconds - from now on until the URL shall be valid; or, if ``--absolute`` - is passed, the Unix timestamp when the temporary URL will expire. - But beyond that, ``time`` can also be specified as an ISO 8601 timestamp - in one of following formats: - - i) Complete date: YYYY-MM-DD (eg 1997-07-16) - - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) - - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) - - Please be aware that if you don't provide the UTC designator (i.e., Z) - the timestamp is generated using your local timezone. If only a date is - specified, the time part used will equal to ``00:00:00``. - - ``path`` option sets the full path to the Swift object. - Example: ``/v1/AUTH_account/c/o``. ``key`` option is - the secret temporary URL key set on the Swift cluster. To set a key, run - ``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary - URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not - forget to append the desired objectname at the end of the path portion (and before the - query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the - URL by using the ``--iso8601`` option. - -Auth ----- - - ``auth`` - - Display authentication variables in shell friendly format. Command to run to export storage - URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. - Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic - authentication: ``swift auth -v -U test:tester -K testing``. - -Examples -~~~~~~~~ - -In this section we present some example usage of the ``swift`` CLI. To keep the -examples as short as possible, these examples assume that the relevant authentication -options have been set using environment variables. You can obtain the full list of -commands and options available in the ``swift`` CLI by executing the following: - -.. code-block:: bash - - > swift --help - > swift <command> --help - -Simple examples ---------------- - -List the existing swift containers: - -.. code-block:: bash - - > swift list - - container_1 - -Create a new container: - -.. code-block:: bash - - > swift post TestContainer - -Upload an object into a container: - -.. code-block:: bash - - > swift upload TestContainer testSwift.txt - - testSwift.txt - -List the contents of a container: - -.. code-block:: bash - - > swift list TestContainer - - testSwift.txt - -Copy an object to new destination: - -.. code-block:: bash - - > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt - - SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt - -Delete an object from a container: - -.. code-block:: bash - - > swift delete TestContainer testSwift.txt - - testSwift.txt - -Delete a container: - -.. code-block:: bash - - > swift delete TestContainer - - TestContainer - -Display auth related authentication variables in shell friendly format: - -.. code-block:: bash - - > swift auth - - export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 - export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 - -Download an object from a container: - -.. code-block:: bash - - > swift download TestContainer testSwift.txt - - testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] - -.. We need the backslash below in order to indent the note -\ - - .. note:: - - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name <object-name> is an option that will upload - file and name object to <object-name> or upload directory and use <object-name> as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. - -For example: - -.. code-block:: bash - - > swift upload TestContainer /home/swift/testSwift/testSwift.txt - - home/swift/testSwift/testSwift.txt - - > swift list TestContainer - - home/swift/testSwift/testSwift.txt - -More complex examples ---------------------- - -Swift has a single object size limit of 5GiB. In order to upload files larger -than this, we must create a large object that consists of smaller segments. -The example below shows how to upload a large video file as a static large -object in 1GiB segments: - -.. code-block:: bash - - > swift upload videos --use-slo --segment-size 1G myvideo.mp4 - - myvideo.mp4 segment 8 - myvideo.mp4 segment 4 - myvideo.mp4 segment 2 - myvideo.mp4 segment 7 - myvideo.mp4 segment 0 - myvideo.mp4 segment 1 - myvideo.mp4 segment 3 - myvideo.mp4 segment 6 - myvideo.mp4 segment 5 - myvideo.mp4 - -This command will upload segments to a container named ``videos_segments``, and -create a manifest file describing the entire object in the ``videos`` container. -For more information on large objects, see the documentation `here -<http://docs.openstack.org/developer/swift/overview_large_objects.html>`_. - -.. code-block:: bash - - > swift list videos - - myvideo.mp4 - - > swift list videos_segments - - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 - myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 - -Firstly, the key should be set, then generate a temporary URL for a Swift object: - -.. code-block:: bash - - > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" - - > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ - /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 - - /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ - 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst new file mode 100644 index 0000000..bec1f5e --- /dev/null +++ b/doc/source/cli/index.rst @@ -0,0 +1,960 @@ +==== +CLI +==== + +The ``swift`` tool is a command line utility for communicating with an OpenStack +Object Storage (swift) environment. It allows one to perform several types of +operations. + + +For help on a specific :command:`swift` command, enter: + +.. code-block:: console + + $ swift COMMAND --help + +.. _swift_command_usage: + +swift usage +~~~~~~~~~~~ + +.. code-block:: console + + Usage: swift [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth <auth_url>] + [--auth-version <auth_version> | + --os-identity-api-version <auth_version> ] + [--user <username>] + [--key <api_key>] [--retries <num_retries>] + [--os-username <auth-user-name>] [--os-password <auth-password>] + [--os-user-id <auth-user-id>] + [--os-user-domain-id <auth-user-domain-id>] + [--os-user-domain-name <auth-user-domain-name>] + [--os-tenant-id <auth-tenant-id>] + [--os-tenant-name <auth-tenant-name>] + [--os-project-id <auth-project-id>] + [--os-project-name <auth-project-name>] + [--os-project-domain-id <auth-project-domain-id>] + [--os-project-domain-name <auth-project-domain-name>] + [--os-auth-url <auth-url>] [--os-auth-token <auth-token>] + [--os-storage-url <storage-url>] [--os-region-name <region-name>] + [--os-service-type <service-type>] + [--os-endpoint-type <endpoint-type>] + [--os-cacert <ca-certificate>] [--insecure] + [--os-cert <client-certificate-file>] + [--os-key <client-certificate-key-file>] + [--no-ssl-compression] + <subcommand> [--help] [<subcommand options>] + +**Subcommands:** + +``delete`` + Delete a container or objects within a container. + +``download`` + Download objects from containers. + +``list`` + Lists the containers for the account or the objects + for a container. + +``post`` + Updates meta information for the account, container, + or object; creates containers if not present. + +``copy`` + Copies object, optionally adds meta + +``stat`` + Displays information for the account, container, + or object. + +``upload`` + Uploads files or directories to the given container. + +``capabilities`` + List cluster capabilities. + +``tempurl`` + Create a temporary URL. + +``auth`` + Display auth related environment variables. + +.. _swift_command_options: + +swift optional arguments +~~~~~~~~~~~~~~~~~~~~~~~~ + +``--version`` + show program's version number and exit + +``-h, --help`` + show this help message and exit + +``--os-help`` + Show OpenStack authentication options. + +``-s, --snet`` + Use SERVICENET internal network. + +``-v, --verbose`` + Print more info. + +``--debug`` + Show the curl commands and results of all http queries + regardless of result status. + +``--info`` + Show the curl commands and results of all http queries + which return an error. + +``-q, --quiet`` + Suppress status output. + +``-A AUTH, --auth=AUTH`` + URL for obtaining an auth token. + +``-V AUTH_VERSION, --auth-version=AUTH_VERSION, --os-identity-api-version=AUTH_VERSION`` + Specify a version for authentication. Defaults to + ``env[ST_AUTH_VERSION]``, ``env[OS_AUTH_VERSION]``, + ``env[OS_IDENTITY_API_VERSION]`` or 1.0. + +``-U USER, --user=USER`` + User name for obtaining an auth token. + +``-K KEY, --key=KEY`` + Key for obtaining an auth token. + +``-R RETRIES, --retries=RETRIES`` + The number of times to retry a failed connection. + +``--insecure`` + Allow swiftclient to access servers without having to + verify the SSL certificate. Defaults to + ``env[SWIFTCLIENT_INSECURE]`` (set to 'true' to enable). + +``--no-ssl-compression`` + This option is deprecated and not used anymore. SSL + compression should be disabled by default by the + system SSL library. + +Authentication +~~~~~~~~~~~~~~ + +This section covers the options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are just a subset of those that can be used +to successfully authenticate. These are the most common and recommended +combinations. + +You should obtain the details of your authentication version and credentials +from your storage provider. These details should make it clearer which of the +authentication sections below are most likely to allow you to connect to your +storage account. + +Keystone v3 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-name project1 --os-project-domain-name domain1 \ + --os-username user --os-user-domain-name domain1 \ + --os-password password list + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-id 0123456789abcdef0123456789abcdef \ + --os-user-id abcdef0123456789abcdef0123456789 \ + --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following combinations of environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=3 + OS_USERNAME=user + OS_USER_DOMAIN_NAME=domain1 + OS_PASSWORD=password + OS_PROJECT_NAME=project1 + OS_PROJECT_DOMAIN_NAME=domain1 + OS_AUTH_URL=https://api.example.com:5000/v3 + + ST_AUTH_VERSION=3 + OS_USER_ID=abcdef0123456789abcdef0123456789 + OS_PASSWORD=password + OS_PROJECT_ID=0123456789abcdef0123456789abcdef + OS_AUTH_URL=https://api.example.com:5000/v3 + +Keystone v2 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v2.0 \ + --os-tenant-name tenant \ + --os-username user --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=2.0 + OS_USERNAME=user + OS_PASSWORD=password + OS_TENANT_NAME=tenant + OS_AUTH_URL=https://api.example.com:5000/v2.0 + +Legacy auth systems +------------------- + +You can configure swift to work with any number of other authentication systems +that we will not cover in this document. If your storage provider is not using +Keystone to provide access tokens, please contact them for instructions on the +required options. It is likely that the options will need to be specified as +below: + +.. code-block:: bash + + swift -A https://api.example.com/v1.0 -U user -K api_key list + +Specifying the options above manually on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=1.0 + ST_AUTH=https://api.example.com/v1.0 + ST_USER=user + ST_KEY=key + +It is also possible that you need to use a completely separate auth system, in which +case ``swiftclient`` cannot request a token for you. In this case you should make the +authentication request separately and access your storage using the token and +storage URL options shown below: + +.. code-block:: bash + + swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ + list + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +CLI commands +~~~~~~~~~~~~ + +.. _swift_auth: + +Auth +---- + +.. code-block:: console + + Usage: swift auth + +Display authentication variables in shell friendly format. Command to run to export storage +URL and auth token into ``OS_STORAGE_URL`` and ``OS_AUTH_TOKEN``: ``swift auth``. +Command to append to a runcom file (e.g. ``~/.bashrc``, ``/etc/profile``) for automatic +authentication: ``swift auth -v -U test:tester -K testing``. + +.. _swift_stat: + +swift stat +---------- + +.. code-block:: console + + Usage: swift stat [--lh] [--header <header:value>] + [<container> [<object>]] + +Displays information for the account, container, or object depending on +the arguments given (if any). In verbose mode, the storage URL and the +authentication token are displayed as well. + +**Positional arguments:** + +``[container]`` + Name of container to stat from. + +``[object]`` + Name of object to stat. + +**Optional arguments:** + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-H, --header <header:value>`` + Adds a custom request header to use for stat. + +.. _swift_list: + +swift list +---------- + +.. code-block:: console + + Usage: swift list [--long] [--lh] [--totals] [--prefix <prefix>] + [--delimiter <delimiter>] [--header <header:value>] + [<container>] + +Lists the containers for the account or the objects for a container. +The ``-p <prefix>`` or ``--prefix <prefix>`` is an option that will only +list items beginning with that prefix. The ``-d <delimiter>`` or +``--delimiter <delimiter>`` is an option (for container listings only) +that will roll up items with the given delimiter (see `OpenStack Swift +general documentation <http://docs.openstack.org/swift/latest/>` for +what this means). + +The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` +and ``ls -lh``, the latter providing sizes in human readable format +(For example: ``3K``, ``12M``, etc). The latter two switches use more +overhead to retrieve the displayed details, which is directly proportional +to the number of container or objects listed. + +**Positional arguments:** + +``[container]`` + Name of container to list object in. + +**Optional arguments:** + +``-l, --long`` + Long listing format, similar to ls -l. + +``--lh`` + Report sizes in human readable format similar to + ls -lh. + +``-t, --totals`` + Used with -l or --lh, only report totals. + +``-p <prefix>, --prefix <prefix>`` + Only list items beginning with the prefix. + +``-d <delim>, --delimiter <delim>`` + Roll up items with the given delimiter. For containers + only. See OpenStack Swift API documentation for what + this means. + +``-H, --header <header:value>`` + Adds a custom request header to use for listing. + +.. _swift_upload: + +swift upload +------------ + +.. code-block:: console + + Usage: swift upload [--changed] [--skip-identical] [--segment-size <size>] + [--segment-container <container>] [--leave-segments] + [--object-threads <thread>] [--segment-threads <threads>] + [--header <header>] [--use-slo] [--ignore-checksum] + [--object-name <object-name>] + <container> <file_or_directory> [<file_or_directory>] [...] + +Uploads the files and directories specified by the remaining arguments to the +given container. The ``-c`` or ``--changed`` is an option that will only +upload files that have changed since the last upload. The +``--object-name <object-name>`` is an option that will upload a file and +name object to ``<object-name>`` or upload a directory and use ``<object-name>`` +as object prefix. If the file name is "-", client reads content from standard +input. In this case ``--object-name`` is required to set the name of the object +and no other files may be given. The ``-S <size>`` or ``--segment-size <size>`` +and ``--leave-segments`` are options as well (see ``--help`` for more). + +**Positional arguments:** + +``<container>`` + Name of container to upload to. + +``<file_or_directory>`` + Name of file or directory to upload. Specify multiple + times for multiple uploads. + +**Optional arguments:** + +``-c, --changed`` + Only upload files that have changed since the last + upload. + +``--skip-identical`` + Skip uploading files that are identical on both sides. + +``-S, --segment-size <size>`` + Upload files in segments no larger than <size> (in + Bytes) and then create a "manifest" file that will + download all the segments as if it were the original + file. + +``--segment-container <container>`` + Upload the segments into the specified container. If + not specified, the segments will be uploaded to a + <container>_segments container to not pollute the + main <container> listings. + +``--leave-segments`` + Indicates that you want the older segments of manifest + objects left alone (in the case of overwrites). + +``--object-threads <threads>`` + Number of threads to use for uploading full objects. + Default is 10. + +``--segment-threads <threads>`` + Number of threads to use for uploading object segments. + Default is 10. + +``-H, --header <header:value>`` + Adds a customized request header. This option may be + repeated. Example: -H "content-type:text/plain" + -H "Content-Length: 4000". + +``--use-slo`` + When used in conjunction with --segment-size it will + create a Static Large Object instead of the default + Dynamic Large Object. + +``--object-name <object-name>`` + Upload file and name object to <object-name> or upload + dir and use <object-name> as object prefix instead of + folder name. + +``--ignore-checksum`` + Turn off checksum validation for uploads. + + +.. _swift_post: + +swift post +---------- + +.. code-block:: console + + Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to] + [--sync-key <sync-key>] [--meta <name:value>] + [--header <header>] + [<container> [<object>]] + +Updates meta information for the account, container, or object depending +on the arguments given. If the container is not found, the ``swiftclient`` +will create it automatically, but this is not true for accounts and +objects. Containers also allow the ``-r <read-acl>`` (or ``--read-acl +<read-acl>``) and ``-w <write-acl>`` (or ``--write-acl <write-acl>``) options. +The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, +and is used to define the user metadata items to set in the form ``Name:Value``. +You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` + +For more information about ACL formats see the documentation: +`ACLs <http://docs.openstack.org/swift/latest/misc.html#acls>`_. + +**Positional arguments:** + +``[container]`` + Name of container to post to. + +``[object]`` + Name of object to post. + +**Optional arguments:** + +``-r, --read-acl <acl>`` + Read ACL for containers. Quick summary of ACL syntax: + ``.r:*``, ``.r:-.example.com``, ``.r:www.example.com``, + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-w, --write-acl <acl>`` + Write ACL for containers. Quick summary of ACL syntax: + ``account1`` (v1.0 identity API only), + ``account1:*``, ``account2:user2`` (v2.0+ identity API). + +``-t, --sync-to <sync-to>`` + Sync To for containers, for multi-cluster replication. + +``-k, --sync-key <sync-key>`` + Sync Key for containers, for multi-cluster replication. + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. + This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_download: + +swift download +-------------- + +.. code-block:: console + + Usage: swift download [--all] [--marker <marker>] [--prefix <prefix>] + [--output <out_file>] [--output-dir <out_directory>] + [--object-threads <threads>] [--ignore-checksum] + [--container-threads <threads>] [--no-download] + [--skip-identical] [--remove-prefix] + [--header <header:value>] [--no-shuffle] + [<container> [<object>] [...]] + +Downloads everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. For a +single object download, you may use the ``-o <filename>`` or ``--output <filename>`` +option to redirect the output to a specific file or ``-`` to +redirect to stdout. The ``--ignore-checksum`` is an option that turn off +checksum validation. You can specify optional headers with the repeatable +cURL-like option ``-H [--header <name:value>]``. ``--ignore-mtime`` ignores the +``x-object-meta-mtime`` metadata entry on the object (if present) and instead +creates the downloaded files with fresh atime and mtime values. + +**Positional arguments:** + +``<container>`` + Name of container to download from. To download a + whole account, omit this and specify --all. + +``<object>`` + Name of object to download. Specify multiple times + for multiple objects. Omit this to download all + objects from the container. + +**Optional arguments:** + +``-a, --all`` + Indicates that you really want to download + everything in the account. + +``-m, --marker <marker>`` + Marker to use when starting a container or account + download. + +``-p, --prefix <prefix>`` + Only download items beginning with <prefix> + +``-r, --remove-prefix`` + An optional flag for --prefix <prefix>, use this + option to download items without <prefix> + +``-o, --output <out_file>`` + For a single file download, stream the output to + <out_file>. Specifying "-" as <out_file> will + redirect to stdout. + +``-D, --output-dir <out_directory>`` + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. + +``--object-threads <threads>`` + Number of threads to use for downloading objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for downloading containers. + Default is 10. + +``--no-download`` + Perform download(s), but don't actually write anything + to disk. + +``-H, --header <header:value>`` + Adds a customized request header to the query, like + "Range" or "If-Match". This option may be repeated. + + Example: --header "content-type:text/plain" + +``--skip-identical`` + Skip downloading files that are identical on both + sides. + +``--ignore-checksum`` + Turn off checksum validation for downloads. + +``--no-shuffle`` + By default, when downloading a complete account or + container, download order is randomised in order to + reduce the load on individual drives when multiple + clients are executed simultaneously to download the + same set of objects (e.g. a nightly automated download + script to multiple servers). Enable this option to + submit download jobs to the thread pool in the order + they are listed in the object store. + +.. _swift_delete: + +swift delete +------------ + +.. code-block:: console + + Usage: swift delete [--all] [--leave-segments] + [--object-threads <threads>] + [--container-threads <threads>] + [--header <header:value>] + [<container> [<object>] [...]] + +Deletes everything in the account (with ``--all``), or everything in a +container, or a list of objects depending on the arguments given. Segments +of manifest objects will be deleted as well, unless you specify the +``--leave-segments`` option. + +**Positional arguments:** + +``[<container>]`` + Name of container to delete from. + +``[<object>]`` + Name of object to delete. Specify multiple times + for multiple objects. + +**Optional arguments:** + +``-a, --all`` + Delete all containers and objects. + +``--leave-segments`` + Do not delete segments of manifest objects. + +``-H, --header <header:value>`` + Adds a custom request header to use for deleting + objects or an entire container. + + +``--object-threads <threads>`` + Number of threads to use for deleting objects. + Default is 10. + +``--container-threads <threads>`` + Number of threads to use for deleting containers. + Default is 10. + +.. _swift_copy: + +swift copy +---------- + +.. code-block:: console + + Usage: swift copy [--destination </container/object>] [--fresh-metadata] + [--meta <name:value>] [--header <header>] <container> + <object> [<object>] [...] + +Copies an object to a new destination or adds user metadata to an object. Depending +on the options supplied, you can preserve existing metadata in contrast to the post +command. The ``--destination`` option sets the copy target destination in the form +``/container/object``. If not set, the object will be copied onto itself which is useful +for adding metadata. You can use the ``-M`` or ``--fresh-metadata`` option to copy +an object without existing user meta data, and the ``-m`` or ``--meta`` option +to define user meta data items to set in the form ``Name:Value``. You can repeat +this option. For example: ``copy -m Color:Blue -m Size:Large``. + +**Positional arguments:** + +``<container>`` + Name of container to copy from. + +``<object>`` + Name of object to copy. Specify multiple times for multiple objects + +**Optional arguments:** + +``-d, --destination </container[/object]>`` + The container and name of the destination object. Name + of destination object can be omitted, then will be + same as name of source object. Supplying multiple + objects and destination with object name is invalid. + +``-M, --fresh-metadata`` + Copy the object without any existing metadata, + If not set, metadata will be preserved or appended + +``-m, --meta <name:value>`` + Sets a meta data item. This option may be repeated. + + Example: -m Color:Blue -m Size:Large + +``-H, --header <header:value>`` + Adds a customized request header. This option may be repeated. + + Example: -H "content-type:text/plain" -H "Content-Length: 4000" + +.. _swift_capabilities: + +swift capabilities +------------------ + +.. code-block:: console + + Usage: swift capabilities [--json] [<proxy_url>] + +Displays cluster capabilities. The output includes the list of the +activated Swift middlewares as well as relevant options for each ones. +Additionally the command displays relevant options for the Swift core. If +the ``proxy-url`` option is not provided, the storage URL retrieved after +authentication is used as ``proxy-url``. + +**Optional positional arguments:** + +``<proxy_url>`` + Proxy URL of the cluster to retrieve capabilities. + +``--json`` + Print the cluster capabilities in JSON format. + +.. _swift_tempurl: + +swift tempurl +------------- + +.. code-block:: console + + Usage: swift tempurl [--absolute] [--prefix-based] + <method> <seconds> <path> <key> + +Generates a temporary URL for a Swift object. ``method`` option sets an HTTP method to +allow for this temporary URL that is usually ``GET`` or ``PUT``. ``time`` option sets +the amount of time the temporary URL will be valid for. +``time`` can be specified as an integer, denoting the number of seconds +from now on until the URL shall be valid; or, if ``--absolute`` +is passed, the Unix timestamp when the temporary URL will expire. +But beyond that, ``time`` can also be specified as an ISO 8601 timestamp +in one of following formats: + + i) Complete date: YYYY-MM-DD (eg 1997-07-16) + + ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (eg 1997-07-16T19:20:30) + + iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (eg 1997-07-16T19:20:30Z) + +Please be aware that if you don't provide the UTC designator (i.e., Z) +the timestamp is generated using your local timezone. If only a date is +specified, the time part used will equal to ``00:00:00``. + +``path`` option sets the full path to the Swift object. +Example: ``/v1/AUTH_account/c/o``. ``key`` option is +the secret temporary URL key set on the Swift cluster. To set a key, run +``swift post -m "Temp-URL-Key: <your secret key>"``. To generate a prefix-based temporary +URL use the ``--prefix-based`` option. This URL will contain the path to the prefix. Do not +forget to append the desired objectname at the end of the path portion (and before the +query portion) before sharing the URL. It is possible to use ISO 8601 UTC timestamps within the +URL by using the ``--iso8601`` option. + +**Positional arguments:** + +``<method>`` + An HTTP method to allow for this temporary URL. + Usually 'GET' or 'PUT'. + +``<seconds>`` + The amount of time in seconds the temporary URL will be + valid for; or, if --absolute is passed, the Unix + timestamp when the temporary URL will expire. + +``<path>`` + The full path to the Swift object. + + Example: /v1/AUTH_account/c/o + or: http://saio:8080/v1/AUTH_account/c/o + +``<key>`` + The secret temporary URL key set on the Swift cluster. + To set a key, run 'swift post -m + "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"' + +**Optional arguments:** + +``--absolute`` + Interpret the <seconds> positional argument as a Unix + timestamp rather than a number of seconds in the + future. + +``--prefix-based`` + If present, a prefix-based tempURL will be generated. + +Examples +~~~~~~~~ + +In this section we present some example usage of the ``swift`` CLI. To keep the +examples as short as possible, these examples assume that the relevant authentication +options have been set using environment variables. You can obtain the full list of +commands and options available in the ``swift`` CLI by executing the following: + +.. code-block:: bash + + > swift --help + > swift <command> --help + +Simple examples +--------------- + +List the existing swift containers: + +.. code-block:: bash + + > swift list + + container_1 + +Create a new container: + +.. code-block:: bash + + > swift post TestContainer + +Upload an object into a container: + +.. code-block:: bash + + > swift upload TestContainer testSwift.txt + + testSwift.txt + +List the contents of a container: + +.. code-block:: bash + + > swift list TestContainer + + testSwift.txt + +Copy an object to new destination: + +.. code-block:: bash + + > swift copy -d /DestContainer/testSwift.txt SourceContainer testSwift.txt + + SourceContainer/testSwift.txt copied to /DestContainer/testSwift.txt + +Delete an object from a container: + +.. code-block:: bash + + > swift delete TestContainer testSwift.txt + + testSwift.txt + +Delete a container: + +.. code-block:: bash + + > swift delete TestContainer + + TestContainer + +Display auth related authentication variables in shell friendly format: + +.. code-block:: bash + + > swift auth + + export OS_STORAGE_URL=http://127.0.0.1:8080/v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7 + export OS_AUTH_TOKEN=c597015ae19943a18438b52ef3762e79 + +Download an object from a container: + +.. code-block:: bash + + > swift download TestContainer testSwift.txt + + testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. + +For example: + +.. code-block:: bash + + > swift upload TestContainer /home/swift/testSwift/testSwift.txt + + home/swift/testSwift/testSwift.txt + + > swift list TestContainer + + home/swift/testSwift/testSwift.txt + +More complex examples +--------------------- + +Swift has a single object size limit of 5GiB. In order to upload files larger +than this, we must create a large object that consists of smaller segments. +The example below shows how to upload a large video file as a static large +object in 1GiB segments: + +.. code-block:: bash + + > swift upload videos --use-slo --segment-size 1G myvideo.mp4 + + myvideo.mp4 segment 8 + myvideo.mp4 segment 4 + myvideo.mp4 segment 2 + myvideo.mp4 segment 7 + myvideo.mp4 segment 0 + myvideo.mp4 segment 1 + myvideo.mp4 segment 3 + myvideo.mp4 segment 6 + myvideo.mp4 segment 5 + myvideo.mp4 + +This command will upload segments to a container named ``videos_segments``, and +create a manifest file describing the entire object in the ``videos`` container. +For more information on large objects, see the documentation `here +<https://docs.openstack.org/swift/latest/overview_large_objects.html>`_. + +.. code-block:: bash + + > swift list videos + + myvideo.mp4 + + > swift list videos_segments + + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 + +Firstly, the key should be set, then generate a temporary URL for a Swift object: + +.. code-block:: bash + + > swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4" + + > swift tempurl GET 6000 /v1/AUTH_bf5e63572f7a420a83fcf0aa8c72c2c7\ + /firstcontainer/clean.sh b3968d0207b54ece87cccc06515a89d4 + + /v1/AUTH_/firstcontainer/clean.sh?temp_url_sig=\ + 9218fc288cc09e5edd857b6a3d43cf2122b906dc&temp_url_expires=1472203614 diff --git a/doc/source/index.rst b/doc/source/index.rst index f123b7b..3c2cb1e 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,7 +16,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 - cli + cli/index service-api client-api diff --git a/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml new file mode 100644 index 0000000..0aae5cf --- /dev/null +++ b/releasenotes/notes/340_notes-1777780bbfdb4d96.yaml @@ -0,0 +1,20 @@ +--- +features: + + - The ``swift`` CLI now supports streaming from stdin. If "-" is given + as the source, the object content is read from stdin. The + ``--object-name`` must be given when content is loaded from stdin. + - Tolerate RFC-compliant ETags returned from the server. + - Skip checksum validation on partial downloads. + - Buffer reads from disk, resulting in much faster upload throughput. + - > + Added support for ISO 8601 timestamps for tempurl, matching the + feature in Swift 2.13.0. + - Added an option to ignore mtime metadata entry (``--ignore-mtime``). + - > + When using SwiftService to delete many objects, the bulk delete page + size will now be respected. Previously, exceeding this limit would + prevent any objects from being deleted. + - Expose `--prefix` as an option for st_delete. + - Imported docs content from openstack-manuals project. + - Various other minor bug fixes and improvements. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 0000000..b27aa96 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,356 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# swift documentation build configuration file, created by +# sphinx-quickstart on Mon Oct 3 17:01:55 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import datetime + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'reno.sphinxext', + 'openstackdocstheme', +] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Swift Client Release Notes' +copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year + +# Release notes are version independent. +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# +# today = '' +# +# Else, today_fmt is used as the format for a strftime call. +# +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +# todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# "<project> v<release> documentation" by default. +# +# html_title = u'swift v2.10.0' + +# A shorter title for the navigation bar. Default is the same as html_title. +# +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'SwiftClientReleaseNotesdoc' + +# -- Options for LaTeX output --------------------------------------------- + +# latex_elements = { +# # The paper size ('letterpaper' or 'a4paper'). +# # +# # 'papersize': 'letterpaper', + +# # The font size ('10pt', '11pt' or '12pt'). +# # +# # 'pointsize': '10pt', + +# # Additional stuff for the LaTeX preamble. +# # +# # 'preamble': '', + +# # Latex figure (float) alignment +# # +# # 'figure_align': 'htbp', +# } + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +# latex_documents = [ +# (master_doc, 'swift.tex', u'swift Documentation', +# u'swift', 'manual'), +# ] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# +# latex_use_parts = False + +# If true, show page references after internal links. +# +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# +# latex_appendices = [] + +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + +# If false, no module index is generated. +# +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +# man_pages = [ +# (master_doc, 'swift', u'swift Documentation', +# [author], 1) +# ] + +# If true, show URL addresses after external links. +# +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +# texinfo_documents = [ +# (master_doc, 'swift', u'swift Documentation', +# author, 'swift', 'One line description of project.', +# 'Miscellaneous'), +# ] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + +locale_dirs = ['locale/'] + +# -- Options for openstackdocstheme ------------------------------------------- +repository_name = 'openstack/python-swiftclient' +bug_project = 'python-swiftclient' +bug_tag = '' diff --git a/releasenotes/source/current.rst b/releasenotes/source/current.rst new file mode 100644 index 0000000..87a748f --- /dev/null +++ b/releasenotes/source/current.rst @@ -0,0 +1,5 @@ +==================================== + Current (Unreleased) Release Notes +==================================== + +.. release-notes:: diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 0000000..6e3419d --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,11 @@ +============================ + Swift Client Release Notes +============================ + +.. toctree:: + :maxdepth: 1 + + current + pike + ocata + newton diff --git a/releasenotes/source/newton.rst b/releasenotes/source/newton.rst new file mode 100644 index 0000000..59418a3 --- /dev/null +++ b/releasenotes/source/newton.rst @@ -0,0 +1,6 @@ +============================= + Newton Series Release Notes +============================= + +.. release-notes:: + :branch: stable/newton diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst new file mode 100644 index 0000000..726307b --- /dev/null +++ b/releasenotes/source/ocata.rst @@ -0,0 +1,6 @@ +============================ + Ocata Series Release Notes +============================ + +.. release-notes:: + :branch: stable/ocata diff --git a/releasenotes/source/pike.rst b/releasenotes/source/pike.rst new file mode 100644 index 0000000..e2c4806 --- /dev/null +++ b/releasenotes/source/pike.rst @@ -0,0 +1,6 @@ +=========================== + Pike Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/pike @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://docs.openstack.org/developer/python-swiftclient +home-page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology diff --git a/swiftclient/client.py b/swiftclient/client.py index 80bc4a3..7db75f0 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -25,7 +25,7 @@ from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError from six.moves import http_client from six.moves.urllib.parse import quote as _quote, unquote -from six.moves.urllib.parse import urlparse, urlunparse +from six.moves.urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time import six @@ -550,9 +550,22 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): insecure = kwargs.get('insecure', False) timeout = kwargs.get('timeout', None) - auth_version = kwargs.get('auth_version', '2.0') + auth_version = kwargs.get('auth_version', None) debug = logger.isEnabledFor(logging.DEBUG) + # Add the version suffix in case of versionless Keystone endpoints. If + # auth_version is also unset it is likely that it is v3 + if len(urlparse(auth_url).path) <= 1: + if auth_version and auth_version in AUTH_VERSIONS_V2: + auth_url = urljoin(auth_url, "v2.0") + else: + auth_url = urljoin(auth_url, "v3") + auth_version = '3' + logger.debug("Versionless auth_url - using %s as endpoint" % auth_url) + + # Legacy default if not set + if auth_version is None: + auth_version = 'v2.0' ksclient, exceptions = _import_keystone_client(auth_version) try: diff --git a/swiftclient/service.py b/swiftclient/service.py index 8301ae9..7b5ecd4 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -50,6 +50,7 @@ from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +DISK_BUFFER = 2 ** 16 logger = logging.getLogger("swiftclient.service") @@ -203,6 +204,7 @@ _default_local_options = { 'shuffle': False, 'destination': None, 'fresh_metadata': False, + 'ignore_mtime': False, } POLICY = 'X-Storage-Policy' @@ -1125,14 +1127,14 @@ class SwiftService(object): if options['skip_identical']: filename = out_file if out_file else path try: - fp = open(filename, 'rb') + fp = open(filename, 'rb', DISK_BUFFER) except IOError: pass else: with fp: md5sum = md5() while True: - data = fp.read(65536) + data = fp.read(DISK_BUFFER) if not data: break md5sum.update(data) @@ -1140,7 +1142,7 @@ class SwiftService(object): try: start_time = time() - get_args = {'resp_chunk_size': 65536, + get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} if options['skip_identical']: @@ -1223,10 +1225,10 @@ class SwiftService(object): if not no_file: if out_file: - fp = open(out_file, 'wb') + fp = open(out_file, 'wb', DISK_BUFFER) else: if basename(path): - fp = open(path, 'wb') + fp = open(path, 'wb', DISK_BUFFER) else: pseudodir = True @@ -1240,7 +1242,8 @@ class SwiftService(object): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if 'x-object-meta-mtime' in headers and not no_file: + if ('x-object-meta-mtime' in headers and not no_file + and not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: @@ -1731,7 +1734,7 @@ class SwiftService(object): } fp = None try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) @@ -1808,8 +1811,10 @@ class SwiftService(object): return chunks def _is_identical(self, chunk_data, path): + if path is None: + return False try: - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) except IOError: return False @@ -1818,7 +1823,7 @@ class SwiftService(object): to_read = chunk['bytes'] md5sum = md5() while to_read: - data = fp.read(min(65536, to_read)) + data = fp.read(min(DISK_BUFFER, to_read)) if not data: return False md5sum.update(data) @@ -2032,7 +2037,7 @@ class SwiftService(object): try: if path is not None: content_length = getsize(path) - fp = open(path, 'rb') + fp = open(path, 'rb', DISK_BUFFER) contents = LengthWrapper(fp, content_length, md5=options['checksum']) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 58b9f54..43fcf47 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,6 +17,7 @@ from __future__ import print_function, unicode_literals import argparse +import io import json import logging import signal @@ -26,7 +27,7 @@ from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 from six.moves.urllib.parse import unquote, urlparse -from sys import argv as sys_argv, exit, stderr +from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime from swiftclient import RequestException @@ -272,6 +273,9 @@ Optional arguments: script to multiple servers). Enable this option to submit download jobs to the thread pool in the order they are listed in the object store. + --ignore-mtime Ignore the 'X-Object-Meta-Mtime' header when + downloading an object. Instead, create atime and mtime + with fresh timestamps. '''.strip("\n") @@ -332,6 +336,12 @@ def st_download(parser, args, output_manager): 'nightly automated download script to multiple servers). Enable this ' 'option to submit download jobs to the thread pool in the order they ' 'are listed in the object store.') + parser.add_argument( + '--ignore-mtime', action='store_true', dest='ignore_mtime', + default=False, help='By default, the object-meta-mtime header is used ' + 'to store the access and modified timestamp for the downloaded file. ' + 'With this option, the header is ignored and the timestamps are ' + 'created freshly.') (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': @@ -892,7 +902,9 @@ Uploads specified files and directories to the given container. Positional arguments: <container> Name of container to upload to. <file_or_directory> Name of file or directory to upload. Specify multiple - times for multiple uploads. + times for multiple uploads. If "-" is specified, reads + content from standard input (--object-name is required + in this case). Optional arguments: -c, --changed Only upload files that have changed since the last @@ -1000,6 +1012,11 @@ def st_upload(parser, args, output_manager): else: container = args[0] files = args[1:] + from_stdin = '-' in files + if from_stdin and len(files) > 1: + output_manager.error( + 'upload from stdin cannot be used along with other files') + return if options['object_name'] is not None: if len(files) > 1: @@ -1007,6 +1024,10 @@ def st_upload(parser, args, output_manager): return else: orig_path = files[0] + elif from_stdin: + output_manager.error( + 'object-name must be specified with uploads from stdin') + return if options['segment_size']: try: @@ -1045,6 +1066,14 @@ def st_upload(parser, args, output_manager): objs = [] dir_markers = [] for f in files: + if f == '-': + fd = io.open(stdin.fileno(), mode='rb') + objs.append(SwiftUploadObject( + fd, object_name=options['object_name'])) + # We ensure that there is exactly one "file" to upload in + # this case -- stdin + break + if isfile(f): objs.append(f) elif isdir(f): @@ -1058,7 +1087,7 @@ def st_upload(parser, args, output_manager): # Now that we've collected all the required files and dir markers # build the tuples for the call to upload - if options['object_name'] is not None: + if options['object_name'] is not None and not from_stdin: objs = [ SwiftUploadObject( o, object_name=o.replace( diff --git a/test-requirements.txt b/test-requirements.txt index 0ce9cec..a9a0c7f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,5 @@ mock>=1.2 oslosphinx>=4.7.0 # Apache-2.0 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 +reno>=1.8.0,!=2.3.1 # Apache-2.0 +openstackdocstheme>=1.16.0 # Apache-2.0 diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 2a477fe..5ccc081 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1934,7 +1934,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) written_content.write.assert_called_once_with(b'objcontent') mock_conn.get_object.assert_called_once_with( @@ -1978,7 +1978,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) mock_utime.assert_called_once_with( 'test_o', (1454113727.682512, 1454113727.682512)) written_content.write.assert_called_once_with(b'objcontent') @@ -2024,7 +2024,7 @@ class TestServiceDownload(_TestServiceBase): 'headers_receipt': 3 } ) - mock_open.assert_called_once_with('test_o', 'wb') + mock_open.assert_called_once_with('test_o', 'wb', 65536) self.assertEqual(0, len(mock_utime.mock_calls)) written_content.write.assert_called_once_with(b'objcontent') @@ -2034,6 +2034,52 @@ class TestServiceDownload(_TestServiceBase): ) self.assertEqual(expected_r, actual_r) + def test_download_object_job_ignore_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + _opts['ignore_mtime'] = True + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb', 65536) + self.assertEqual([], mock_utime.mock_calls) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self.assertEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 3f87c6d..110fb01 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -27,6 +27,7 @@ from time import localtime, mktime, strftime, strptime from requests.packages.urllib3.exceptions import InsecureRequestWarning import six +import sys import swiftclient from swiftclient.service import SwiftError @@ -473,7 +474,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) makedirs.reset_mock() @@ -490,7 +491,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks @@ -507,7 +508,7 @@ class TestShell(unittest.TestCase): connection.return_value.get_object.assert_called_with( 'container', 'object', headers={}, resp_chunk_size=65536, response_dict={}) - mock_open.assert_called_with('object', 'wb') + mock_open.assert_called_with('object', 'wb', 65536) sr.assert_called_once_with('object', mock.ANY, mock.ANY, False) self.assertEqual([], makedirs.mock_calls) @@ -553,7 +554,7 @@ class TestShell(unittest.TestCase): mock_shuffle.assert_any_call(['container']) mock_shuffle.assert_any_call(['object']) mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -577,7 +578,7 @@ class TestShell(unittest.TestCase): argv = ["", "download", "--all", "--no-shuffle"] swiftclient.shell.main(argv) self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb') + mock_open.assert_called_once_with('container/object', 'wb', 65536) self.assertEqual([ mock.call('container'), mock.call('container/pseudo'), @@ -610,7 +611,7 @@ class TestShell(unittest.TestCase): response_dict={})] connection.return_value.get_object.assert_has_calls( calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + mock_open.assert_called_once_with('object', 'wb', 65536) self.assertEqual([ mock.call('pseudo'), ], mock_mkdir.mock_calls) @@ -721,7 +722,7 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY, }, query_string='multipart-manifest=put', - response_dict={}) + response_dict=mock.ANY) @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): @@ -907,6 +908,44 @@ class TestShell(unittest.TestCase): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.shell.io.open') + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin(self, upload_mock, io_open_mock): + def fake_open(fd, mode): + mock_io = mock.Mock() + mock_io.fileno.return_value = fd + return mock_io + + io_open_mock.side_effect = fake_open + + argv = ["", "upload", "container", "-", "--object-name", "foo"] + swiftclient.shell.main(argv) + upload_mock.assert_called_once_with("container", mock.ANY) + # This is a little convoluted: we want to examine the first call ([0]), + # the argv list([1]), the second parameter ([1]), and the first + # element. This is because the upload method takes a container and a + # list of SwiftUploadObjects. + swift_upload_obj = upload_mock.mock_calls[0][1][1][0] + self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_no_name(self, upload_mock): + argv = ["", "upload", "container", "-"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find('object-name must be specified') >= 0) + + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_from_stdin_and_others(self, upload_mock): + argv = ["", "upload", "container", "-", "foo", "--object-name", "bar"] + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, argv) + self.assertEqual(0, len(upload_mock.mock_calls)) + self.assertTrue(out.err.find( + 'upload from stdin cannot be used') >= 0) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 0) @mock.patch('swiftclient.service.Connection') diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 5384ac7..3de5f02 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -575,6 +575,26 @@ class TestGetAuth(MockHttpTest): self.assertTrue(url.startswith("http")) self.assertTrue(token) + def test_get_auth_keystone_versionless(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://authurl', 'user', 'key', {}) + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_versionless_auth_version_set(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://auth_url', 'user', 'key', + {}, auth_version='2.0') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v2.0', + fake_ks.calls[0].get('auth_url')) + def test_auth_with_session(self): mock_session = mock.MagicMock() mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' diff --git a/tools/tox_install.sh b/tools/tox_install.sh index 15aa9de..f3a83e9 100755 --- a/tools/tox_install.sh +++ b/tools/tox_install.sh @@ -4,28 +4,27 @@ # with installing the client from source. We should remove the version pin in # the constraints file before applying it for from-source installation. +CONSTRAINTS_FILE="$1" +shift 1 + set -e -if [[ -z "$CONSTRAINTS_FILE" ]]; then - echo 'WARNING: expected $CONSTRAINTS_FILE to be set' >&2 - PIP_FLAGS=(-U) -else - # NOTE(tonyb): Place this in the tox enviroment's log dir so it will get - # published to logs.openstack.org for easy debugging. - localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - - if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" - fi - curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - - pip install -c"$localfile" openstack-requirements - - # This is the main purpose of the script: Allow local installation of - # the current repo. It is listed in constraints file and thus any - # install will be constrained and we need to unconstrain it. - edit-constraints "$localfile" -- "$CLIENT_NAME" - PIP_FLAGS=(-c"$localfile" -U) +# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get +# published to logs.openstack.org for easy debugging. +localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + +if [[ "$CONSTRAINTS_FILE" != http* ]]; then + CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" fi +# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep +curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" + +python -m pip install -c"$localfile" openstack-requirements + +# This is the main purpose of the script: Allow local installation of +# the current repo. It is listed in constraints file and thus any +# install will be constrained and we need to unconstrain it. +python "$(which edit-constraints)" "$localfile" -- $CLIENT_NAME -pip install "${PIP_FLAGS[@]}" "$@" +python -m pip install -c"$localfile" -U "$@" +exit $? @@ -5,13 +5,13 @@ skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} +install_command = "{toxinidir}/tools/tox_install.sh" "{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}" {opts} {packages} +list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} BRANCH_NAME=master CLIENT_NAME=python-swiftclient - CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -25,7 +25,7 @@ passenv = SWIFT_* *_proxy [testenv:pep8] commands = - flake8 swiftclient tests + python -m flake8 swiftclient tests [testenv:venv] commands = {posargs} @@ -71,3 +71,6 @@ exclude = .venv,.tox,dist,doc,*egg usedevelop = False deps = bindep commands = bindep test + +[testenv:releasenotes] +commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html |